charcode 0.1.5

Character encoding conversion implementing the WHATWG Encoding Standard, with no external dependencies.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141

//! Big5, including the Hong Kong Supplementary Character Set extensions.

use crate::ascii::ascii_prefix_len_capped;
use crate::index;
use crate::result::{DecoderResult, EncoderResult};
use crate::sink::{ByteSink, DECODER_HEADROOM, ENCODER_HEADROOM};
use crate::tables::big5::{
    BIG5_DECODE, BIG5_ENCODE_BUCKETS, BIG5_ENCODE_CODE_POINTS, BIG5_ENCODE_POINTERS,
};

/// The four pointers that decode to a base letter plus a combining mark; an index
/// entry can only hold a single code point, so the standard lists them separately.
fn combining_pair(pointer: usize) -> Option<(char, char)> {
    match pointer {
        1133 => Some(('\u{00CA}', '\u{0304}')),
        1135 => Some(('\u{00CA}', '\u{030C}')),
        1164 => Some(('\u{00EA}', '\u{0304}')),
        1166 => Some(('\u{00EA}', '\u{030C}')),
        _ => None,
    }
}

#[derive(Debug, Clone, Copy, Default)]
pub(crate) struct Big5Decoder {
    leading: u8,
}

impl Big5Decoder {
    pub(crate) fn new() -> Self {
        Self::default()
    }

    pub(crate) fn decode(
        &mut self,
        src: &[u8],
        sink: &mut ByteSink,
        last: bool,
    ) -> (DecoderResult, usize) {
        let mut read = 0usize;
        loop {
            if !sink.has_room(DECODER_HEADROOM) {
                return (DecoderResult::OutputFull, read);
            }

            if self.leading == 0 {
                let run = ascii_prefix_len_capped(&src[read..], sink.room());
                if run > 0 {
                    sink.write_slice(&src[read..read + run]);
                    read += run;
                    continue;
                }
            }

            let Some(byte) = src.get(read).copied() else {
                if !last {
                    return (DecoderResult::InputEmpty, read);
                }
                if self.leading == 0 {
                    return (DecoderResult::InputEmpty, read);
                }
                self.leading = 0;
                return (DecoderResult::Malformed(1), read);
            };
            read += 1;

            if self.leading != 0 {
                let leading = self.leading;
                self.leading = 0;
                let offset = if byte < 0x7F { 0x40 } else { 0x62 };
                let pointer = if (0x40..=0x7E).contains(&byte) || (0xA1..=0xFE).contains(&byte) {
                    Some((usize::from(leading) - 0x81) * 157 + (usize::from(byte) - offset))
                } else {
                    None
                };
                if let Some(pointer) = pointer {
                    if let Some((first, second)) = combining_pair(pointer) {
                        sink.write_char(first);
                        sink.write_char(second);
                        continue;
                    }
                    if let Some(code_point) = index::code_point_wide(&BIG5_DECODE, pointer) {
                        sink.write_code_point(code_point);
                        continue;
                    }
                }
                if byte.is_ascii() {
                    read -= 1;
                    return (DecoderResult::Malformed(1), read);
                }
                return (DecoderResult::Malformed(2), read);
            }

            if byte.is_ascii() {
                sink.write_byte(byte);
            } else if (0x81..=0xFE).contains(&byte) {
                self.leading = byte;
            } else {
                return (DecoderResult::Malformed(1), read);
            }
        }
    }
}

#[derive(Debug, Clone, Copy, Default)]
pub(crate) struct Big5Encoder;

impl Big5Encoder {
    pub(crate) fn encode(&mut self, src: &str, sink: &mut ByteSink) -> (EncoderResult, usize) {
        let mut read = 0usize;
        loop {
            if !sink.has_room(ENCODER_HEADROOM) {
                return (EncoderResult::OutputFull, read);
            }
            let rest = &src[read..];
            let Some(c) = rest.chars().next() else {
                return (EncoderResult::InputEmpty, read);
            };
            let run = ascii_prefix_len_capped(rest.as_bytes(), sink.room());
            if run > 0 {
                sink.write_slice(&rest.as_bytes()[..run]);
                read += run;
                continue;
            }
            read += c.len_utf8();

            let Some(pointer) = index::pointer_wide(
                &BIG5_ENCODE_CODE_POINTS,
                &BIG5_ENCODE_POINTERS,
                &BIG5_ENCODE_BUCKETS,
                u32::from(c),
            ) else {
                return (EncoderResult::Unmappable(c), read);
            };
            let pointer = u32::from(pointer);
            let leading = pointer / 157 + 0x81;
            let trailing = pointer % 157;
            let offset = if trailing < 0x3F { 0x40 } else { 0x62 };
            sink.write_slice(&[leading as u8, (trailing + offset) as u8]);
        }
    }
}