textcode 0.3.1

Text encoding/decoding library. Supports: UTF-8, ISO6937, ISO8859, GB2312
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100

//! UTF-16 text codec.
//!
//! Decoding rules:
//! - Detects an optional UTF-16 BOM (FE FF / FF FE).
//! - If no BOM is present, UTF-16BE is assumed.
//! - Valid UTF-16 code units and surrogate pairs are converted to UTF-8.
//! - Invalid sequences (orphan surrogates, malformed pairs, truncated data)
//!   are replaced with a fallback character ('?') and decoding continues.
//!
//! Encoding rules:
//! - Text is encoded as UTF-16BE without BOM.

use std::io::Write;

use crate::{
    Textcode,
    write_decode_fallback,
    write_utf8,
};

pub struct Utf16;

impl Textcode for Utf16 {
    fn decode<W: Write, R: AsRef<[u8]>>(src: R, dst: &mut W) -> std::io::Result<usize> {
        let src = src.as_ref();
        let mut written = 0;

        // Optional BOM
        let (be, mut i) = if src.len() >= 2 {
            match (src[0], src[1]) {
                (0xFE, 0xFF) => (true, 2),
                (0xFF, 0xFE) => (false, 2),
                _ => (true, 0), // default BE
            }
        } else {
            (true, 0)
        };

        while i < src.len() {
            if i + 2 > src.len() {
                // incomplete u16
                written += write_decode_fallback(dst)?;
                break;
            }

            let u1 = if be {
                u16::from_be_bytes([src[i], src[i + 1]])
            } else {
                u16::from_le_bytes([src[i], src[i + 1]])
            };
            i += 2;

            let cp: u32;

            if (0xD800 ..= 0xDBFF).contains(&u1) {
                // high surrogate: need low surrogate
                if i + 2 > src.len() {
                    // incomplete surrogate pair
                    written += write_decode_fallback(dst)?;
                    break;
                }

                let u2 = if be {
                    u16::from_be_bytes([src[i], src[i + 1]])
                } else {
                    u16::from_le_bytes([src[i], src[i + 1]])
                };

                if (0xDC00 ..= 0xDFFF).contains(&u2) {
                    i += 2;
                    cp = 0x10000 + (((u1 as u32 - 0xD800) << 10) | (u2 as u32 - 0xDC00));
                } else {
                    written += write_decode_fallback(dst)?;
                    continue;
                }
            } else if (0xDC00 ..= 0xDFFF).contains(&u1) {
                // orphan low surrogate
                written += write_decode_fallback(dst)?;
                continue;
            } else {
                cp = u1 as u32;
            };

            written += write_utf8(dst, cp)?;
        }

        Ok(written)
    }

    fn encode<W: Write, R: AsRef<str>>(src: R, dst: &mut W) -> std::io::Result<usize> {
        // UTF-16BE, without BOM
        let src = src.as_ref();
        let mut written = 0usize;
        for u in src.encode_utf16() {
            dst.write_all(&u.to_be_bytes())?;
            written += 2;
        }
        Ok(written)
    }
}