rustpython_common/encodings/
utf16.rs

1//! UTF-16 encode / incremental decode.
2
3use super::wide::{self, emit_utf16, is_surrogate, push_codepoint, read_u16, resolve_bom};
4use super::*;
5use crate::wtf8::Wtf8Buf;
6
7pub use super::wide::ByteOrder;
8
9pub const ENCODING_NAME: &str = "utf-16";
10pub const ENCODING_NAME_LE: &str = "utf-16-le";
11pub const ENCODING_NAME_BE: &str = "utf-16-be";
12
13const ERR_REASON: &str = "surrogates not allowed";
14
15pub fn encode<Ctx, E>(
16    ctx: Ctx,
17    errors: &E,
18    order: ByteOrder,
19    bom: bool,
20) -> Result<Vec<u8>, Ctx::Error>
21where
22    Ctx: EncodeContext,
23    E: EncodeErrorHandler<Ctx>,
24{
25    wide::encode_wide(ctx, errors, order.is_big_endian(), bom, 2, ERR_REASON)
26}
27
28/// Decode one incremental chunk.
29///
30/// Returns `(text, consumed, byteorder)` where `byteorder` is CPython's
31/// `-1` / `0` / `1`.
32pub fn decode<Ctx, E>(
33    mut ctx: Ctx,
34    errors: &E,
35    order: ByteOrder,
36    final_decode: bool,
37) -> Result<(Wtf8Buf, usize, i32), Ctx::Error>
38where
39    Ctx: DecodeContext,
40    E: DecodeErrorHandler<Ctx>,
41{
42    let (big_endian, skip, byteorder) = resolve_bom(ctx.remaining_data(), order, 2);
43    ctx.advance(skip);
44    let mut out = Wtf8Buf::new();
45    loop {
46        let rest = ctx.remaining_data();
47        if rest.len() < 2 {
48            if rest.is_empty() || !final_decode {
49                break;
50            }
51            let start = ctx.position();
52            let end = ctx.full_data().len();
53            let replace = ctx.handle_error(errors, start..end, Some("truncated data"))?;
54            out.push_wtf8(replace.as_ref());
55            continue;
56        }
57        let ch = read_u16(rest, big_endian);
58        if !is_surrogate(ch as u32) {
59            push_codepoint(&mut out, ch as u32);
60            ctx.advance(2);
61            continue;
62        }
63        if ch >= 0xdc00 {
64            let start = ctx.position();
65            let replace = ctx.handle_error(errors, start..start + 2, Some("illegal encoding"))?;
66            out.push_wtf8(replace.as_ref());
67            continue;
68        }
69        if rest.len() < 4 {
70            if !final_decode {
71                break;
72            }
73            let start = ctx.position();
74            let end = ctx.full_data().len();
75            let replace = ctx.handle_error(errors, start..end, Some("unexpected end of data"))?;
76            out.push_wtf8(replace.as_ref());
77            continue;
78        }
79        let ch2 = read_u16(&rest[2..], big_endian);
80        if (0xdc00..=0xdfff).contains(&ch2) {
81            let c = (((ch as u32 & 0x3ff) << 10) | (ch2 as u32 & 0x3ff)) + 0x10000;
82            push_codepoint(&mut out, c);
83            ctx.advance(4);
84        } else {
85            let start = ctx.position();
86            let replace =
87                ctx.handle_error(errors, start..start + 2, Some("illegal UTF-16 surrogate"))?;
88            out.push_wtf8(replace.as_ref());
89        }
90    }
91    Ok((out, ctx.position(), byteorder))
92}
93
94/// Encode a scalar (including a surrogate) as one or two UTF-16 units.
95pub fn encode_codepoint(out: &mut Vec<u8>, cp: u32, order: ByteOrder) {
96    emit_utf16(out, cp, order.is_big_endian());
97}
rustpython_common/encodings/utf16.rs

rustpython_common/encodings/
utf16.rs
