mdict_rs/
encoding.rs

1use std::borrow::Cow;
2
3use encoding_rs::{BIG5, GBK};
4
5use crate::error::{Error, Result};
6use crate::types::{ContainerKind, Header};
7
8#[derive(Debug, Clone, Copy, PartialEq, Eq)]
9pub enum TextEncoding {
10    Utf8,
11    Utf16Le,
12    Gbk,
13    Big5,
14}
15
16impl TextEncoding {
17    pub fn for_container(kind: ContainerKind, header: &Header) -> Result<Self> {
18        match kind {
19            ContainerKind::Mdd => Ok(Self::Utf16Le),
20            ContainerKind::Mdx => {
21                let label = header
22                    .encoding_label
23                    .as_deref()
24                    .unwrap_or("UTF-8")
25                    .trim()
26                    .to_ascii_uppercase();
27                match label.as_str() {
28                    "UTF-8" | "UTF8" => Ok(Self::Utf8),
29                    "UTF-16" | "UTF-16LE" | "UTF16" => Ok(Self::Utf16Le),
30                    "GBK" | "GB2312" | "GB18030" => Ok(Self::Gbk),
31                    "BIG5" => Ok(Self::Big5),
32                    _ => Err(Error::Unsupported("text encoding")),
33                }
34            }
35        }
36    }
37
38    pub fn unit_size(self) -> usize {
39        match self {
40            Self::Utf16Le => 2,
41            Self::Utf8 | Self::Gbk | Self::Big5 => 1,
42        }
43    }
44
45    pub fn decode(self, bytes: &[u8], context: &'static str) -> Result<String> {
46        match self {
47            Self::Utf8 => String::from_utf8(bytes.to_vec()).map_err(|_| Error::Decode {
48                context,
49                encoding: "utf-8",
50            }),
51            Self::Utf16Le => {
52                if bytes.len() % 2 != 0 {
53                    return Err(Error::Decode {
54                        context,
55                        encoding: "utf-16le",
56                    });
57                }
58                let units = bytes
59                    .chunks_exact(2)
60                    .map(|chunk| u16::from_le_bytes([chunk[0], chunk[1]]))
61                    .collect::<Vec<_>>();
62                String::from_utf16(&units).map_err(|_| Error::Decode {
63                    context,
64                    encoding: "utf-16le",
65                })
66            }
67            Self::Gbk => decode_encoding_rs(GBK, bytes, context, "gbk"),
68            Self::Big5 => decode_encoding_rs(BIG5, bytes, context, "big5"),
69        }
70    }
71
72    pub fn split_terminated<'a>(
73        self,
74        bytes: &'a [u8],
75        offset: usize,
76        context: &'static str,
77    ) -> Result<(&'a [u8], usize)> {
78        match self {
79            Self::Utf16Le => {
80                let mut index = offset;
81                while index + 1 < bytes.len() {
82                    if bytes[index] == 0 && bytes[index + 1] == 0 {
83                        return Ok((&bytes[offset..index], index + 2));
84                    }
85                    index += 2;
86                }
87                Err(Error::InvalidData(format!(
88                    "missing UTF-16 terminator for {context}"
89                )))
90            }
91            Self::Utf8 | Self::Gbk | Self::Big5 => {
92                let tail = bytes
93                    .get(offset..)
94                    .ok_or(Error::InvalidFormat("terminated string offset overflow"))?;
95                let rel = tail.iter().position(|byte| *byte == 0).ok_or_else(|| {
96                    Error::InvalidData(format!("missing terminator for {context}"))
97                })?;
98                let end = offset + rel;
99                Ok((&bytes[offset..end], end + 1))
100            }
101        }
102    }
103
104    pub fn normalize_key(self, key: &str, case_sensitive: bool, strip_key: bool) -> String {
105        let trimmed = if strip_key {
106            key.trim_matches(char::is_whitespace)
107        } else {
108            key
109        };
110        let without_nul = trimmed.trim_matches('\0');
111        if case_sensitive {
112            without_nul.to_owned()
113        } else {
114            without_nul.to_lowercase()
115        }
116    }
117}
118
119fn decode_encoding_rs(
120    encoding: &'static encoding_rs::Encoding,
121    bytes: &[u8],
122    context: &'static str,
123    name: &'static str,
124) -> Result<String> {
125    let decoded = encoding
126        .decode_without_bom_handling_and_without_replacement(bytes)
127        .ok_or(Error::Decode {
128            context,
129            encoding: name,
130        })?;
131    Ok(match decoded {
132        Cow::Borrowed(text) => text.to_owned(),
133        Cow::Owned(text) => text,
134    })
135}
mdict_rs/encoding.rs

mdict_rs/
encoding.rs
