ferrijs_std/encoding/
mod.rs

1// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
2// SPDX-License-Identifier: Apache-2.0
3use std::borrow::Cow;
4
5use hex_simd::AsciiCase;
6
7#[derive(Clone, PartialEq)]
8pub enum Encoder {
9    Hex,
10    Base64,
11    /// WHATWG `windows-1252`: bytes 0x80-0x9F carry the punctuation
12    /// block, everything else is Latin-1. What `TextDecoder` uses for
13    /// this label family.
14    Windows1252,
15    /// Node's `latin1` / `binary` Buffer encoding: byte value IS the
16    /// code point, and encoding truncates anything above U+00FF.
17    Latin1,
18    /// Node's `ascii` Buffer encoding, which MASKS the high bit rather
19    /// than treating the byte as Latin-1.
20    Ascii,
21    Utf8,
22    Utf16le,
23    Utf16be,
24}
25
26/// Code points for bytes 0x80-0x9F under `windows-1252`. The five
27/// unassigned slots map to the C1 control of the same value, per the
28/// WHATWG index.
29const WINDOWS_1252_HIGH: [char; 32] = [
30    '\u{20AC}', '\u{0081}', '\u{201A}', '\u{0192}', '\u{201E}', '\u{2026}', '\u{2020}', '\u{2021}',
31    '\u{02C6}', '\u{2030}', '\u{0160}', '\u{2039}', '\u{0152}', '\u{008D}', '\u{017D}', '\u{008F}',
32    '\u{0090}', '\u{2018}', '\u{2019}', '\u{201C}', '\u{201D}', '\u{2022}', '\u{2013}', '\u{2014}',
33    '\u{02DC}', '\u{2122}', '\u{0161}', '\u{203A}', '\u{0153}', '\u{009D}', '\u{017E}', '\u{0178}',
34];
35
36fn windows_1252_to_string(bytes: &[u8]) -> String {
37    bytes
38        .iter()
39        .map(|b| match b {
40            0x80..=0x9F => WINDOWS_1252_HIGH[usize::from(b - 0x80)],
41            other => char::from(*other),
42        })
43        .collect()
44}
45
46fn string_to_windows_1252(string: &str) -> Vec<u8> {
47    string
48        .chars()
49        .map(|c| {
50            if let Some(index) = WINDOWS_1252_HIGH.iter().position(|high| *high == c) {
51                #[allow(clippy::cast_possible_truncation)]
52                return 0x80 + index as u8;
53            }
54            u8::try_from(u32::from(c)).unwrap_or(b'?')
55        })
56        .collect()
57}
58
59/// Node's Buffer encodings. Node and the WHATWG Encoding Standard
60/// disagree on what several labels MEAN — `latin1` is ISO-8859-1 for a
61/// Buffer but `windows-1252` for a `TextDecoder`, and `ascii` masks the
62/// high bit for a Buffer while decoding as `windows-1252` for a
63/// `TextDecoder` — so each consumer looks its label up in its own map.
64const NODE_ENCODING_MAP: phf::Map<&'static str, Encoder> = phf::phf_map! {
65    "buffer" => Encoder::Utf8,
66    "hex" => Encoder::Hex,
67    "base64" => Encoder::Base64,
68    "utf-8" => Encoder::Utf8,
69    "utf8" => Encoder::Utf8,
70    "ucs-2" => Encoder::Utf16le,
71    "ucs2" => Encoder::Utf16le,
72    "utf-16le" => Encoder::Utf16le,
73    "utf16le" => Encoder::Utf16le,
74    "latin1" => Encoder::Latin1,
75    "binary" => Encoder::Latin1,
76    "ascii" => Encoder::Ascii,
77};
78
79/// The WHATWG Encoding Standard's label index, for `TextDecoder`.
80const ENCODING_MAP: phf::Map<&'static str, Encoder> = phf::phf_map! {
81    "ascii" => Encoder::Windows1252,
82    "latin1" => Encoder::Windows1252,
83    "buffer" => Encoder::Utf8,
84    "hex" => Encoder::Hex,
85    "base64" => Encoder::Base64,
86    "unicode-1-1-utf-8" => Encoder::Utf8,
87    "unicode11utf8" => Encoder::Utf8,
88    "unicode20utf8" => Encoder::Utf8,
89    "utf-8" => Encoder::Utf8,
90    "utf8" => Encoder::Utf8,
91    "x-unicode20utf8" => Encoder::Utf8,
92    "csunicode" => Encoder::Utf16le,
93    "iso-10646-ucs-2" => Encoder::Utf16le,
94    "ucs-2" => Encoder::Utf16le,
95    "ucs2" => Encoder::Utf16le,
96    "unicode" => Encoder::Utf16le,
97    "unicodefeff" => Encoder::Utf16le,
98    "utf-16" => Encoder::Utf16le,
99    "utf-16le" => Encoder::Utf16le,
100    "utf16le" => Encoder::Utf16le,
101    "unicodefffe" => Encoder::Utf16be,
102    "utf-16be" => Encoder::Utf16be,
103    "ansi_x3.4-1968" => Encoder::Windows1252,
104    "cp1252" => Encoder::Windows1252,
105    "cp819" => Encoder::Windows1252,
106    "csisolatin1" => Encoder::Windows1252,
107    "ibm819" => Encoder::Windows1252,
108    "iso-8859-1" => Encoder::Windows1252,
109    "iso-ir-100" => Encoder::Windows1252,
110    "iso8859-1" => Encoder::Windows1252,
111    "iso88591" => Encoder::Windows1252,
112    "iso_8859-1" => Encoder::Windows1252,
113    "iso_8859-1:1987" => Encoder::Windows1252,
114    "l1" => Encoder::Windows1252,
115    "us-ascii" => Encoder::Windows1252,
116    "windows-1252" => Encoder::Windows1252,
117    "x-cp1252" => Encoder::Windows1252,
118};
119
120impl Encoder {
121    pub fn from_optional_str(encoding: Option<&str>) -> Result<Self, String> {
122        match encoding {
123            Some(label) if !label.is_empty() => Self::from_str(label),
124            _ => Ok(Self::Utf8),
125        }
126    }
127
128    /// A Node Buffer encoding name.
129    #[allow(clippy::should_implement_trait)]
130    pub fn from_str(encoding: &str) -> Result<Self, String> {
131        NODE_ENCODING_MAP
132            .get(encoding.trim_ascii().to_ascii_lowercase().as_str())
133            .cloned()
134            .ok_or_else(|| ["The \"", encoding, "\" encoding is not supported"].concat())
135    }
136
137    /// A WHATWG Encoding Standard label, as `TextDecoder` takes.
138    pub fn from_web_label(label: &str) -> Result<Self, String> {
139        ENCODING_MAP
140            .get(label.trim_ascii().to_ascii_lowercase().as_str())
141            .cloned()
142            .ok_or_else(|| ["The \"", label, "\" encoding is not supported"].concat())
143    }
144
145    /// A WHATWG label, defaulting to UTF-8 when absent or empty.
146    pub fn from_optional_web_label(label: Option<&str>) -> Result<Self, String> {
147        match label {
148            Some(label) if !label.is_empty() => Self::from_web_label(label),
149            _ => Ok(Self::Utf8),
150        }
151    }
152
153    pub fn encode_to_string(&self, bytes: &[u8], lossy: bool) -> Result<String, String> {
154        match self {
155            Self::Hex => Ok(bytes_to_hex_string(bytes)),
156            Self::Base64 => Ok(bytes_to_b64_string(bytes)),
157            Self::Utf8 => bytes_to_utf8_string(bytes, lossy),
158            Self::Windows1252 => Ok(windows_1252_to_string(bytes)),
159            Self::Latin1 => Ok(bytes.iter().map(|b| char::from(*b)).collect()),
160            Self::Ascii => Ok(bytes.iter().map(|b| char::from(b & 0x7F)).collect()),
161            Self::Utf16le => bytes_to_utf16_string(bytes, Endian::Little, lossy),
162            Self::Utf16be => bytes_to_utf16_string(bytes, Endian::Big, lossy),
163        }
164    }
165
166    #[allow(dead_code)]
167    pub fn encode(&self, bytes: &[u8]) -> Result<Vec<u8>, String> {
168        match self {
169            Self::Hex => Ok(bytes_to_hex(bytes)),
170            Self::Base64 => Ok(bytes_to_b64(bytes)),
171            Self::Utf8 | Self::Windows1252 | Self::Latin1 | Self::Ascii | Self::Utf16le | Self::Utf16be => {
172                Ok(bytes.to_vec())
173            },
174        }
175    }
176
177    pub fn decode<'a, T: Into<Cow<'a, [u8]>>>(&self, bytes: T) -> Result<Vec<u8>, String> {
178        match self {
179            Self::Hex => bytes_from_hex(bytes),
180            Self::Base64 => bytes_from_b64(bytes),
181            Self::Utf8 | Self::Windows1252 | Self::Latin1 | Self::Ascii | Self::Utf16le | Self::Utf16be => {
182                Ok(bytes.into().into())
183            },
184        }
185    }
186
187    pub fn decode_from_string(&self, string: String) -> Result<Vec<u8>, String> {
188        match self {
189            Self::Hex => bytes_from_hex(string.into_bytes()),
190            Self::Base64 => bytes_from_b64(string.into_bytes()),
191            Self::Utf8 => Ok(string.into_bytes()),
192            Self::Windows1252 => Ok(string_to_windows_1252(&string)),
193            // Node truncates rather than refusing: `Buffer.from('€',
194            // 'latin1')` is one byte, the low byte of the code point.
195            #[allow(clippy::cast_possible_truncation)]
196            Self::Latin1 => Ok(string.chars().map(|c| u32::from(c) as u8).collect()),
197            #[allow(clippy::cast_possible_truncation)]
198            Self::Ascii => Ok(string.chars().map(|c| (u32::from(c) as u8) & 0x7F).collect()),
199            Self::Utf16le => Ok(string
200                .encode_utf16()
201                .flat_map(|utf16| utf16.to_le_bytes())
202                .collect::<Vec<u8>>()),
203            Self::Utf16be => Ok(string
204                .encode_utf16()
205                .flat_map(|utf16| utf16.to_be_bytes())
206                .collect::<Vec<u8>>()),
207        }
208    }
209
210    pub fn as_label(&self) -> &str {
211        match self {
212            Self::Hex => "hex",
213            Self::Base64 => "base64",
214            Self::Windows1252 => "windows-1252",
215            Self::Latin1 => "latin1",
216            Self::Ascii => "ascii",
217            Self::Utf8 => "utf-8",
218            Self::Utf16le => "utf-16le",
219            Self::Utf16be => "utf-16be",
220        }
221    }
222}
223
224pub fn bytes_to_hex(bytes: &[u8]) -> Vec<u8> {
225    hex_simd::encode_type(bytes, AsciiCase::Lower)
226}
227
228pub fn bytes_from_hex<'a, T: Into<Cow<'a, [u8]>>>(hex_bytes: T) -> Result<Vec<u8>, String> {
229    hex_simd::decode_to_vec(hex_bytes.into()).map_err(|err| err.to_string())
230}
231
232pub fn bytes_from_b64<'a, T: Into<Cow<'a, [u8]>>>(base64_bytes: T) -> Result<Vec<u8>, String> {
233    let bytes: Cow<'a, [u8]> = base64_bytes.into();
234
235    //need to collect since memchr2_iter is borrowing bytes. This is fine since we're unlikely to contain url safe base64
236    let url_safe_byte_positions: Vec<usize> = memchr::memchr2_iter(b'-', b'_', &bytes).collect();
237
238    if url_safe_byte_positions.is_empty() {
239        return base64_simd::forgiving_decode_to_vec(&bytes).map_err(|e| e.to_string());
240    }
241
242    //doesn't allocate for already owned data
243    let mut bytes = bytes.into_owned();
244    for pos in url_safe_byte_positions {
245        bytes[pos] = match bytes[pos] {
246            b'-' => b'+',
247            b'_' => b'/',
248            _ => unreachable!(),
249        };
250    }
251    base64_simd::forgiving_decode_to_vec(&bytes).map_err(|e| e.to_string())
252}
253
254/// Strict standard-base64 decode (single SIMD pass): rejects url-safe chars,
255/// whitespace and bad padding, matching @smithy/util-base64 semantics.
256pub fn bytes_from_b64_strict(bytes: &[u8]) -> Result<Vec<u8>, String> {
257    base64_simd::STANDARD
258        .decode_to_vec(bytes)
259        .map_err(|e| e.to_string())
260}
261
262pub fn bytes_to_b64_string(bytes: &[u8]) -> String {
263    base64_simd::STANDARD.encode_to_string(bytes)
264}
265
266pub fn bytes_to_b64_url_safe_string(bytes: &[u8]) -> String {
267    base64_simd::URL_SAFE_NO_PAD.encode_to_string(bytes)
268}
269
270pub fn bytes_from_b64_url_safe(bytes: &[u8]) -> Result<Vec<u8>, String> {
271    base64_simd::URL_SAFE_NO_PAD
272        .decode_to_vec(bytes)
273        .map_err(|e| e.to_string())
274}
275
276pub fn bytes_to_b64(bytes: &[u8]) -> Vec<u8> {
277    base64_simd::STANDARD.encode_type(bytes)
278}
279
280pub fn bytes_to_hex_string(bytes: &[u8]) -> String {
281    hex_simd::encode_to_string(bytes, AsciiCase::Lower)
282}
283
284pub fn bytes_to_utf8_string(bytes: &[u8], lossy: bool) -> Result<String, String> {
285    if lossy {
286        Ok(String::from_utf8_lossy(bytes).to_string())
287    } else {
288        String::from_utf8(bytes.to_vec()).map_err(|e| e.to_string())
289    }
290}
291
292#[derive(Clone, Copy)]
293pub enum Endian {
294    Little,
295    Big,
296}
297
298pub fn bytes_to_utf16_string(bytes: &[u8], endian: Endian, lossy: bool) -> Result<String, String> {
299    if !lossy && !bytes.len().is_multiple_of(2) {
300        return Err("Input byte slice length must be even".to_string());
301    }
302
303    let data16: Vec<u16> = match endian {
304        Endian::Little => bytes
305            .as_chunks::<2>()
306            .0
307            .iter()
308            .copied()
309            .map(u16::from_le_bytes)
310            .collect(),
311        Endian::Big => bytes
312            .as_chunks::<2>()
313            .0
314            .iter()
315            .copied()
316            .map(u16::from_be_bytes)
317            .collect(),
318    };
319
320    let mut result = if lossy {
321        String::from_utf16_lossy(&data16)
322    } else {
323        String::from_utf16(&data16).map_err(|e| e.to_string())?
324    };
325
326    // Odd trailing byte in lossy mode produces a replacement character
327    if lossy && !bytes.len().is_multiple_of(2) {
328        result.push('\u{FFFD}');
329    }
330
331    Ok(result)
332}
333
334#[cfg(test)]
335mod tests {
336    use super::*;
337
338    #[test]
339    fn b64_strict_matches_smithy_semantics() {
340        // canonical decodes
341        assert_eq!(bytes_from_b64_strict(b"SGVsbG8=").unwrap(), b"Hello");
342        // url-safe, whitespace, bad-padding are rejected (like @smithy/util-base64)
343        assert!(bytes_from_b64_strict(b"-_8=").is_err());
344        assert!(bytes_from_b64_strict(b"SGVs bG8=").is_err());
345        assert!(bytes_from_b64_strict(b"SGVsbG8").is_err());
346    }
347}
ferrijs_std/encoding/mod.rs

ferrijs_std/encoding/
mod.rs
