sql_dialect_fmt_encoding/
lib.rs

1//! Encoding boundary for formatter inputs.
2//!
3//! The lexer/parser operate on UTF-8 `&str`, but the CLI sees arbitrary bytes.
4//! This crate keeps that boundary explicit: known Unicode encodings are decoded
5//! losslessly, while unknown or malformed byte streams remain opaque so tools do
6//! not accidentally rewrite data they do not understand.
7//!
8//! The entry point is [`DecodedText::decode`], which sniffs a BOM, decodes when it can, and
9//! otherwise keeps the original bytes intact. Edit the recovered text with [`DecodedText::map_text`]
10//! and round-trip back to bytes with [`DecodedText::encode`]; the original encoding (and any BOM) is
11//! preserved end to end.
12//!
13//! ```
14//! use sql_dialect_fmt_encoding::{DecodedText, TextEncoding};
15//! let decoded = DecodedText::decode("select 1\n".as_bytes());
16//! assert_eq!(decoded.encoding(), TextEncoding::Utf8);
17//! let upper = decoded.map_text(|t| t.to_uppercase());
18//! assert_eq!(upper.encode(), b"SELECT 1\n");
19//! ```
20
21const UTF8_BOM: &[u8] = &[0xEF, 0xBB, 0xBF];
22const UTF16_LE_BOM: &[u8] = &[0xFF, 0xFE];
23const UTF16_BE_BOM: &[u8] = &[0xFE, 0xFF];
24
25/// A text encoding that [`DecodedText`] can recognize and round-trip.
26///
27/// `#[non_exhaustive]`: more encodings may be added in future releases, so match with a wildcard
28/// arm.
29#[derive(Clone, Copy, Debug, Eq, PartialEq)]
30#[non_exhaustive]
31pub enum TextEncoding {
32    /// UTF-8 with no byte-order mark.
33    Utf8,
34    /// UTF-8 prefixed with a byte-order mark, which is preserved on re-encoding.
35    Utf8Bom,
36    /// Little-endian UTF-16 (identified by its byte-order mark).
37    Utf16Le,
38    /// Big-endian UTF-16 (identified by its byte-order mark).
39    Utf16Be,
40    /// Bytes that could not be decoded as text and are passed through verbatim.
41    OpaqueBytes,
42}
43
44/// The result of decoding a byte stream: either recovered text in a known [`TextEncoding`], or the
45/// original bytes preserved verbatim because they could not be decoded.
46///
47/// Construct one with [`DecodedText::decode`]; it never loses data and [`DecodedText::encode`]
48/// reproduces the input.
49#[derive(Clone, Debug, Eq, PartialEq)]
50pub struct DecodedText {
51    kind: DecodedKind,
52}
53
54#[derive(Clone, Debug, Eq, PartialEq)]
55enum DecodedKind {
56    Text {
57        encoding: TextEncoding,
58        text: String,
59    },
60    Opaque {
61        bytes: Vec<u8>,
62        reason: OpaqueReason,
63    },
64}
65
66/// Why a byte stream was kept opaque instead of being decoded as text.
67///
68/// `#[non_exhaustive]`: more failure reasons may be added in future releases, so match with a
69/// wildcard arm.
70#[derive(Clone, Copy, Debug, Eq, PartialEq)]
71#[non_exhaustive]
72pub enum OpaqueReason {
73    /// The bytes were not valid UTF-8.
74    InvalidUtf8,
75    /// A UTF-16 stream had an odd number of bytes, so it could not be split into 16-bit units.
76    OddLengthUtf16,
77    /// The 16-bit units did not form valid UTF-16 (for example an unpaired surrogate).
78    InvalidUtf16,
79}
80
81impl DecodedText {
82    /// Decode `bytes`, sniffing a leading byte-order mark to pick the encoding. Valid UTF-8 or
83    /// UTF-16 is recovered as text; anything that fails to decode is preserved verbatim with an
84    /// [`OpaqueReason`]. Never fails and never loses data.
85    pub fn decode(bytes: &[u8]) -> Self {
86        if bytes.starts_with(UTF8_BOM) {
87            return decode_utf8(&bytes[UTF8_BOM.len()..], TextEncoding::Utf8Bom, bytes);
88        }
89        if bytes.starts_with(UTF16_LE_BOM) {
90            return decode_utf16(&bytes[UTF16_LE_BOM.len()..], TextEncoding::Utf16Le, bytes);
91        }
92        if bytes.starts_with(UTF16_BE_BOM) {
93            return decode_utf16(&bytes[UTF16_BE_BOM.len()..], TextEncoding::Utf16Be, bytes);
94        }
95        decode_utf8(bytes, TextEncoding::Utf8, bytes)
96    }
97
98    /// The encoding this text was decoded as, or [`TextEncoding::OpaqueBytes`] when the input could
99    /// not be decoded.
100    pub fn encoding(&self) -> TextEncoding {
101        match &self.kind {
102            DecodedKind::Text { encoding, .. } => *encoding,
103            DecodedKind::Opaque { .. } => TextEncoding::OpaqueBytes,
104        }
105    }
106
107    /// The decoded text, or `None` when the input was kept opaque (see [`Self::opaque_reason`]).
108    pub fn as_str(&self) -> Option<&str> {
109        match &self.kind {
110            DecodedKind::Text { text, .. } => Some(text),
111            DecodedKind::Opaque { .. } => None,
112        }
113    }
114
115    /// Why the input was kept opaque, or `None` when it decoded as text.
116    pub fn opaque_reason(&self) -> Option<OpaqueReason> {
117        match &self.kind {
118            DecodedKind::Text { .. } => None,
119            DecodedKind::Opaque { reason, .. } => Some(*reason),
120        }
121    }
122
123    /// Re-encode back to bytes in the original encoding (re-adding any BOM). For opaque input this
124    /// returns the preserved original bytes, so decode-then-encode is always a faithful round-trip.
125    pub fn encode(&self) -> Vec<u8> {
126        match &self.kind {
127            DecodedKind::Text { encoding, text } => encode_text(*encoding, text),
128            DecodedKind::Opaque { bytes, .. } => bytes.clone(),
129        }
130    }
131
132    /// Apply `edit` to the decoded text, keeping the original encoding. Opaque input is returned
133    /// unchanged, so transformations never run on bytes that could not be understood as text.
134    pub fn map_text(&self, edit: impl FnOnce(&str) -> String) -> Self {
135        match &self.kind {
136            DecodedKind::Text { encoding, text } => DecodedText {
137                kind: DecodedKind::Text {
138                    encoding: *encoding,
139                    text: edit(text),
140                },
141            },
142            DecodedKind::Opaque { .. } => self.clone(),
143        }
144    }
145}
146
147fn decode_utf8(bytes: &[u8], encoding: TextEncoding, original: &[u8]) -> DecodedText {
148    match std::str::from_utf8(bytes) {
149        Ok(text) => DecodedText {
150            kind: DecodedKind::Text {
151                encoding,
152                text: text.to_owned(),
153            },
154        },
155        Err(_) => opaque(original, OpaqueReason::InvalidUtf8),
156    }
157}
158
159fn decode_utf16(bytes: &[u8], encoding: TextEncoding, original: &[u8]) -> DecodedText {
160    if !bytes.len().is_multiple_of(2) {
161        return opaque(original, OpaqueReason::OddLengthUtf16);
162    }
163
164    let words = bytes.chunks_exact(2).map(|chunk| match encoding {
165        TextEncoding::Utf16Le => u16::from_le_bytes([chunk[0], chunk[1]]),
166        TextEncoding::Utf16Be => u16::from_be_bytes([chunk[0], chunk[1]]),
167        _ => unreachable!("decode_utf16 is only called for UTF-16 encodings"),
168    });
169
170    match String::from_utf16(&words.collect::<Vec<_>>()) {
171        Ok(text) => DecodedText {
172            kind: DecodedKind::Text { encoding, text },
173        },
174        Err(_) => opaque(original, OpaqueReason::InvalidUtf16),
175    }
176}
177
178fn encode_text(encoding: TextEncoding, text: &str) -> Vec<u8> {
179    match encoding {
180        TextEncoding::Utf8 => text.as_bytes().to_vec(),
181        TextEncoding::Utf8Bom => {
182            let mut bytes = Vec::with_capacity(UTF8_BOM.len() + text.len());
183            bytes.extend_from_slice(UTF8_BOM);
184            bytes.extend_from_slice(text.as_bytes());
185            bytes
186        }
187        TextEncoding::Utf16Le => {
188            let mut bytes = Vec::with_capacity(UTF16_LE_BOM.len() + text.len() * 2);
189            bytes.extend_from_slice(UTF16_LE_BOM);
190            for word in text.encode_utf16() {
191                bytes.extend_from_slice(&word.to_le_bytes());
192            }
193            bytes
194        }
195        TextEncoding::Utf16Be => {
196            let mut bytes = Vec::with_capacity(UTF16_BE_BOM.len() + text.len() * 2);
197            bytes.extend_from_slice(UTF16_BE_BOM);
198            for word in text.encode_utf16() {
199                bytes.extend_from_slice(&word.to_be_bytes());
200            }
201            bytes
202        }
203        TextEncoding::OpaqueBytes => unreachable!("opaque values are encoded from original bytes"),
204    }
205}
206
207fn opaque(bytes: &[u8], reason: OpaqueReason) -> DecodedText {
208    DecodedText {
209        kind: DecodedKind::Opaque {
210            bytes: bytes.to_vec(),
211            reason,
212        },
213    }
214}
215
216#[cfg(test)]
217mod tests {
218    use super::*;
219
220    #[test]
221    fn utf8_without_bom_round_trips() {
222        let bytes = "SELECT '長芋';\n".as_bytes();
223        let decoded = DecodedText::decode(bytes);
224
225        assert_eq!(decoded.encoding(), TextEncoding::Utf8);
226        assert_eq!(decoded.as_str(), Some("SELECT '長芋';\n"));
227        assert_eq!(decoded.encode(), bytes);
228    }
229
230    #[test]
231    fn utf8_bom_round_trips_and_preserves_bom() {
232        let mut bytes = UTF8_BOM.to_vec();
233        bytes.extend_from_slice("SELECT 1;\n".as_bytes());
234
235        let decoded = DecodedText::decode(&bytes);
236
237        assert_eq!(decoded.encoding(), TextEncoding::Utf8Bom);
238        assert_eq!(decoded.as_str(), Some("SELECT 1;\n"));
239        assert_eq!(decoded.encode(), bytes);
240    }
241
242    #[test]
243    fn utf16_le_round_trips_with_unicode() {
244        let text = "SELECT '長芋';\n";
245        let bytes = encode_text(TextEncoding::Utf16Le, text);
246
247        let decoded = DecodedText::decode(&bytes);
248
249        assert_eq!(decoded.encoding(), TextEncoding::Utf16Le);
250        assert_eq!(decoded.as_str(), Some(text));
251        assert_eq!(decoded.encode(), bytes);
252    }
253
254    #[test]
255    fn utf16_be_round_trips_with_unicode() {
256        let text = "SELECT '長芋';\n";
257        let bytes = encode_text(TextEncoding::Utf16Be, text);
258
259        let decoded = DecodedText::decode(&bytes);
260
261        assert_eq!(decoded.encoding(), TextEncoding::Utf16Be);
262        assert_eq!(decoded.as_str(), Some(text));
263        assert_eq!(decoded.encode(), bytes);
264    }
265
266    #[test]
267    fn opaque_invalid_utf8_preserves_original_bytes() {
268        let bytes = [0x53, 0x45, 0xFF, 0x4C];
269        let decoded = DecodedText::decode(&bytes);
270
271        assert_eq!(decoded.encoding(), TextEncoding::OpaqueBytes);
272        assert_eq!(decoded.as_str(), None);
273        assert_eq!(decoded.opaque_reason(), Some(OpaqueReason::InvalidUtf8));
274        assert_eq!(decoded.encode(), bytes);
275    }
276
277    #[test]
278    fn opaque_invalid_utf16_preserves_original_bytes() {
279        let bytes = [0xFF, 0xFE, 0x00];
280        let decoded = DecodedText::decode(&bytes);
281
282        assert_eq!(decoded.encoding(), TextEncoding::OpaqueBytes);
283        assert_eq!(decoded.opaque_reason(), Some(OpaqueReason::OddLengthUtf16));
284        assert_eq!(decoded.encode(), bytes);
285    }
286
287    #[test]
288    fn map_text_preserves_original_encoding() {
289        let source = encode_text(TextEncoding::Utf16Le, "select 1\n");
290        let decoded = DecodedText::decode(&source);
291
292        let edited = decoded.map_text(|text| text.to_uppercase());
293
294        assert_eq!(edited.encoding(), TextEncoding::Utf16Le);
295        assert_eq!(
296            DecodedText::decode(&edited.encode()).as_str(),
297            Some("SELECT 1\n")
298        );
299    }
300}
sql_dialect_fmt_encoding/lib.rs

sql_dialect_fmt_encoding/
lib.rs
