wtf_string/encoding.rs
1// Copyright (c) 2026 Mike Grier
2//! The [`WtfEncoding`] storage seam and its [`Wtf16`] and [`Wtf8`] encodings.
3
4use alloc::string::String;
5use alloc::vec::Vec;
6
7/// A code-unit encoding for a [`WtfString`](crate::WtfString) / [`WtfStr`](crate::WtfStr).
8///
9/// This trait is the storage-width seam: the API common to every width is written
10/// against `E: WtfEncoding`, while width-specific API (such as the `*const u16`
11/// FFI surface) lives in inherent impls on the concrete instantiations. The two
12/// shipped encodings are [`Wtf16`] (`u16` units) and [`Wtf8`] (`u8` units); each
13/// defines its own encode/decode/comparison/formatting semantics over a
14/// crate-owned `Vec<Unit>` with the same always-terminated model.
15pub trait WtfEncoding {
16 /// The code unit this encoding stores (`u16` for [`Wtf16`]).
17 type Unit: Copy + Ord + core::hash::Hash + core::fmt::Debug;
18
19 /// The NUL code unit (`U+0000`) used as the always-present buffer terminator.
20 ///
21 /// Changing this value is a breaking change to the storage format.
22 const NUL: Self::Unit;
23
24 /// Encode a UTF-8 `str` into this encoding's code units.
25 fn encode_str(s: &str) -> Vec<Self::Unit>;
26
27 /// Decode content code units to a `String` if they are well-formed for this
28 /// encoding, or `None` if they are ill-formed (e.g. an unpaired surrogate),
29 /// which a strict `String` cannot represent.
30 fn decode(units: &[Self::Unit]) -> Option<String>;
31
32 /// Decode content code units to a `String`, replacing any ill-formed sequence
33 /// with the Unicode replacement character (`U+FFFD`).
34 fn decode_lossy(units: &[Self::Unit]) -> String;
35
36 /// Whether content code `units` equal the UTF-8 `str` `s` under this encoding.
37 ///
38 /// The default encodes `s` and compares slices; an encoding can override with
39 /// an allocation-free lazy comparison (as [`Wtf16`] does).
40 fn eq_str(units: &[Self::Unit], s: &str) -> bool {
41 units == Self::encode_str(s).as_slice()
42 }
43
44 /// Write the escaped debug form of `units`, like [`OsStr`](std::ffi::OsStr):
45 /// quoted, with control and non-printable characters escaped.
46 ///
47 /// The default decodes lossily and escapes; an encoding can override to also
48 /// escape *ill-formed* sequences losslessly (as [`Wtf16`] does for a lone
49 /// surrogate), so distinct ill-formed inputs remain distinguishable.
50 fn debug_fmt(units: &[Self::Unit], f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
51 write!(f, "{:?}", Self::decode_lossy(units))
52 }
53}
54
55/// The WTF-16 encoding: arbitrary, ill-formed-surrogate-tolerant UTF-16 stored as
56/// `u16` code units.
57///
58/// This is the v1 encoding and the representation Windows wide (`*W`) APIs consume
59/// directly. It is a pure type-level marker and is never constructed.
60pub enum Wtf16 {}
61
62impl WtfEncoding for Wtf16 {
63 type Unit = u16;
64 const NUL: u16 = 0;
65
66 fn encode_str(s: &str) -> Vec<u16> {
67 s.encode_utf16().collect()
68 }
69
70 fn decode(units: &[u16]) -> Option<String> {
71 String::from_utf16(units).ok()
72 }
73
74 fn decode_lossy(units: &[u16]) -> String {
75 String::from_utf16_lossy(units)
76 }
77
78 fn eq_str(units: &[u16], s: &str) -> bool {
79 // Compare against the lazily-encoded UTF-16 of `s`; no allocation.
80 units.iter().copied().eq(s.encode_utf16())
81 }
82
83 fn debug_fmt(units: &[u16], f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
84 use core::fmt::Write as _;
85 f.write_char('"')?;
86 for unit in core::char::decode_utf16(units.iter().copied()) {
87 match unit {
88 // An apostrophe is literal inside a double-quoted (string-style)
89 // debug; `char::escape_debug` would escape it like a char literal.
90 Ok('\'') => f.write_char('\'')?,
91 Ok(c) => {
92 for esc in c.escape_debug() {
93 f.write_char(esc)?;
94 }
95 }
96 // Preserve a lone surrogate losslessly as an escape, not U+FFFD.
97 Err(e) => write!(f, "\\u{{{:x}}}", e.unpaired_surrogate())?,
98 }
99 }
100 f.write_char('"')
101 }
102}
103
104/// The WTF-8 encoding: arbitrary, ill-formed-tolerant WTF-8 stored as `u8` code
105/// units -- the byte representation a Windows `OsStr` uses.
106///
107/// It matches `OsString`'s WTF-8 layout but is not built on `OsString` (D-3): the
108/// storage is a crate-owned `Vec<u8>`. Construction from units performs no
109/// validation, so the bytes may be well-formed WTF-8 (including encoded
110/// surrogates) or arbitrary. Like [`Wtf16`] it is a pure type-level marker and is
111/// never constructed.
112pub enum Wtf8 {}
113
114impl WtfEncoding for Wtf8 {
115 type Unit = u8;
116 const NUL: u8 = 0;
117
118 fn encode_str(s: &str) -> Vec<u8> {
119 // A UTF-8 `str` is already valid WTF-8: encoding is the identity on bytes.
120 s.as_bytes().to_vec()
121 }
122
123 fn decode(units: &[u8]) -> Option<String> {
124 // Exact decode succeeds only for valid UTF-8; WTF-8-encoded surrogates and
125 // arbitrary bytes are ill-formed for a strict `String`.
126 core::str::from_utf8(units).ok().map(String::from)
127 }
128
129 fn decode_lossy(units: &[u8]) -> String {
130 String::from_utf8_lossy(units).into_owned()
131 }
132
133 fn eq_str(units: &[u8], s: &str) -> bool {
134 // A `str`'s bytes are its WTF-8 encoding; compare without allocating.
135 units == s.as_bytes()
136 }
137
138 fn debug_fmt(units: &[u8], f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
139 use core::fmt::Write as _;
140 f.write_char('"')?;
141 // Split into maximal valid-UTF-8 runs and ill-formed byte runs: valid runs
142 // escape like a string, while each ill-formed byte is escaped losslessly
143 // as `\xNN`, so distinct byte inputs stay distinguishable.
144 for chunk in units.utf8_chunks() {
145 for c in chunk.valid().chars() {
146 // An apostrophe stays literal in string-style debug.
147 if c == '\'' {
148 f.write_char('\'')?;
149 } else {
150 for esc in c.escape_debug() {
151 f.write_char(esc)?;
152 }
153 }
154 }
155 for &b in chunk.invalid() {
156 write!(f, "\\x{b:02x}")?;
157 }
158 }
159 f.write_char('"')
160 }
161}