qubit_codec_text/codec/
utf8_codec.rs

1// =============================================================================
2//    Copyright (c) 2026 Haixing Hu.
3//
4//    SPDX-License-Identifier: Apache-2.0
5//
6//    Licensed under the Apache License, Version 2.0.
7// =============================================================================
8use crate::{
9    Charset,
10    CharsetCodec,
11    CharsetDecodeError,
12    CharsetDecodeErrorKind,
13    CharsetDecodeResult,
14    CharsetEncodeError,
15    CharsetEncodeErrorKind,
16    CharsetEncodeProbe,
17    CharsetEncodeResult,
18    Unicode,
19    Utf8,
20};
21use qubit_codec::Codec;
22
23/// UTF-8 byte-buffer charset codec.
24///
25/// # Examples
26///
27/// ```rust
28/// use qubit_codec_text::{
29///     CharsetCodec,
30///     CharsetEncodeProbe,
31///     Codec,
32///     Charset,
33///     Utf8,
34///     Utf8Codec,
35/// };
36///
37/// let codec = Utf8Codec;
38/// assert_eq!(Charset::UTF_8, codec.charset());
39/// assert_eq!(Utf8::MAX_UNITS_PER_CHAR, codec.max_units_per_value().get());
40///
41/// let mut output = [0_u8; Utf8::MAX_BYTES_PER_CHAR];
42/// let written = codec.encode_len('é', 0).expect("mappable");
43/// unsafe {
44///     codec.encode_unchecked(&'é', &mut output, 0).expect("buffer fits");
45/// }
46/// let (value, consumed) = unsafe {
47///     codec.decode_unchecked(&output[..written], 0).expect("valid UTF-8")
48/// };
49/// assert_eq!(('é', written), (value, consumed.get()));
50/// ```
51#[derive(Clone, Copy, Debug, Default, Eq, Hash, PartialEq)]
52pub struct Utf8Codec;
53
54impl Utf8Codec {
55    /// Returns the UTF-8 encoding descriptor.
56    ///
57    /// # Returns
58    ///
59    /// Returns [`Charset::UTF_8`].
60    #[must_use]
61    #[inline(always)]
62    pub const fn charset(self) -> Charset {
63        Charset::UTF_8
64    }
65}
66
67impl CharsetCodec for Utf8Codec {
68    /// Returns UTF-8 charset descriptor.
69    ///
70    /// # Returns
71    ///
72    /// Returns [`Charset::UTF_8`].
73    #[inline(always)]
74    fn charset(&self) -> Charset {
75        Charset::UTF_8
76    }
77}
78
79impl CharsetEncodeProbe for Utf8Codec {
80    /// Encodes one Unicode scalar value into UTF-8 bytes at `index`.
81    ///
82    /// # Arguments
83    ///
84    /// * `ch` - The Unicode scalar value to encode.
85    /// * `index` - Input character index used for error context.
86    ///
87    /// # Returns
88    ///
89    /// `Ok(usize)` with required encoded bytes (`1..=4`).
90    #[inline(always)]
91    fn encode_len(
92        &self,
93        ch: char,
94        _index: usize,
95    ) -> CharsetEncodeResult<usize> {
96        Ok(Utf8::byte_len(ch))
97    }
98}
99
100unsafe impl Codec for Utf8Codec {
101    type Value = char;
102    type Unit = u8;
103    type DecodeError = CharsetDecodeError;
104    type EncodeError = CharsetEncodeError;
105
106    #[inline(always)]
107    fn min_units_per_value(&self) -> core::num::NonZeroUsize {
108        core::num::NonZeroUsize::MIN
109    }
110
111    #[inline(always)]
112    fn max_units_per_value(&self) -> core::num::NonZeroUsize {
113        // SAFETY: UTF-8 encodes every scalar value as at least one byte.
114        unsafe {
115            core::num::NonZeroUsize::new_unchecked(Utf8::MAX_UNITS_PER_CHAR)
116        }
117    }
118
119    #[inline(always)]
120    unsafe fn decode_unchecked(
121        &self,
122        input: &[u8],
123        index: usize,
124    ) -> CharsetDecodeResult<(char, core::num::NonZeroUsize)> {
125        let (ch, consumed) = decode_prefix(input, index)?;
126        debug_assert!(consumed.get() <= input.len() - index);
127        Ok((ch, consumed))
128    }
129
130    #[inline(always)]
131    unsafe fn encode_unchecked(
132        &self,
133        ch: &char,
134        output: &mut [u8],
135        index: usize,
136    ) -> CharsetEncodeResult<usize> {
137        let written = encode_char(*ch, output, index)?;
138        debug_assert_eq!(written, Utf8::byte_len(*ch));
139        debug_assert!(written <= output.len() - index);
140        Ok(written)
141    }
142}
143
144/// Decodes the first UTF-8 character from a closed byte slice starting at
145/// `index`.
146///
147/// The caller normally provides at least `Utf8::MAX_UNITS_PER_CHAR` readable
148/// bytes from `index`. If fewer bytes are present, this function treats the
149/// slice as closed at EOF: complete shorter UTF-8 sequences still decode, while
150/// truncated sequences return [`CharsetDecodeErrorKind::IncompleteSequence`].
151///
152/// # Arguments
153///
154/// * `input` - UTF-8 byte slice to decode from.
155/// * `index` - Start offset in `input`; must be `<= input.len()`.
156///
157/// # Returns
158///
159/// Returns the decoded character and the non-zero number of consumed bytes.
160///
161/// # Errors
162///
163/// * `CharsetDecodeErrorKind::InvalidInputIndex` when `index` is greater than
164///   `input.len()`.
165/// * `CharsetDecodeErrorKind::MalformedSequence` when the first byte or
166///   continuation bytes are invalid for UTF-8.
167/// * `CharsetDecodeErrorKind::IncompleteSequence` when EOF appears before the
168///   complete UTF-8 sequence is available.
169#[inline]
170fn decode_prefix(
171    input: &[u8],
172    index: usize,
173) -> CharsetDecodeResult<(char, core::num::NonZeroUsize)> {
174    if index > input.len() {
175        let kind = CharsetDecodeErrorKind::InvalidInputIndex {
176            input_len: input.len(),
177        };
178        return Err(CharsetDecodeError::new(Charset::UTF_8, kind, index));
179    }
180    if index == input.len() {
181        let kind = CharsetDecodeErrorKind::IncompleteSequence {
182            required: 1,
183            available: 0,
184        };
185        return Err(CharsetDecodeError::new(Charset::UTF_8, kind, index));
186    }
187    let first = input[index];
188    let length = match Utf8::byte_len_from_leading_byte(first) {
189        Some(length) => length,
190        None => {
191            let kind = CharsetDecodeErrorKind::MalformedSequence {
192                value: Some(first as u32),
193            };
194            return Err(CharsetDecodeError::new(Charset::UTF_8, kind, index));
195        }
196    };
197    if !has_units(input.len(), index, length) {
198        validate_partial(input, index)?;
199        let kind = CharsetDecodeErrorKind::IncompleteSequence {
200            required: length,
201            available: input.len() - index,
202        };
203        return Err(CharsetDecodeError::new(Charset::UTF_8, kind, index));
204    }
205    let code_point = match length {
206        1 => first as u32,
207        2 => decode_two(input, index)?,
208        3 => decode_three(input, index)?,
209        4 => decode_four(input, index)?,
210        _ => unreachable!("UTF-8 sequence length is limited to four bytes"),
211    };
212    let ch = Unicode::to_char(code_point)
213        .expect("well-formed UTF-8 decodes to a Unicode scalar");
214    Ok((
215        ch,
216        core::num::NonZeroUsize::new(length)
217            .expect("well-formed UTF-8 sequence has non-zero length"),
218    ))
219}
220
221/// Encodes one Unicode scalar value into UTF-8 at `index` in `output`.
222///
223/// The function writes the byte sequence for `ch` and returns how many bytes
224/// were written.
225///
226/// # Arguments
227///
228/// * `ch` - The character to encode.
229/// * `output` - Destination buffer.
230/// * `index` - Start offset in `output`; must satisfy `index <= output.len()`.
231///
232/// # Returns
233///
234/// `Ok(usize)` with the number of UTF-8 bytes written (`1..=4`).
235///
236/// # Errors
237///
238/// * `CharsetEncodeErrorKind::BufferTooSmall` if the destination does not have
239///   enough space starting from `index`.
240#[inline]
241fn encode_char(
242    ch: char,
243    output: &mut [u8],
244    index: usize,
245) -> CharsetEncodeResult<usize> {
246    if index > output.len() {
247        let kind = CharsetEncodeErrorKind::BufferTooSmall {
248            required: required_index(index, 1),
249            available: 0,
250        };
251        return Err(CharsetEncodeError::new(Charset::UTF_8, kind, index));
252    }
253    let length = Utf8::byte_len(ch);
254    let available = output.len() - index;
255    if available < length {
256        let kind = CharsetEncodeErrorKind::BufferTooSmall {
257            required: required_index(index, length),
258            available,
259        };
260        return Err(CharsetEncodeError::new(Charset::UTF_8, kind, index));
261    }
262    let mut scratch = [0_u8; Utf8::MAX_BYTES_PER_CHAR];
263    let encoded = ch.encode_utf8(&mut scratch);
264    output[index..index + length].copy_from_slice(encoded.as_bytes());
265    Ok(length)
266}
267
268#[inline(always)]
269const fn has_units(len: usize, index: usize, required_units: usize) -> bool {
270    match index.checked_add(required_units) {
271        Some(end) => len >= end,
272        None => false,
273    }
274}
275
276#[inline(always)]
277const fn required_index(index: usize, required_units: usize) -> usize {
278    match index.checked_add(required_units) {
279        Some(required) => required,
280        None => usize::MAX,
281    }
282}
283
284/// Decodes a two-byte UTF-8 sequence starting at `index`.
285///
286/// # Arguments
287///
288/// * `input` - Byte slice containing the sequence.
289/// * `index` - Start offset of a two-byte leading byte.
290///
291/// # Returns
292///
293/// The decoded Unicode scalar value as a `u32` on success.
294///
295/// # Errors
296///
297/// * `CharsetDecodeErrorKind::MalformedSequence` when the second byte is not a
298///   valid UTF-8 continuation byte.
299#[inline]
300fn decode_two(input: &[u8], index: usize) -> CharsetDecodeResult<u32> {
301    let second = input[index + 1];
302    if !Utf8::is_continuation_byte(second) {
303        let kind = CharsetDecodeErrorKind::MalformedSequence {
304            value: Some(second as u32),
305        };
306        return Err(CharsetDecodeError::new(
307            Charset::UTF_8,
308            kind,
309            required_index(index, 1),
310        )
311        .with_consumed(2));
312    }
313    Ok((((input[index] & 0x1f) as u32) << 6) | ((second & 0x3f) as u32))
314}
315
316/// Validates the bytes already present in an incomplete UTF-8 prefix.
317///
318/// This is used after the total sequence length is known, to catch malformed
319/// continuation bytes before more data arrives.
320///
321/// # Arguments
322///
323/// * `input` - Prefix slice being decoded.
324/// * `index` - Start offset of the current UTF-8 sequence.
325///
326/// # Returns
327///
328/// `Ok(())` if currently available bytes are structurally valid, otherwise a
329/// decoding error describing the first malformed position.
330#[inline]
331fn validate_partial(input: &[u8], index: usize) -> CharsetDecodeResult<()> {
332    if has_units(input.len(), index, 2)
333        && !is_valid_second_byte(input[index], input[index + 1])
334    {
335        let kind = CharsetDecodeErrorKind::MalformedSequence {
336            value: Some(input[index + 1] as u32),
337        };
338        return Err(CharsetDecodeError::new(
339            Charset::UTF_8,
340            kind,
341            required_index(index, 1),
342        )
343        .with_consumed(2));
344    }
345    if has_units(input.len(), index, 3)
346        && !Utf8::is_continuation_byte(input[index + 2])
347    {
348        let kind = CharsetDecodeErrorKind::MalformedSequence {
349            value: Some(input[index + 2] as u32),
350        };
351        return Err(CharsetDecodeError::new(
352            Charset::UTF_8,
353            kind,
354            required_index(index, 2),
355        )
356        .with_consumed(3));
357    }
358    Ok(())
359}
360
361/// Checks whether `second` is legal for a UTF-8 leading byte `first`.
362///
363/// # Arguments
364///
365/// * `first` - UTF-8 leading byte.
366/// * `second` - Byte to validate as the first continuation-like byte.
367///
368/// # Returns
369///
370/// `true` when the pair `(first, second)` is valid for UTF-8 sequence decoding,
371/// otherwise `false`.
372#[inline(always)]
373fn is_valid_second_byte(first: u8, second: u8) -> bool {
374    match first {
375        0xc2..=0xdf => Utf8::is_continuation_byte(second),
376        0xe0 => (0xa0..=0xbf).contains(&second),
377        0xed => (0x80..=0x9f).contains(&second),
378        0xe1..=0xec | 0xee..=0xef => Utf8::is_continuation_byte(second),
379        0xf0 => (0x90..=0xbf).contains(&second),
380        0xf1..=0xf3 => Utf8::is_continuation_byte(second),
381        0xf4 => (0x80..=0x8f).contains(&second),
382        _ => false,
383    }
384}
385
386/// Decodes a three-byte UTF-8 sequence starting at `index`.
387///
388/// # Arguments
389///
390/// * `input` - Byte slice containing the sequence.
391/// * `index` - Start offset of a three-byte leading byte.
392///
393/// # Returns
394///
395/// The decoded Unicode scalar value as a `u32` on success.
396///
397/// # Errors
398///
399/// * `CharsetDecodeErrorKind::MalformedSequence` when the second or third byte
400///   is invalid.
401#[inline]
402fn decode_three(input: &[u8], index: usize) -> CharsetDecodeResult<u32> {
403    let first = input[index];
404    let second = input[index + 1];
405    let third = input[index + 2];
406    if !is_valid_second_byte(first, second) {
407        let kind = CharsetDecodeErrorKind::MalformedSequence {
408            value: Some(second as u32),
409        };
410        return Err(CharsetDecodeError::new(
411            Charset::UTF_8,
412            kind,
413            required_index(index, 1),
414        )
415        .with_consumed(2));
416    }
417    if !Utf8::is_continuation_byte(third) {
418        let kind = CharsetDecodeErrorKind::MalformedSequence {
419            value: Some(third as u32),
420        };
421        return Err(CharsetDecodeError::new(
422            Charset::UTF_8,
423            kind,
424            required_index(index, 2),
425        )
426        .with_consumed(3));
427    }
428    Ok((((first & 0x0f) as u32) << 12)
429        | (((second & 0x3f) as u32) << 6)
430        | ((third & 0x3f) as u32))
431}
432
433/// Decodes a four-byte UTF-8 sequence starting at `index`.
434///
435/// # Arguments
436///
437/// * `input` - Byte slice containing the sequence.
438/// * `index` - Start offset of a four-byte leading byte.
439///
440/// # Returns
441///
442/// The decoded Unicode scalar value as a `u32` on success.
443///
444/// # Errors
445///
446/// * `CharsetDecodeErrorKind::MalformedSequence` when any continuation byte is
447///   invalid.
448#[inline]
449fn decode_four(input: &[u8], index: usize) -> CharsetDecodeResult<u32> {
450    let first = input[index];
451    let second = input[index + 1];
452    let third = input[index + 2];
453    let fourth = input[index + 3];
454    if !is_valid_second_byte(first, second) {
455        let kind = CharsetDecodeErrorKind::MalformedSequence {
456            value: Some(second as u32),
457        };
458        return Err(CharsetDecodeError::new(
459            Charset::UTF_8,
460            kind,
461            required_index(index, 1),
462        )
463        .with_consumed(2));
464    }
465    if !Utf8::is_continuation_byte(third) {
466        let kind = CharsetDecodeErrorKind::MalformedSequence {
467            value: Some(third as u32),
468        };
469        return Err(CharsetDecodeError::new(
470            Charset::UTF_8,
471            kind,
472            required_index(index, 2),
473        )
474        .with_consumed(3));
475    }
476    if !Utf8::is_continuation_byte(fourth) {
477        let kind = CharsetDecodeErrorKind::MalformedSequence {
478            value: Some(fourth as u32),
479        };
480        return Err(CharsetDecodeError::new(
481            Charset::UTF_8,
482            kind,
483            required_index(index, 3),
484        )
485        .with_consumed(4));
486    }
487    Ok((((first & 0x07) as u32) << 18)
488        | (((second & 0x3f) as u32) << 12)
489        | (((third & 0x3f) as u32) << 6)
490        | ((fourth & 0x3f) as u32))
491}
qubit_codec_text/codec/utf8_codec.rs

qubit_codec_text/codec/
utf8_codec.rs
