xutf 0.1.0

Permissive UTF-8/16/32 transcoding, comparison and BOM detection with SIMD ASCII fast paths
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112

//! The [`Encoding`] trait: per-encoding codepoint codec (`codepoint_cvt`).

use core::marker::PhantomData;

use crate::unit::Unit;

pub mod sealed {
	pub trait Sealed {}

	impl Sealed for crate::utf8::Utf8 {}
	impl<const FOREIGN: bool> Sealed for crate::utf16::Utf16<FOREIGN> {}
	impl<const FOREIGN: bool> Sealed for crate::utf32::Utf32<FOREIGN> {}
}

/// Encoding family, used to detect "same encoding" fast paths at
/// monomorphization time.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum Kind {
	Utf8,
	Utf16,
	Utf32,
}

/// A UTF encoding: a stateless codec between raw `u32` codepoints and code
/// units.
///
/// All operations are *permissive*: no validation is performed, decoding
/// never fails, and invalid input produces unspecified-but-safe codepoints
/// (truncated UTF-8 tails decode to `0`, lone UTF-16 surrogates pass
/// through). This trait is sealed because optimized same-encoding paths rely
/// on each [`Kind`] having exactly one code-unit type.
pub trait Encoding: sealed::Sealed + 'static {
	/// Code-unit type.
	type Unit: Unit;
	/// Encoding family.
	const KIND: Kind;
	/// `true` when code units are byte-swapped relative to native byte order.
	const FOREIGN: bool;
	/// Maximum code units per codepoint (`max_out`).
	const MAX_UNITS: usize;

	/// Length in units of the sequence starting with `front` (`rlength`).
	/// Always in `1..=MAX_UNITS`, even for garbage input.
	fn run_length(front: Self::Unit) -> usize;
	/// Length in units of `cp` once encoded (`length`). Assumes a valid
	/// codepoint outside the surrogate range.
	fn encoded_length(cp: u32) -> usize;
	/// Encodes `cp` into `out[..encoded_length(cp)]`, returning the unit
	/// count.
	///
	/// # Panics
	/// If `out.len() < encoded_length(cp)`.
	fn encode(cp: u32, out: &mut [Self::Unit]) -> usize;
	/// Decodes one codepoint from the front of `input`, advancing it by the
	/// consumed unit count.
	///
	/// # Panics
	/// If `input` is empty.
	fn decode(input: &mut &[Self::Unit]) -> u32;
}

/// `true` when `A` and `B` are byte-for-byte the same encoding.
#[inline(always)]
pub const fn same_encoding<A: Encoding, B: Encoding>() -> bool {
	A::KIND as u8 == B::KIND as u8 && A::FOREIGN == B::FOREIGN
}

/// Iterator over the raw codepoints of an encoded slice.
///
/// Created by [`codepoints`].
pub struct Codepoints<'a, E: Encoding> {
	rest:      &'a [E::Unit],
	_encoding: PhantomData<E>,
}
impl<E: Encoding> Clone for Codepoints<'_, E> {
	#[inline(always)]
	fn clone(&self) -> Self {
		Self { rest: self.rest, _encoding: PhantomData }
	}
}

impl<E: Encoding> Iterator for Codepoints<'_, E> {
	type Item = u32;

	#[inline]
	fn next(&mut self) -> Option<u32> {
		if self.rest.is_empty() {
			None
		} else {
			Some(E::decode(&mut self.rest))
		}
	}

	#[inline]
	fn size_hint(&self) -> (usize, Option<usize>) {
		// Each codepoint consumes 1..=MAX_UNITS units.
		(self.rest.len().div_ceil(E::MAX_UNITS), Some(self.rest.len()))
	}
}

/// Iterates the raw (unvalidated) codepoints of `input`.
#[inline]
pub const fn codepoints<E: Encoding>(input: &[E::Unit]) -> Codepoints<'_, E> {
	Codepoints { rest: input, _encoding: PhantomData }
}

/// Iterates `input` as `char`s, substituting U+FFFD for unit sequences that
/// decode to invalid scalar values.
#[inline]
pub fn chars<E: Encoding>(input: &[E::Unit]) -> impl Iterator<Item = char> + '_ {
	codepoints::<E>(input).map(|cp| char::from_u32(cp).unwrap_or(char::REPLACEMENT_CHARACTER))
}