use crate::syntax::CodePoint;
#[cfg(test)]
mod tests;
pub fn decode_utf16(units: &[u16]) -> Vec<CodePoint> {
let mut out = Vec::with_capacity(units.len());
let mut i = 0;
while i < units.len() {
let u = units[i];
if (0xD800..=0xDBFF).contains(&u)
&& let Some(&lo) = units.get(i + 1)
&& (0xDC00..=0xDFFF).contains(&lo)
{
let cp = 0x1_0000 + (((u as u32) - 0xD800) << 10) + ((lo as u32) - 0xDC00);
out.push(cp);
i += 2;
continue;
}
out.push(u as u32);
i += 1;
}
out
}
pub fn decode_bytes(bytes: &[u8]) -> Vec<CodePoint> {
let mut out = Vec::with_capacity(bytes.len());
let mut rest = bytes;
loop {
match core::str::from_utf8(rest) {
Ok(s) => {
out.extend(s.chars().map(|c| c as u32));
return out;
}
Err(e) => {
let valid = e.valid_up_to();
if let Ok(s) = core::str::from_utf8(&rest[..valid]) {
out.extend(s.chars().map(|c| c as u32));
}
out.push(0xDC00 + rest[valid] as u32);
rest = &rest[valid + 1..];
}
}
}
}
pub fn decode_str(s: &str) -> Vec<CodePoint> {
s.chars().map(|c| c as u32).collect()
}