use std::borrow::Cow;
#[must_use]
pub fn decode_dvb_string(bytes: &[u8]) -> String {
if bytes.is_empty() {
return String::new();
}
let (charset, body) = split_charset(bytes);
let decoded = match charset {
Charset::Iso6937 => decode_iso_6937(body),
Charset::Iso8859(n) => decode_iso_8859(n, body),
Charset::Utf8 => String::from_utf8_lossy(body).into_owned(),
Charset::Ucs2Be => decode_ucs2_be(body),
Charset::Unsupported(_indicator) => body.iter().map(|_| '\u{FFFD}').collect(),
};
decoded
.chars()
.filter_map(|c| match c as u32 {
0x86 | 0x87 => None,
0x8A => Some(' '),
0x0A => Some(' '),
code if code < 0x20 => None,
code if (0x80..0xA0).contains(&code) => None,
_ => Some(c),
})
.collect()
}
#[must_use]
pub fn decode(bytes: &[u8]) -> Cow<'_, str> {
if bytes.iter().all(|&b| b.is_ascii() && b >= 0x20) {
return Cow::Borrowed(std::str::from_utf8(bytes).unwrap_or(""));
}
Cow::Owned(decode_dvb_string(bytes))
}
#[derive(Debug)]
enum Charset {
Iso6937,
Iso8859(u8),
Utf8,
Ucs2Be,
Unsupported(u8),
}
fn split_charset(bytes: &[u8]) -> (Charset, &[u8]) {
match bytes[0] {
b if b >= 0x20 => (Charset::Iso6937, bytes),
0x00 => (Charset::Iso6937, &bytes[1..]),
0x08 => (Charset::Unsupported(0x08), &bytes[1..]),
0x01..=0x0B => (Charset::Iso8859(bytes[0] + 4), &bytes[1..]),
0x10 if bytes.len() >= 3 && bytes[1] == 0x00 => {
(Charset::Iso8859(bytes[2]), &bytes[3..])
}
0x11 => (Charset::Ucs2Be, &bytes[1..]),
0x15 => (Charset::Utf8, &bytes[1..]),
other => (Charset::Unsupported(other), &bytes[1..]),
}
}
fn decode_iso_6937(bytes: &[u8]) -> String {
let mut out = String::with_capacity(bytes.len());
let mut i = 0;
while i < bytes.len() {
let b = bytes[i];
if (0xC0..=0xCF).contains(&b) {
match combining_mark(b) {
Some(mark) if i + 1 < bytes.len() => {
let base = bytes[i + 1];
if let Some(c) = combine(b, base) {
out.push(c);
} else {
out.push(iso_6937_single(base));
out.push(mark);
}
i += 2;
}
_ => {
out.push('\u{FFFD}');
i += 1;
}
}
continue;
}
out.push(iso_6937_single(b));
i += 1;
}
out
}
fn iso_6937_single(b: u8) -> char {
match b {
0x00..=0x7F => b as char,
0x86 | 0x87 | 0x8A => b as char,
0x80..=0x9F => '\u{FFFD}',
0xA0 => '\u{00A0}', 0xA1 => '¡',
0xA2 => '¢',
0xA3 => '£',
0xA4 => '\u{20AC}', 0xA5 => '¥',
0xA6 => '\u{FFFD}', 0xA7 => '§',
0xA8 => '\u{00A4}', 0xA9 => '\u{2018}', 0xAA => '\u{201C}', 0xAB => '«',
0xAC => '\u{2190}', 0xAD => '\u{2191}', 0xAE => '\u{2192}', 0xAF => '\u{2193}', 0xB0 => '°',
0xB1 => '±',
0xB2 => '²',
0xB3 => '³',
0xB4 => '\u{00D7}', 0xB5 => 'µ',
0xB6 => '¶',
0xB7 => '·',
0xB8 => '\u{00F7}', 0xB9 => '\u{2019}', 0xBA => '\u{201D}', 0xBB => '»',
0xBC => '¼',
0xBD => '½',
0xBE => '¾',
0xBF => '¿',
0xC0..=0xCF => '\u{FFFD}',
0xD0 => '\u{2015}', 0xD1 => '¹',
0xD2 => '®',
0xD3 => '©',
0xD4 => '\u{2122}', 0xD5 => '\u{266A}', 0xD6 => '¬',
0xD7 => '\u{00A6}', 0xD8..=0xDB => '\u{FFFD}', 0xDC => '\u{215B}', 0xDD => '\u{215C}', 0xDE => '\u{215D}', 0xDF => '\u{215E}', 0xE0 => '\u{2126}', 0xE1 => 'Æ',
0xE2 => '\u{0110}', 0xE3 => 'ª',
0xE4 => '\u{0126}', 0xE5 => '\u{FFFD}', 0xE6 => '\u{0132}', 0xE7 => '\u{013F}', 0xE8 => '\u{0141}', 0xE9 => 'Ø',
0xEA => '\u{0152}', 0xEB => 'º',
0xEC => 'Þ',
0xED => '\u{0166}', 0xEE => '\u{014A}', 0xEF => '\u{0149}', 0xF0 => '\u{0138}', 0xF1 => 'æ',
0xF2 => '\u{0111}', 0xF3 => 'ð',
0xF4 => '\u{0127}', 0xF5 => '\u{0131}', 0xF6 => '\u{0133}', 0xF7 => '\u{0140}', 0xF8 => '\u{0142}', 0xF9 => 'ø',
0xFA => '\u{0153}', 0xFB => 'ß',
0xFC => '\u{00FE}', 0xFD => '\u{0167}', 0xFE => '\u{014B}', 0xFF => '\u{00AD}', }
}
fn combining_mark(prefix: u8) -> Option<char> {
Some(match prefix {
0xC1 => '\u{0300}', 0xC2 => '\u{0301}', 0xC3 => '\u{0302}', 0xC4 => '\u{0303}', 0xC5 => '\u{0304}', 0xC6 => '\u{0306}', 0xC7 => '\u{0307}', 0xC8 => '\u{0308}', 0xCA => '\u{030A}', 0xCB => '\u{0327}', 0xCD => '\u{030B}', 0xCE => '\u{0328}', 0xCF => '\u{030C}', _ => return None,
})
}
fn combine(prefix: u8, base: u8) -> Option<char> {
Some(match (prefix, base) {
(0xC1, b'A') => 'À', (0xC1, b'E') => 'È', (0xC1, b'I') => 'Ì',
(0xC1, b'O') => 'Ò', (0xC1, b'U') => 'Ù',
(0xC1, b'a') => 'à', (0xC1, b'e') => 'è', (0xC1, b'i') => 'ì',
(0xC1, b'o') => 'ò', (0xC1, b'u') => 'ù',
(0xC2, b'A') => 'Á', (0xC2, b'E') => 'É', (0xC2, b'I') => 'Í',
(0xC2, b'O') => 'Ó', (0xC2, b'U') => 'Ú', (0xC2, b'Y') => 'Ý',
(0xC2, b'a') => 'á', (0xC2, b'e') => 'é', (0xC2, b'i') => 'í',
(0xC2, b'o') => 'ó', (0xC2, b'u') => 'ú', (0xC2, b'y') => 'ý',
(0xC2, b'C') => 'Ć', (0xC2, b'c') => 'ć', (0xC2, b'L') => 'Ĺ',
(0xC2, b'l') => 'ĺ', (0xC2, b'N') => 'Ń', (0xC2, b'n') => 'ń',
(0xC2, b'R') => 'Ŕ', (0xC2, b'r') => 'ŕ', (0xC2, b'S') => 'Ś',
(0xC2, b's') => 'ś', (0xC2, b'Z') => 'Ź', (0xC2, b'z') => 'ź',
(0xC3, b'A') => 'Â', (0xC3, b'E') => 'Ê', (0xC3, b'I') => 'Î',
(0xC3, b'O') => 'Ô', (0xC3, b'U') => 'Û',
(0xC3, b'a') => 'â', (0xC3, b'e') => 'ê', (0xC3, b'i') => 'î',
(0xC3, b'o') => 'ô', (0xC3, b'u') => 'û',
(0xC4, b'A') => 'Ã', (0xC4, b'N') => 'Ñ', (0xC4, b'O') => 'Õ',
(0xC4, b'a') => 'ã', (0xC4, b'n') => 'ñ', (0xC4, b'o') => 'õ',
(0xC4, b'I') => 'Ĩ', (0xC4, b'i') => 'ĩ', (0xC4, b'U') => 'Ũ',
(0xC4, b'u') => 'ũ',
(0xC5, b'A') => 'Ā', (0xC5, b'a') => 'ā', (0xC5, b'E') => 'Ē',
(0xC5, b'e') => 'ē', (0xC5, b'I') => 'Ī', (0xC5, b'i') => 'ī',
(0xC5, b'O') => 'Ō', (0xC5, b'o') => 'ō', (0xC5, b'U') => 'Ū',
(0xC5, b'u') => 'ū',
(0xC6, b'A') => 'Ă', (0xC6, b'a') => 'ă', (0xC6, b'G') => 'Ğ',
(0xC6, b'g') => 'ğ', (0xC6, b'U') => 'Ŭ', (0xC6, b'u') => 'ŭ',
(0xC7, b'C') => 'Ċ', (0xC7, b'c') => 'ċ', (0xC7, b'E') => 'Ė',
(0xC7, b'e') => 'ė', (0xC7, b'G') => 'Ġ', (0xC7, b'g') => 'ġ',
(0xC7, b'I') => 'İ', (0xC7, b'Z') => 'Ż', (0xC7, b'z') => 'ż',
(0xC8, b'A') => 'Ä', (0xC8, b'E') => 'Ë', (0xC8, b'I') => 'Ï',
(0xC8, b'O') => 'Ö', (0xC8, b'U') => 'Ü', (0xC8, b'Y') => 'Ÿ',
(0xC8, b'a') => 'ä', (0xC8, b'e') => 'ë', (0xC8, b'i') => 'ï',
(0xC8, b'o') => 'ö', (0xC8, b'u') => 'ü', (0xC8, b'y') => 'ÿ',
(0xCA, b'A') => 'Å', (0xCA, b'a') => 'å', (0xCA, b'U') => 'Ů',
(0xCA, b'u') => 'ů',
(0xCB, b'C') => 'Ç', (0xCB, b'c') => 'ç', (0xCB, b'G') => 'Ģ',
(0xCB, b'g') => 'ģ', (0xCB, b'K') => 'Ķ', (0xCB, b'k') => 'ķ',
(0xCB, b'L') => 'Ļ', (0xCB, b'l') => 'ļ', (0xCB, b'N') => 'Ņ',
(0xCB, b'n') => 'ņ', (0xCB, b'R') => 'Ŗ', (0xCB, b'r') => 'ŗ',
(0xCB, b'S') => 'Ş', (0xCB, b's') => 'ş', (0xCB, b'T') => 'Ţ',
(0xCB, b't') => 'ţ',
(0xCD, b'O') => 'Ő', (0xCD, b'o') => 'ő', (0xCD, b'U') => 'Ű',
(0xCD, b'u') => 'ű',
(0xCE, b'A') => 'Ą', (0xCE, b'a') => 'ą', (0xCE, b'E') => 'Ę',
(0xCE, b'e') => 'ę', (0xCE, b'I') => 'Į', (0xCE, b'i') => 'į',
(0xCE, b'U') => 'Ų', (0xCE, b'u') => 'ų',
(0xCF, b'C') => 'Č', (0xCF, b'c') => 'č', (0xCF, b'D') => 'Ď',
(0xCF, b'd') => 'ď', (0xCF, b'E') => 'Ě', (0xCF, b'e') => 'ě',
(0xCF, b'L') => 'Ľ', (0xCF, b'l') => 'ľ', (0xCF, b'N') => 'Ň',
(0xCF, b'n') => 'ň', (0xCF, b'R') => 'Ř', (0xCF, b'r') => 'ř',
(0xCF, b'S') => 'Š', (0xCF, b's') => 'š', (0xCF, b'T') => 'Ť',
(0xCF, b't') => 'ť', (0xCF, b'Z') => 'Ž', (0xCF, b'z') => 'ž',
_ => return None,
})
}
fn decode_iso_8859(n: u8, bytes: &[u8]) -> String {
use encoding_rs::*;
let encoding: &'static Encoding = match n {
2 => ISO_8859_2,
3 => ISO_8859_3,
4 => ISO_8859_4,
5 => ISO_8859_5,
6 => ISO_8859_6,
7 => ISO_8859_7,
8 => ISO_8859_8,
9 => WINDOWS_1254,
10 => ISO_8859_10,
11 => WINDOWS_874,
13 => ISO_8859_13,
14 => ISO_8859_14,
15 => ISO_8859_15,
_ => return bytes.iter().map(|&b| b as char).collect(),
};
let (cow, _, _) = encoding.decode(bytes);
cow.into_owned()
}
fn decode_ucs2_be(bytes: &[u8]) -> String {
let code_units: Vec<u16> = bytes
.chunks_exact(2)
.map(|pair| u16::from_be_bytes([pair[0], pair[1]]))
.collect();
String::from_utf16_lossy(&code_units)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn decode_empty_input_returns_empty_string() {
assert_eq!(decode_dvb_string(&[]), "");
}
#[test]
fn decode_plain_ascii_is_borrowed() {
let cow = decode(b"HELLO");
assert!(matches!(cow, Cow::Borrowed(_)));
assert_eq!(cow, "HELLO");
}
#[test]
fn decode_iso6937_latin_accent_chars() {
assert_eq!(decode_dvb_string(&[0x00, 0xC2, b'A']), "Á");
assert_eq!(decode_dvb_string(&[0x00, 0xC1, b'e']), "è");
assert_eq!(decode_dvb_string(&[0x00, 0xC8, b'o']), "ö");
}
#[test]
fn decode_selector_0x01_yields_iso8859_5_cyrillic() {
let s = decode_dvb_string(&[0x01, 0xB0, 0xB1]);
assert!(s.chars().all(|c| c != '\u{FFFD}'), "got: {s:?}");
assert!(!s.is_empty());
}
#[test]
fn decode_selector_0x10_extended_yields_iso8859_nn() {
let s = decode_dvb_string(&[0x10, 0x00, 0x09, b'A', b'B']);
assert_eq!(s, "AB");
}
#[test]
fn decode_selector_0x11_ucs2_be() {
let s = decode_dvb_string(&[0x11, 0x00, 0x41, 0x00, 0x42]);
assert_eq!(s, "AB");
}
#[test]
fn decode_selector_0x15_utf8_passthrough() {
let s = decode_dvb_string(&[0x15, 0xC3, 0xA9, 0xC3, 0xA9]);
assert_eq!(s, "éé");
}
#[test]
fn decode_control_chars_stripped_linefeed_becomes_space() {
let s = decode_dvb_string(b"A\x01B\nC");
assert_eq!(s, "AB C");
}
#[test]
fn emphasis_on_off_markers_stripped_per_annex_a2() {
let s = decode_dvb_string(&[0x00, b'A', 0x86, b'B', 0x87, b'C']);
assert_eq!(s, "ABC");
}
#[test]
fn decode_annex_a2_crlf_0x8a_becomes_space() {
let s = decode_dvb_string(&[0x00, b'A', 0x8A, b'B']);
assert_eq!(s, "A B");
}
#[test]
fn reserved_selector_0x08_is_unsupported() {
let s = decode_dvb_string(&[0x08, 0x41, 0x42]);
assert!(s.chars().all(|c| c == '\u{FFFD}'));
assert_eq!(s.chars().count(), 2);
}
#[test]
fn unknown_selector_returns_replacement_characters() {
let s = decode_dvb_string(&[0x1F, 0xAA, 0xBB, 0xCC]);
assert_eq!(s.chars().count(), 3);
assert!(s.chars().all(|c| c == '\u{FFFD}'));
}
#[test]
fn figure_a1_gr_area_single_byte_mappings() {
let pins: &[(u8, char)] = &[
(0xA0, '\u{00A0}'), (0xA1, '¡'),
(0xA2, '¢'),
(0xA3, '£'),
(0xA4, '\u{20AC}'), (0xA5, '¥'),
(0xA7, '§'),
(0xA8, '\u{00A4}'), (0xA9, '\u{2018}'), (0xAA, '\u{201C}'), (0xAB, '«'),
(0xAC, '\u{2190}'), (0xAD, '\u{2191}'), (0xAE, '\u{2192}'), (0xAF, '\u{2193}'), (0xB0, '°'),
(0xB1, '±'),
(0xB2, '²'),
(0xB3, '³'),
(0xB4, '\u{00D7}'), (0xB5, 'µ'),
(0xB6, '¶'),
(0xB7, '·'),
(0xB8, '\u{00F7}'), (0xB9, '\u{2019}'), (0xBA, '\u{201D}'), (0xBB, '»'),
(0xBC, '¼'),
(0xBD, '½'),
(0xBE, '¾'),
(0xBF, '¿'),
(0xD0, '\u{2015}'), (0xD1, '¹'),
(0xD2, '®'),
(0xD3, '©'),
(0xD4, '\u{2122}'), (0xD5, '\u{266A}'), (0xD6, '¬'),
(0xD7, '\u{00A6}'), (0xDC, '\u{215B}'), (0xDD, '\u{215C}'), (0xDE, '\u{215D}'), (0xDF, '\u{215E}'), (0xE0, '\u{2126}'), (0xE1, 'Æ'),
(0xE2, '\u{0110}'), (0xE3, 'ª'),
(0xE4, '\u{0126}'), (0xE6, '\u{0132}'), (0xE7, '\u{013F}'), (0xE8, '\u{0141}'), (0xE9, 'Ø'),
(0xEA, '\u{0152}'), (0xEB, 'º'),
(0xEC, 'Þ'),
(0xED, '\u{0166}'), (0xEE, '\u{014A}'), (0xEF, '\u{0149}'), (0xF0, '\u{0138}'), (0xF1, 'æ'),
(0xF2, '\u{0111}'), (0xF3, 'ð'),
(0xF4, '\u{0127}'), (0xF5, '\u{0131}'), (0xF6, '\u{0133}'), (0xF7, '\u{0140}'), (0xF8, '\u{0142}'), (0xF9, 'ø'),
(0xFA, '\u{0153}'), (0xFB, 'ß'),
(0xFC, '\u{00FE}'), (0xFD, '\u{0167}'), (0xFE, '\u{014B}'), (0xFF, '\u{00AD}'), ];
for &(byte, want) in pins {
let got = decode_dvb_string(&[0x00, byte]);
assert_eq!(
got,
want.to_string(),
"byte {byte:#04x}: want {want:?} (U+{:04X}), got {got:?}",
want as u32
);
}
}
#[test]
fn figure_a1_undefined_positions_are_replacement() {
for byte in [0xA6u8, 0xD8, 0xD9, 0xDA, 0xDB, 0xE5] {
let got = decode_dvb_string(&[0x00, byte]);
assert_eq!(got, "\u{FFFD}", "byte {byte:#04x} should be U+FFFD");
}
}
#[test]
fn figure_a1_combining_precomposed() {
assert_eq!(decode_dvb_string(&[0x00, 0xCA, b'a']), "å"); assert_eq!(decode_dvb_string(&[0x00, 0xCA, b'A']), "Å");
assert_eq!(decode_dvb_string(&[0x00, 0xCF, b's']), "š"); assert_eq!(decode_dvb_string(&[0x00, 0xCF, b'Z']), "Ž");
assert_eq!(decode_dvb_string(&[0x00, 0xCE, b'e']), "ę"); assert_eq!(decode_dvb_string(&[0x00, 0xCD, b'o']), "ő"); assert_eq!(decode_dvb_string(&[0x00, 0xC7, b'z']), "ż"); assert_eq!(decode_dvb_string(&[0x00, 0xC5, b'a']), "ā"); assert_eq!(decode_dvb_string(&[0x00, 0xC6, b'g']), "ğ"); }
#[test]
fn figure_a1_combining_fallback_emits_base_plus_mark() {
assert_eq!(decode_dvb_string(&[0x00, 0xC5, b'x']), "x\u{0304}");
}
#[test]
fn figure_a1_combining_undefined_or_dangling_prefix() {
assert_eq!(decode_dvb_string(&[0x00, 0xC0, b'a']), "\u{FFFD}a");
assert_eq!(decode_dvb_string(&[0x00, 0xC9, b'a']), "\u{FFFD}a");
assert_eq!(decode_dvb_string(&[0x00, 0xCC, b'a']), "\u{FFFD}a");
assert_eq!(decode_dvb_string(&[0x00, 0xC2]), "\u{FFFD}");
}
}