#[cfg(unix)]
pub fn stream_lines(bytes: &[u8]) -> Vec<String> {
pieces(bytes).map(decode_line).collect()
}
#[cfg(windows)]
pub fn stream_lines(bytes: &[u8]) -> Vec<String> {
stream_lines_at(bytes, active_codepage())
}
fn pieces(bytes: &[u8]) -> impl Iterator<Item = &[u8]> {
let mut end = bytes.len();
while end > 0 && bytes[end - 1] == b'\n' {
end -= 1;
}
bytes[..end].split(|&b| b == b'\n')
}
#[cfg(unix)]
fn decode_line(line: &[u8]) -> String {
String::from_utf8_lossy(line).into_owned()
}
#[cfg(windows)]
fn stream_lines_at(bytes: &[u8], codepage: u32) -> Vec<String> {
pieces(bytes)
.map(|line| decode_line_at(line, codepage))
.collect()
}
#[cfg(windows)]
fn decode_line_at(line: &[u8], codepage: u32) -> String {
match std::str::from_utf8(line) {
Ok(text) => text.to_owned(),
Err(_) => decode_with_codepage(line, codepage),
}
}
#[cfg(windows)]
fn active_codepage() -> u32 {
use windows_sys::Win32::Globalization::GetOEMCP;
use windows_sys::Win32::System::Console::GetConsoleOutputCP;
match unsafe { GetConsoleOutputCP() } {
0 => unsafe { GetOEMCP() },
cp => cp,
}
}
#[cfg(windows)]
fn decode_with_codepage(bytes: &[u8], codepage: u32) -> String {
use windows_sys::Win32::Globalization::MultiByteToWideChar;
if bytes.is_empty() {
return String::new();
}
let needed = unsafe {
MultiByteToWideChar(
codepage,
0,
bytes.as_ptr(),
bytes.len() as i32,
std::ptr::null_mut(),
0,
)
};
if needed <= 0 {
return String::from_utf8_lossy(bytes).into_owned();
}
let mut wide = vec![0u16; needed as usize];
let written = unsafe {
MultiByteToWideChar(
codepage,
0,
bytes.as_ptr(),
bytes.len() as i32,
wide.as_mut_ptr(),
needed,
)
};
if written <= 0 {
return String::from_utf8_lossy(bytes).into_owned();
}
String::from_utf16_lossy(&wide[..written as usize])
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn an_empty_stream_is_one_empty_line() {
assert_eq!(stream_lines(b""), vec![String::new()]);
}
#[test]
fn trailing_newlines_are_stripped_and_interior_blanks_survive() {
assert_eq!(
stream_lines(b"a\n\nb\n\n\n"),
vec!["a".to_string(), String::new(), "b".to_string()]
);
}
#[cfg(unix)]
#[test]
fn invalid_utf8_on_unix_still_becomes_replacement_chars() {
assert_eq!(
stream_lines(b"gr\x81\xE1e"),
vec!["gr\u{FFFD}\u{FFFD}e".to_string()]
);
}
#[test]
fn utf8_input_round_trips_exactly() {
assert_eq!(
stream_lines("grüße\n".as_bytes()),
vec!["grüße".to_string()]
);
}
#[cfg(windows)]
#[test]
fn cp437_bytes_decode_through_the_codepage() {
assert_eq!(
stream_lines_at(b"gr\x81\xE1e\n", 437),
vec!["grüße".to_string()]
);
}
#[cfg(windows)]
#[test]
fn valid_utf8_is_never_re_decoded() {
assert_eq!(
stream_lines_at("grüße\n".as_bytes(), 437),
vec!["grüße".to_string()]
);
}
#[cfg(windows)]
#[test]
fn decode_granularity_is_the_line() {
let mut bytes = Vec::from("utf8-grüße\n".as_bytes());
bytes.extend_from_slice(b"oem-gr\x81\xE1e\n");
assert_eq!(
stream_lines_at(&bytes, 437),
vec!["utf8-grüße".to_string(), "oem-grüße".to_string()]
);
}
}