datui_lib/formats/
nul_tail.rs

1//! A text file that ends in a run of NUL bytes ends where the run starts. Loggers that
2//! preallocate a file at a fixed size and never write past their last line leave the
3//! rest as NULs, which a CSV reader would take as one last row of junk text.
4//!
5//! Finding the run costs one byte read for a file that has none: its last byte. Only
6//! a file whose last byte is NUL is mapped and scanned back to its last other byte.
7
8use std::fs::File;
9use std::io::{Read, Seek, SeekFrom};
10use std::path::Path;
11
12/// Where the text of `bytes` ends: before its trailing run of NULs, or `None` when it
13/// has none, or when the NULs are part of the text (UTF-16 or UTF-32, by its byte-order
14/// mark, or by a NUL beside the last character).
15pub fn text_end(bytes: &[u8]) -> Option<usize> {
16    if bytes.last() != Some(&0) {
17        return None;
18    }
19    // Text in two- or four-byte units carries NULs inside it, and its last unit can
20    // end in one: `\n` in UTF-16LE is `0A 00`.
21    if bytes.starts_with(b"\xFF\xFE") || bytes.starts_with(b"\xFE\xFF") {
22        return None;
23    }
24    // A compressed file is not text, and its trailer can end in a zero: gzip's length.
25    const COMPRESSED: [&[u8]; 4] = [b"\x1F\x8B", b"\x28\xB5\x2F\xFD", b"BZh", b"\xFD7zXZ\x00"];
26    if COMPRESSED.iter().any(|magic| bytes.starts_with(magic)) {
27        return None;
28    }
29    let end = bytes.iter().rposition(|&b| b != 0).map_or(0, |at| at + 1);
30    if end >= 2 && bytes[end - 2] == 0 {
31        return None;
32    }
33    Some(end)
34}
35
36/// The text of the file `file`, `len` bytes long, mapped and cut before its trailing
37/// NULs; `None` when it has none, so the file is read as it is.
38fn mapped(file: &File, len: u64) -> std::io::Result<Option<(memmap2::Mmap, usize)>> {
39    if len == 0 || !ends_in_nul(file)? {
40        return Ok(None);
41    }
42    // SAFETY: read-only, as Polars maps a file it scans; a file changed under the map
43    // is read as it stands, as it would be by Polars.
44    let map = unsafe { memmap2::Mmap::map(file)? };
45    Ok(text_end(&map).map(|end| (map, end)))
46}
47
48/// Whether the last byte of `file` is NUL: the one read a file without a NUL tail costs.
49fn ends_in_nul(mut file: &File) -> std::io::Result<bool> {
50    let mut last = [1u8];
51    file.seek(SeekFrom::End(-1))?;
52    file.read_exact(&mut last)?;
53    file.rewind()?;
54    Ok(last[0] == 0)
55}
56
57/// How many bytes of the file at `file` are text, when it ends in NULs.
58pub fn text_len(file: &File) -> std::io::Result<Option<u64>> {
59    let len = file.metadata()?.len();
60    Ok(mapped(file, len)?.map(|(_, end)| end as u64))
61}
62
63/// Whether the file at `path` holds no text: empty, or nothing but NULs. One byte read
64/// for a file that ends in anything else.
65pub fn holds_nothing(path: &Path) -> bool {
66    let Ok(file) = File::open(path) else {
67        return false;
68    };
69    match file.metadata() {
70        Ok(m) if m.len() == 0 => true,
71        Ok(_) => text_len(&file).is_ok_and(|len| len == Some(0)),
72        Err(_) => false,
73    }
74}
75
76/// The text of the file at `path` as a buffer Polars scans in place, when the file
77/// ends in NULs; `None` when it does not, and the file is scanned by its path.
78pub fn text_buffer(path: &Path) -> std::io::Result<Option<polars_buffer::Buffer<u8>>> {
79    let file = File::open(path)?;
80    let len = file.metadata()?.len();
81    Ok(mapped(&file, len)?.map(|(map, end)| polars_buffer::Buffer::from_owner(map).sliced(..end)))
82}
83
84/// `bytes`, decompressed into memory, without a trailing run of NULs.
85pub fn trim(bytes: &mut Vec<u8>) {
86    if let Some(end) = text_end(bytes) {
87        bytes.truncate(end);
88    }
89}
90
91#[cfg(test)]
92mod tests {
93    use super::*;
94
95    #[test]
96    fn a_trailing_run_of_nuls_is_cut() {
97        assert_eq!(text_end(b"a,b\n1,2\n\0\0\0\0"), Some(8));
98        assert_eq!(text_end(&[0u8; 4096]), Some(0), "all NULs: no text");
99        assert_eq!(text_end(b"a\n1\0"), Some(3));
100    }
101
102    /// A normal file is read as it is: one byte looked at, and no cut.
103    #[test]
104    fn a_file_without_a_nul_tail_is_left_alone() {
105        assert_eq!(text_end(b"a,b\n1,2\n"), None);
106        assert_eq!(text_end(b""), None);
107        assert_eq!(
108            text_end(b"a,\0,b\n1,2"),
109            None,
110            "an interior NUL is the text's"
111        );
112        assert_eq!(
113            text_end(b"\x1F\x8B\x08\x00rest\x46\x00\x00\x00"),
114            None,
115            "gzip's length ends in zeros"
116        );
117        let dir = tempfile::tempdir().unwrap();
118        let path = dir.path().join("plain.csv");
119        std::fs::write(&path, "a,b\n1,2\n").unwrap();
120        assert!(text_buffer(&path).unwrap().is_none());
121        let empty = dir.path().join("empty.csv");
122        std::fs::write(&empty, "").unwrap();
123        assert!(text_buffer(&empty).unwrap().is_none());
124    }
125
126    #[test]
127    fn utf16_text_keeps_its_nuls() {
128        let le: Vec<u8> = "a,b\n1,2\n"
129            .encode_utf16()
130            .flat_map(u16::to_le_bytes)
131            .collect();
132        assert_eq!(le.last(), Some(&0));
133        assert_eq!(text_end(&le), None, "no byte-order mark");
134        let mut bom = b"\xFF\xFE".to_vec();
135        bom.extend_from_slice(&le);
136        bom.extend_from_slice(&[0, 0, 0, 0]);
137        assert_eq!(text_end(&bom), None, "a byte-order mark");
138        let be: Vec<u8> = "a,b\n".encode_utf16().flat_map(u16::to_be_bytes).collect();
139        assert_eq!(text_end(&be), None);
140    }
141
142    #[test]
143    fn a_padded_file_is_scanned_up_to_its_text() {
144        let dir = tempfile::tempdir().unwrap();
145        let path = dir.path().join("padded.csv");
146        let mut bytes = b"a,b\n1,2\n".to_vec();
147        bytes.resize(4096 + 8, 0);
148        std::fs::write(&path, &bytes).unwrap();
149        let buffer = text_buffer(&path).unwrap().expect("a NUL tail");
150        assert_eq!(buffer.as_slice(), b"a,b\n1,2\n");
151        let file = File::open(&path).unwrap();
152        assert_eq!(text_len(&file).unwrap(), Some(8));
153        let mut owned = bytes.clone();
154        trim(&mut owned);
155        assert_eq!(owned, b"a,b\n1,2\n");
156    }
157}
datui_lib/formats/nul_tail.rs

datui_lib/formats/
nul_tail.rs
