Skip to main content

readcon_core/
compression.rs

1//=============================================================================
2// Transparent compression support
3//=============================================================================
4
5use std::io::{self, Read};
6use std::path::Path;
7
8/// Detected compression format based on magic bytes.
9///
10/// `Zstd` is only constructed when the `zstd` Cargo feature is enabled.
11/// Builds without the feature treat `.zst` files as opaque bytes and
12/// return an error from [`read_file_contents`] indicating the feature
13/// is required.
14#[derive(Debug, Clone, Copy, PartialEq, Eq)]
15pub enum Compression {
16    None,
17    Gzip,
18    /// zstd data: a frame (`28 b5 2f fd`) or a skippable frame
19    /// (`5? 2a 4d 18`, RFC 8878). Build with `--features zstd`.
20    Zstd,
21}
22
23/// A Zstandard frame or a skippable frame, in the byte order on disk.
24///
25/// RFC 8878 gives a frame the magic `28 b5 2f fd` and a skippable frame
26/// one of `50 2a 4d 18` through `5f 2a 4d 18`. A file from `pzstd` can
27/// start with the skippable frame.
28fn is_zstd_magic(bytes: &[u8]) -> bool {
29    bytes.len() >= 4
30        && (bytes[..4] == [0x28, 0xb5, 0x2f, 0xfd]
31            || ((0x50..=0x5f).contains(&bytes[0])
32                && bytes[1] == 0x2a
33                && bytes[2] == 0x4d
34                && bytes[3] == 0x18))
35}
36
37/// Detect compression format from the first bytes of a file.
38///
39/// - `1f 8b` = gzip
40/// - `28 b5 2f fd` = zstd frame
41/// - `5? 2a 4d 18` = zstd skippable frame
42/// - Otherwise = uncompressed
43pub fn detect_compression(bytes: &[u8]) -> Compression {
44    if bytes.len() >= 2 && bytes[0] == 0x1f && bytes[1] == 0x8b {
45        return Compression::Gzip;
46    }
47    if is_zstd_magic(bytes) {
48        return Compression::Zstd;
49    }
50    Compression::None
51}
52
53/// Detect compression format from a file extension.
54///
55/// Returns `Compression::Gzip` for `.gz`, `Compression::Zstd` for
56/// `.zst`, `Compression::None` otherwise.
57pub fn detect_compression_from_extension(path: &Path) -> Compression {
58    match path.extension().and_then(|e| e.to_str()) {
59        Some("gz") => Compression::Gzip,
60        Some("zst") => Compression::Zstd,
61        _ => Compression::None,
62    }
63}
64
65/// Size threshold below which we use `read_to_string` instead of mmap.
66const MMAP_THRESHOLD: u64 = 64 * 1024;
67
68/// Reads file contents, decompressing if needed.
69///
70/// Detection strategy:
71/// 1. Read the first 4 bytes to check magic bytes.
72/// 2. If gzip or zstd: decompress entire file to a String.
73/// 3. If uncompressed and < 64 KiB: read the rest into a String.
74/// 4. If uncompressed and >= 64 KiB: memory-mapped I/O.
75///
76/// The path is opened once and every byte comes from that handle, so a
77/// pipe works and a file replaced during the call is not mixed in.
78pub fn read_file_contents(path: &Path) -> Result<FileContents, Box<dyn std::error::Error>> {
79    read_file_contents_detected(path).map(|(contents, _)| contents)
80}
81
82/// [`read_file_contents`], also returning the compression it detected.
83pub fn read_file_contents_detected(
84    path: &Path,
85) -> Result<(FileContents, Compression), Box<dyn std::error::Error>> {
86    read_contents(path, true)
87}
88
89/// [`read_file_contents`] as one owned `String`. An uncompressed file is
90/// read into it at any size instead of being mapped, so the text is held
91/// once and later changes to the file do not reach it.
92pub fn read_file_text(path: &Path) -> Result<String, Box<dyn std::error::Error>> {
93    match read_contents(path, false)?.0 {
94        FileContents::Owned(text) => Ok(text),
95        FileContents::Mapped(_) => unreachable!("read_contents maps only when asked to"),
96    }
97}
98
99/// Bytes of one file, mapped when a plain file is large enough.
100///
101/// Compressed input is decoded into a buffer. A plain file under 64 KiB,
102/// or anything that is not a regular file, is read into a buffer. A larger
103/// plain file stays mapped for the parse and is not copied. The bytes are
104/// not checked as UTF-8; the parser reports a bad byte when it reaches it.
105pub enum ByteSource {
106    Buf(Vec<u8>),
107    Map(memmap2::Mmap),
108}
109
110impl ByteSource {
111    pub fn as_slice(&self) -> &[u8] {
112        match self {
113            ByteSource::Buf(bytes) => bytes,
114            ByteSource::Map(map) => map,
115        }
116    }
117}
118
119/// Load `path` for a parse. See [`ByteSource`].
120pub fn read_bytes_owned(path: &Path) -> Result<ByteSource, Box<dyn std::error::Error>> {
121    let mut file = std::fs::File::open(path)?;
122    let metadata = file.metadata()?;
123    let mut magic = [0u8; 4];
124    let n = (&mut file).take(4).read(&mut magic)?;
125    match detect_compression(&magic[..n]) {
126        Compression::None if metadata.is_file() && metadata.len() >= MMAP_THRESHOLD => {
127            let map = unsafe { memmap2::Mmap::map(&file)? };
128            Ok(ByteSource::Map(map))
129        }
130        Compression::None => {
131            let mut bytes = Vec::with_capacity(metadata.len().saturating_add(4) as usize);
132            bytes.extend_from_slice(&magic[..n]);
133            file.read_to_end(&mut bytes)?;
134            Ok(ByteSource::Buf(bytes))
135        }
136        Compression::Gzip | Compression::Zstd => {
137            drop(file);
138            Ok(ByteSource::Buf(read_file_bytes(path)?))
139        }
140    }
141}
142
143/// File bytes in one read when the file is not compressed.
144///
145/// Gzip and zstd still go through the decoder. Plain text skips the
146/// four-byte probe and the UTF-8 `String` allocation.
147pub fn read_file_bytes(path: &Path) -> Result<Vec<u8>, Box<dyn std::error::Error>> {
148    let bytes = std::fs::read(path)?;
149    let magic = &bytes[..bytes.len().min(4)];
150    match detect_compression(magic) {
151        Compression::None => Ok(bytes),
152        Compression::Gzip => {
153            let mut decoder = flate2::read::MultiGzDecoder::new(std::io::Cursor::new(bytes));
154            let mut out = Vec::new();
155            decoder.read_to_end(&mut out)?;
156            Ok(out)
157        }
158        Compression::Zstd => {
159            #[cfg(feature = "zstd")]
160            {
161                let mut decoder = zstd::stream::read::Decoder::new(std::io::Cursor::new(bytes))?;
162                let mut out = Vec::new();
163                decoder.read_to_end(&mut out)?;
164                Ok(out)
165            }
166            #[cfg(not(feature = "zstd"))]
167            {
168                let _ = bytes;
169                Err(zstd_unsupported().into())
170            }
171        }
172    }
173}
174
175fn read_contents(
176    path: &Path,
177    map_large: bool,
178) -> Result<(FileContents, Compression), Box<dyn std::error::Error>> {
179    let file = std::fs::File::open(path)?;
180    let metadata = file.metadata()?;
181
182    // gzip needs 2 magic bytes and zstd needs 4. They stay in front of the
183    // rest of the stream, since a pipe cannot seek back.
184    let mut magic = Vec::with_capacity(4);
185    (&file).take(4).read_to_end(&mut magic)?;
186    let compression = detect_compression(&magic);
187    let mut stream = magic.as_slice().chain(&file);
188
189    let contents = match compression {
190        Compression::Gzip => {
191            // Every gzip member, in order (CON-compress.1, D16).
192            let mut decoder = flate2::read::MultiGzDecoder::new(stream);
193            let mut contents = String::new();
194            decoder.read_to_string(&mut contents)?;
195            FileContents::Owned(contents)
196        }
197        Compression::Zstd => {
198            #[cfg(feature = "zstd")]
199            {
200                let mut decoder = zstd::stream::read::Decoder::new(stream)?;
201                let mut contents = String::new();
202                decoder.read_to_string(&mut contents)?;
203                FileContents::Owned(contents)
204            }
205            #[cfg(not(feature = "zstd"))]
206            {
207                return Err(zstd_unsupported().into());
208            }
209        }
210        Compression::None if metadata.len() < MMAP_THRESHOLD => {
211            let mut bytes = Vec::with_capacity(metadata.len() as usize);
212            stream.read_to_end(&mut bytes)?;
213            let contents = String::from_utf8(bytes).map_err(|_| {
214                io::Error::new(
215                    io::ErrorKind::InvalidData,
216                    "stream did not contain valid UTF-8",
217                )
218            })?;
219            FileContents::Owned(contents)
220        }
221        Compression::None if map_large => {
222            let mmap = unsafe { memmap2::Mmap::map(&file)? };
223            FileContents::Mapped(mmap)
224        }
225        Compression::None => {
226            let mut bytes = Vec::with_capacity(metadata.len() as usize);
227            stream.read_to_end(&mut bytes)?;
228            // The message `as_str` gives for the same bytes when mapped.
229            FileContents::Owned(String::from_utf8(bytes).map_err(|e| e.utf8_error())?)
230        }
231    };
232    Ok((contents, compression))
233}
234
235#[cfg(not(feature = "zstd"))]
236fn zstd_unsupported() -> io::Error {
237    io::Error::new(
238        io::ErrorKind::Unsupported,
239        "zstd-compressed input detected; rebuild readcon-core with --features zstd",
240    )
241}
242
243/// The bytes of `path` as one stream, decompressed while it is read.
244///
245/// Compression is detected by magic bytes, as in [`read_file_contents`],
246/// and every gzip member is read in order.
247/// The only memory used is the decoder's own buffers, so it does not grow
248/// with the file.
249pub fn open_decoded(path: &Path) -> Result<Box<dyn Read + Send>, Box<dyn std::error::Error>> {
250    let mut file = std::fs::File::open(path)?;
251    let mut magic = Vec::with_capacity(4);
252    (&mut file).take(4).read_to_end(&mut magic)?;
253    let compression = detect_compression(&magic);
254    let stream = io::Cursor::new(magic).chain(file);
255    Ok(match compression {
256        Compression::Gzip => Box::new(flate2::read::MultiGzDecoder::new(stream)),
257        #[cfg(feature = "zstd")]
258        Compression::Zstd => Box::new(zstd::stream::read::Decoder::new(stream)?),
259        #[cfg(not(feature = "zstd"))]
260        Compression::Zstd => return Err(zstd_unsupported().into()),
261        Compression::None => Box::new(stream),
262    })
263}
264
265/// Holds file contents either as an owned String or a memory-mapped region.
266pub enum FileContents {
267    Owned(String),
268    Mapped(memmap2::Mmap),
269}
270
271impl FileContents {
272    pub fn as_str(&self) -> Result<&str, std::str::Utf8Error> {
273        match self {
274            FileContents::Owned(s) => Ok(s.as_str()),
275            FileContents::Mapped(m) => std::str::from_utf8(m),
276        }
277    }
278
279    pub fn as_bytes(&self) -> &[u8] {
280        match self {
281            FileContents::Owned(s) => s.as_bytes(),
282            FileContents::Mapped(m) => m,
283        }
284    }
285
286    /// Like [`Self::as_str`], but validates mapped files in newline-aligned
287    /// slices (in parallel with the `parallel` feature).
288    pub fn as_str_sliced(&self) -> Result<&str, std::str::Utf8Error> {
289        match self {
290            FileContents::Owned(s) => Ok(s.as_str()),
291            FileContents::Mapped(m) => crate::frame_index::validate_utf8_sliced(m),
292        }
293    }
294}
295
296/// Mapped bytes between two releases of the pages already read.
297const RELEASE_STEP: usize = 32 << 20;
298
299/// The text of one file, read from the front.
300///
301/// With `map`, an uncompressed file of 64 KiB or more stays mapped, and
302/// [`Self::release_before`] advises `MADV_DONTNEED` for the pages before the
303/// read position. On Linux that drops the file pages, so resident memory
304/// stays near [`RELEASE_STEP`] instead of the file size. macOS gets the same
305/// call, but its `MADV_DONTNEED` does not discard pages. Outside Unix the
306/// release is a no-op. On those systems the whole file can become resident.
307/// A mapped file is not a snapshot and must not change while it is read; a
308/// truncation can end the process with `SIGBUS`.
309pub struct ForwardText {
310    contents: FileContents,
311    released: usize,
312}
313
314impl ForwardText {
315    /// The text of `path`. Without `map` it is [`read_file_text`].
316    pub fn open(path: &Path, map: bool) -> Result<Self, Box<dyn std::error::Error>> {
317        let contents = if map {
318            read_file_contents(path)?
319        } else {
320            FileContents::Owned(read_file_text(path)?)
321        };
322        // One sequential check. Discarding the pages here would fault
323        // them back in during the parse, and a parallel check would
324        // start a worker pool for a single slice.
325        if let FileContents::Mapped(m) = &contents {
326            std::str::from_utf8(m)?;
327        }
328        Ok(Self {
329            contents,
330            released: 0,
331        })
332    }
333
334    pub fn as_str(&self) -> &str {
335        match &self.contents {
336            FileContents::Owned(s) => s,
337            // SAFETY: `open` checked every byte, in slices that end after a
338            // newline. A mapped file must not change while mapped, as for
339            // every mapped reader in this crate.
340            FileContents::Mapped(m) => unsafe { std::str::from_utf8_unchecked(m) },
341        }
342    }
343
344    /// Release the mapped pages before byte `pos` once [`RELEASE_STEP`]
345    /// bytes have been read since the last release. A later read of them
346    /// maps them in again. A `pos` past the end counts as the end.
347    pub fn release_before(&mut self, pos: usize) -> io::Result<()> {
348        if let FileContents::Mapped(m) = &self.contents
349            && pos >= self.released + RELEASE_STEP
350        {
351            let pos = pos.min(m.len());
352            release(m, self.released, pos)?;
353            self.released = pos;
354        }
355        Ok(())
356    }
357}
358
359/// Drop resident pages of `map` on `workers` threads, then the caller
360/// unmaps. Each thread advises a disjoint byte range. The page size is
361/// `sysconf(SC_PAGESIZE)`, not a host-tuned cutoff.
362pub fn advise_dontneed_parallel(map: &memmap2::Mmap, workers: usize) {
363    let len = map.len();
364    if len == 0 {
365        return;
366    }
367    let workers = workers.max(1);
368    if workers == 1 {
369        let _ = release(map, 0, len);
370        return;
371    }
372    let step = len.div_ceil(workers);
373    let advise_one = |worker: usize| {
374        let start = worker.saturating_mul(step);
375        if start >= len {
376            return;
377        }
378        let end = (start + step).min(len);
379        let _ = release(map, start, end);
380    };
381    // Inside the read's pool this stays on those workers. Outside it,
382    // a private scope avoids the global pool.
383    #[cfg(feature = "parallel")]
384    if rayon::current_thread_index().is_some() {
385        use rayon::prelude::*;
386        (0..workers).into_par_iter().for_each(advise_one);
387        return;
388    }
389    std::thread::scope(|scope| {
390        for worker in 0..workers {
391            scope.spawn(move || advise_one(worker));
392        }
393    });
394}
395
396/// Advise `MADV_DONTNEED` for the whole 64 KiB blocks of the map that
397/// `start..end` reaches, cut at the map end.
398fn release(map: &memmap2::Mmap, start: usize, end: usize) -> io::Result<()> {
399    // A multiple of every Linux page size. The map starts on a page, so the
400    // range stays inside it, and the page that contains `end` stays mapped.
401    const BLOCK: usize = 64 * 1024;
402    let start = start & !(BLOCK - 1);
403    let end = end.min(map.len()) & !(BLOCK - 1);
404    if end <= start {
405        return Ok(());
406    }
407    // SAFETY: `start..end` lies inside the map, which is shared and read
408    // only, so dropped pages read back from the file.
409    #[cfg(unix)]
410    unsafe {
411        map.unchecked_advise_range(memmap2::UncheckedAdvice::DontNeed, start, end - start)?;
412    }
413    #[cfg(not(unix))]
414    let _ = map;
415    Ok(())
416}
417
418/// Creates a gzip-compressed writer wrapping a file at the given path.
419pub fn gzip_writer(path: &Path) -> io::Result<flate2::write::GzEncoder<std::fs::File>> {
420    let file = std::fs::File::create(path)?;
421    Ok(flate2::write::GzEncoder::new(
422        file,
423        flate2::Compression::default(),
424    ))
425}
426
427/// Creates a zstd-compressed writer wrapping a file at the given path.
428///
429/// Returns a finished writer ready for streaming line writes; the caller
430/// is responsible for dropping the writer to flush the final frame.
431/// Available only when the `zstd` Cargo feature is enabled.
432#[cfg(feature = "zstd")]
433pub fn zstd_writer(
434    path: &Path,
435) -> io::Result<zstd::stream::write::AutoFinishEncoder<'static, std::fs::File>> {
436    let file = std::fs::File::create(path)?;
437    // Level 3 is the zstd CLI default; balances ratio against speed.
438    let encoder = zstd::stream::write::Encoder::new(file, 3)?;
439    Ok(encoder.auto_finish())
440}
441
442#[cfg(test)]
443mod tests {
444    use super::*;
445    use std::io::Write;
446
447    fn text_of_at_least(len: usize) -> Vec<u8> {
448        let frame = include_bytes!("../resources/test/tiny_cuh2.con");
449        frame.repeat(len / frame.len() + 2)
450    }
451
452    fn large_text() -> Vec<u8> {
453        text_of_at_least(MMAP_THRESHOLD as usize)
454    }
455
456    fn mapped_text(path: &Path) -> Result<String, String> {
457        let contents = read_file_contents(path).map_err(|e| e.to_string())?;
458        contents
459            .as_str()
460            .map(str::to_owned)
461            .map_err(|e| e.to_string())
462    }
463
464    #[test]
465    fn file_text_matches_the_mapped_read() {
466        let dir = tempfile::tempdir().unwrap();
467        let small = dir.path().join("small.con");
468        std::fs::write(&small, include_bytes!("../resources/test/tiny_cuh2.con")).unwrap();
469        let large = dir.path().join("large.con");
470        std::fs::write(&large, large_text()).unwrap();
471        assert!(matches!(
472            read_file_contents(&large).unwrap(),
473            FileContents::Mapped(_)
474        ));
475        let gz = dir.path().join("large.con.gz");
476        let mut enc = gzip_writer(&gz).unwrap();
477        enc.write_all(&large_text()).unwrap();
478        enc.finish().unwrap();
479        for path in [&small, &large, &gz] {
480            let text = read_file_text(path).unwrap();
481            assert_eq!(Ok(text), mapped_text(path), "{}", path.display());
482        }
483    }
484
485    #[test]
486    fn file_text_reports_invalid_utf8_as_the_mapped_read() {
487        let dir = tempfile::tempdir().unwrap();
488        for (name, len) in [("small.con", 100), ("large.con", large_text().len())] {
489            let mut bytes = large_text()[..len].to_vec();
490            bytes[len - 40] = 0xff;
491            let path = dir.path().join(name);
492            std::fs::write(&path, &bytes).unwrap();
493            let err = read_file_text(&path).unwrap_err().to_string();
494            assert_eq!(Err(err), mapped_text(&path), "{name}");
495        }
496    }
497
498    #[test]
499    fn forward_text_reads_back_released_pages() {
500        let dir = tempfile::tempdir().unwrap();
501        let bytes = text_of_at_least(RELEASE_STEP + (200 << 10));
502        let path = dir.path().join("two-steps.con");
503        std::fs::write(&path, &bytes).unwrap();
504        let mut mapped = ForwardText::open(&path, true).unwrap();
505        assert!(matches!(mapped.contents, FileContents::Mapped(_)));
506        assert_eq!(mapped.as_str().as_bytes(), bytes.as_slice());
507        for pos in (0..=bytes.len()).step_by(1 << 20) {
508            mapped.release_before(pos).unwrap();
509        }
510        assert_eq!(mapped.released, RELEASE_STEP);
511        assert_eq!(mapped.as_str().as_bytes(), bytes.as_slice());
512        let owned = ForwardText::open(&path, false).unwrap();
513        assert!(matches!(owned.contents, FileContents::Owned(_)));
514        assert_eq!(owned.as_str(), mapped.as_str());
515    }
516
517    /// The guard is private anonymous memory right after the file map, which
518    /// `MADV_DONTNEED` would zero.
519    ///
520    /// The kernel puts a new map at the top of the highest free gap that fits
521    /// it. The test frees the space below the guard after it fills every gap
522    /// above, so the file map must end at the guard. A map made by another
523    /// test thread in between would take that space, so the test body runs
524    /// alone in a child process of this test binary.
525    #[cfg(target_os = "linux")]
526    #[test]
527    fn release_before_past_the_end_leaves_the_next_mapping_alone() {
528        const NAME: &str =
529            "compression::tests::release_before_past_the_end_leaves_the_next_mapping_alone";
530        const CHILD: &str = "READCON_RELEASE_TEST_CHILD";
531        if std::env::var_os(CHILD).is_none() {
532            let out = std::process::Command::new(std::env::current_exe().unwrap())
533                .args([NAME, "--exact", "--test-threads=1"])
534                .env(CHILD, "1")
535                .output()
536                .unwrap();
537            let stdout = String::from_utf8_lossy(&out.stdout);
538            assert!(
539                out.status.success() && stdout.contains("test result: ok. 1 passed;"),
540                "{stdout}{}",
541                String::from_utf8_lossy(&out.stderr)
542            );
543            return;
544        }
545        const GUARD: usize = 1 << 20;
546        let dir = tempfile::tempdir().unwrap();
547        let bytes = text_of_at_least(3 * MMAP_THRESHOLD as usize);
548        let path = dir.path().join("short.con");
549        std::fs::write(&path, &bytes).unwrap();
550        let page = unsafe { libc::sysconf(libc::_SC_PAGESIZE) } as usize;
551        let map_len = bytes.len().next_multiple_of(page);
552        let anonymous = |len: usize| {
553            let map = unsafe {
554                libc::mmap(
555                    std::ptr::null_mut(),
556                    len,
557                    libc::PROT_READ | libc::PROT_WRITE,
558                    libc::MAP_PRIVATE | libc::MAP_ANONYMOUS,
559                    -1,
560                    0,
561                )
562            };
563            assert_ne!(map, libc::MAP_FAILED);
564            map
565        };
566        let region = anonymous(map_len + GUARD);
567        let guard =
568            unsafe { std::slice::from_raw_parts_mut(region.cast::<u8>().add(map_len), GUARD) };
569        guard.fill(0xab);
570        // Each filler takes the highest gap that fits the file map. The first
571        // one below the region shows that no such gap is left above it.
572        let mut fillers = Vec::new();
573        loop {
574            let filler = anonymous(map_len);
575            if filler < region {
576                unsafe { libc::munmap(filler, map_len) };
577                break;
578            }
579            fillers.push(filler);
580        }
581        unsafe { libc::munmap(region, map_len) };
582        let mut text = ForwardText::open(&path, true).unwrap();
583        assert!(
584            std::ptr::eq(text.as_str().as_ptr(), region.cast::<u8>()),
585            "the file map does not end at the guard"
586        );
587        for pos in [bytes.len() + RELEASE_STEP, 3 * RELEASE_STEP, usize::MAX] {
588            text.release_before(pos).unwrap();
589        }
590        assert_eq!(text.released, bytes.len());
591        assert!(guard.iter().all(|&b| b == 0xab));
592        assert_eq!(text.as_str().as_bytes(), bytes.as_slice());
593        unsafe { libc::munmap(guard.as_mut_ptr().cast(), GUARD) };
594        for filler in fillers {
595            unsafe { libc::munmap(filler, map_len) };
596        }
597    }
598
599    #[test]
600    fn forward_text_reports_invalid_utf8_as_the_mapped_read() {
601        let dir = tempfile::tempdir().unwrap();
602        let mut bytes = text_of_at_least(RELEASE_STEP + (200 << 10));
603        let at = bytes.len() - 100;
604        bytes[at] = 0xc3;
605        let path = dir.path().join("bad.con");
606        std::fs::write(&path, &bytes).unwrap();
607        for map in [true, false] {
608            let err = ForwardText::open(&path, map).err().unwrap().to_string();
609            assert_eq!(Err(err), mapped_text(&path), "map {map}");
610        }
611    }
612
613    #[test]
614    fn skippable_frame_magic_is_zstd() {
615        assert_eq!(
616            detect_compression(&[0x50, 0x2a, 0x4d, 0x18]),
617            Compression::Zstd
618        );
619        assert_eq!(
620            detect_compression(&[0x5f, 0x2a, 0x4d, 0x18]),
621            Compression::Zstd
622        );
623        assert_eq!(
624            detect_compression(&[0x4f, 0x2a, 0x4d, 0x18]),
625            Compression::None
626        );
627        assert_eq!(detect_compression(&[0x50, 0x2a, 0x4d]), Compression::None);
628        assert_eq!(
629            detect_compression(&[0x28, 0xb5, 0x2f, 0xfd]),
630            Compression::Zstd
631        );
632    }
633
634    fn skippable_prefix(nibble: u8, payload: &[u8]) -> Vec<u8> {
635        let mut bytes = Vec::new();
636        bytes.extend_from_slice(&(0x184D_2A50u32 + u32::from(nibble)).to_le_bytes());
637        bytes.extend_from_slice(&(payload.len() as u32).to_le_bytes());
638        bytes.extend_from_slice(payload);
639        bytes
640    }
641
642    #[cfg(not(feature = "zstd"))]
643    #[test]
644    fn a_leading_skippable_frame_needs_the_zstd_feature() {
645        let dir = tempfile::tempdir().unwrap();
646        let path = dir.path().join("skip-first.zst");
647        std::fs::write(&path, skippable_prefix(0, b"index")).unwrap();
648        let err = read_file_text(&path).unwrap_err().to_string();
649        assert!(err.contains("zstd-compressed input detected"), "{err}");
650        match open_decoded(&path) {
651            Err(err) => {
652                assert!(
653                    err.to_string().contains("zstd-compressed input detected"),
654                    "{err}"
655                );
656            }
657            Ok(_) => panic!("a skippable frame was treated as uncompressed"),
658        }
659    }
660
661    #[cfg(feature = "zstd")]
662    #[test]
663    fn a_leading_skippable_frame_decodes_with_the_frames_after_it() {
664        let text = include_str!("../resources/test/tiny_cuh2.con");
665        let frame = zstd::stream::encode_all(text.as_bytes(), 1).unwrap();
666        let dir = tempfile::tempdir().unwrap();
667        let first = dir.path().join("skip-first.con.zst");
668        let mut bytes = skippable_prefix(0, b"index");
669        bytes.extend_from_slice(&frame);
670        std::fs::write(&first, &bytes).unwrap();
671        assert_eq!(read_file_text(&first).unwrap(), text);
672        let frames = crate::iterators::read_all_frames(&first).unwrap();
673        assert_eq!(frames.len(), 1);
674
675        let mut between = frame.clone();
676        between.extend(skippable_prefix(0, b"index"));
677        between.extend_from_slice(&frame);
678        let mid = dir.path().join("skip-between.con.zst");
679        std::fs::write(&mid, &between).unwrap();
680        assert_eq!(read_file_text(&mid).unwrap(), format!("{text}{text}"));
681    }
682}