lzip_parallel/
lib.rs

1//! lzip — Pure Rust parallel ZIP decompressor.
2//!
3//! Architecture:
4//!   Parse the ZIP Central Directory (at file tail) → Vec<EntryLocation>.
5//!   Dispatch every file entry to a rayon worker that DEFLATE-decodes it
6//!   independently.  Results reassembled in Central Directory order.
7//!
8//! In-memory slice (< 10 MB): all entries decoded in one parallel pass.
9//! Streaming (Read + Seek): Central Directory parsed by seeking to tail;
10//!   entries read and decoded in parallel batches of 64 — the file is
11//!   never fully loaded into RAM.
12
13pub mod batch;
14pub mod central_dir;
15pub mod chunk;
16pub mod deflate_scan;
17pub mod entry;
18pub mod parallel;
19pub mod reader;
20
21pub use entry::{ZipEntry, ZipError};
22pub use reader::StreamingZipRead;
23
24/// **Introspection / emit marker** — record one functional-status row for the
25/// nornir test matrix. Wraps `nornir_testmatrix::functional_status` behind the
26/// `testmatrix` feature (a compiled-out `#[inline]` no-op otherwise, with no
27/// nornir dep in the default build). Mirrors the korp-collectors reference
28/// wiring so `nornir test --features testmatrix` SEES the lzip CLI.
29#[inline]
30pub fn functional_status(component: &str, check: &str, ok: bool, detail: &str) {
31    #[cfg(feature = "testmatrix")]
32    nornir_testmatrix::functional_status(component, check, ok, detail);
33    #[cfg(not(feature = "testmatrix"))]
34    {
35        let _ = (component, check, ok, detail);
36    }
37}
38
39/// Files below this threshold are decompressed in a single parallel pass.
40/// Files at or above use batched parallel decode.
41const SMALL_FILE_THRESHOLD: usize = 10 * 1024 * 1024; // 10 MB
42
43// ── Parallelism ───────────────────────────────────────────────────────────────
44//
45// Per-entry / per-segment DEFLATE decode fans out through the constellation's
46// ONE shared no-barrier engine — `gatling::gatling_forkjoin::gatling_for_each`
47// (N workers draining a shared atomic cursor, one per core). The old private
48// rayon `ThreadPool` (`thread_pool()` / `LZIP_THREADS`) is gone; rayon is no
49// longer a dependency.
50
51// ── Public API ────────────────────────────────────────────────────────────────
52
53/// Decompress all file entries in a ZIP byte slice.
54///
55/// For small slices (< 10 MB) decodes all entries in one parallel pass.
56/// For larger slices uses the streaming reader with a `Cursor` wrapper.
57/// Returns entries in Central Directory order; directory entries are excluded.
58pub fn decompress_zip(data: &[u8]) -> Result<Vec<ZipEntry>, ZipError> {
59    if data.len() < SMALL_FILE_THRESHOLD {
60        parallel::decompress_parallel(data)
61    } else {
62        decompress_zip_stream(std::io::Cursor::new(data))
63    }
64}
65
66/// Decompress only entries whose name contains `needle`.
67///
68/// Filters at the central directory level — non-matching entries are never
69/// read or decompressed.  Fast prefix/suffix/substring match.
70///
71/// ```no_run
72/// // Extract only matching files from a ZIP
73/// let data = std::fs::read("archive.zip").unwrap();
74/// let entries = lzip_parallel::decompress_zip_filter(&data, "pom.xml").unwrap();
75/// ```
76pub fn decompress_zip_filter(data: &[u8], needle: &str) -> Result<Vec<ZipEntry>, ZipError> {
77    if data.len() < SMALL_FILE_THRESHOLD {
78        parallel::decompress_parallel_filter(data, needle)
79    } else {
80        let reader = StreamingZipRead::with_filter(std::io::Cursor::new(data), needle)?;
81        reader.collect()
82    }
83}
84
85/// Decompress only entries whose name exactly matches one of `names`.
86///
87/// Filters at the central directory level — non-matching entries are never
88/// read or decompressed. Use this when you need specific files from a wheel/zip.
89///
90/// ```no_run
91/// let data = std::fs::read("pkg.whl").unwrap();
92/// let entries = lzip_parallel::decompress_zip_filter_set(&data, &["METADATA", "RECORD"]).unwrap();
93/// ```
94pub fn decompress_zip_filter_set(data: &[u8], names: &[&str]) -> Result<Vec<ZipEntry>, ZipError> {
95    if data.len() < SMALL_FILE_THRESHOLD {
96        parallel::decompress_parallel_filter_set(data, names)
97    } else {
98        let reader = StreamingZipRead::with_filter_set(std::io::Cursor::new(data), names)?;
99        reader.collect()
100    }
101}
102
103/// Decompress only entries whose name ends with one of the given `suffixes`.
104///
105/// Useful for extracting all files of a certain type (e.g. all `.py` files from a wheel).
106///
107/// ```no_run
108/// let data = std::fs::read("pkg.whl").unwrap();
109/// let entries = lzip_parallel::decompress_zip_filter_suffixes(&data, &[".py", ".pyi"]).unwrap();
110/// ```
111pub fn decompress_zip_filter_suffixes(data: &[u8], suffixes: &[&str]) -> Result<Vec<ZipEntry>, ZipError> {
112    if data.len() < SMALL_FILE_THRESHOLD {
113        parallel::decompress_parallel_filter_suffixes(data, suffixes)
114    } else {
115        let reader = StreamingZipRead::with_filter_suffixes(std::io::Cursor::new(data), suffixes)?;
116        reader.collect()
117    }
118}
119
120/// Decompress all file entries from any `Read + Seek` source without loading
121/// the full file into memory.
122///
123/// Parses the Central Directory by seeking to the file tail, then reads and
124/// decodes entries in parallel batches of 64.  Directory entries are excluded.
125/// Entries are returned sorted by their position in the file.
126pub fn decompress_zip_stream<R: std::io::Read + std::io::Seek>(
127    source: R,
128) -> Result<Vec<ZipEntry>, ZipError> {
129    StreamingZipRead::new(source)?.collect()
130}
lzip_parallel/lib.rs

lzip_parallel/
lib.rs
