lzip_parallel/
lib.rs

1//! lzip — Pure Rust parallel ZIP decompressor.
2//!
3//! Architecture:
4//!   Parse the ZIP Central Directory (at file tail) → Vec<EntryLocation>.
5//!   Dispatch every file entry to a rayon worker that DEFLATE-decodes it
6//!   independently.  Results reassembled in Central Directory order.
7//!
8//! In-memory slice (< 10 MB): all entries decoded in one parallel pass.
9//! Streaming (Read + Seek): Central Directory parsed by seeking to tail;
10//!   entries read and decoded in parallel batches of 64 — the file is
11//!   never fully loaded into RAM.
12
13pub mod batch;
14pub mod central_dir;
15pub mod chunk;
16pub mod deflate_scan;
17pub mod entry;
18pub mod parallel;
19pub mod reader;
20
21pub use entry::{ZipEntry, ZipError};
22pub use reader::StreamingZipRead;
23
24/// Files below this threshold are decompressed in a single parallel pass.
25/// Files at or above use batched parallel decode.
26const SMALL_FILE_THRESHOLD: usize = 10 * 1024 * 1024; // 10 MB
27
28// ── Dedicated thread pool ─────────────────────────────────────────────────────
29
30use std::sync::OnceLock;
31
32static POOL: OnceLock<rayon::ThreadPool> = OnceLock::new();
33
34/// Get (or create) lzip's dedicated rayon thread pool.
35///
36/// Sized to available parallelism, separate from any caller-owned pool.
37/// Override with the `LZIP_THREADS` env var.
38pub fn thread_pool() -> &'static rayon::ThreadPool {
39    POOL.get_or_init(|| {
40        let n = std::env::var("LZIP_THREADS")
41            .ok()
42            .and_then(|v| v.parse().ok())
43            .unwrap_or_else(|| {
44                std::thread::available_parallelism()
45                    .map(|n| n.get())
46                    .unwrap_or(4)
47            });
48        rayon::ThreadPoolBuilder::new()
49            .num_threads(n)
50            .thread_name(|i| format!("lzip-{i}"))
51            .build()
52            .expect("failed to create lzip thread pool")
53    })
54}
55
56// ── Public API ────────────────────────────────────────────────────────────────
57
58/// Decompress all file entries in a ZIP byte slice.
59///
60/// For small slices (< 10 MB) decodes all entries in one parallel pass.
61/// For larger slices uses the streaming reader with a `Cursor` wrapper.
62/// Returns entries in Central Directory order; directory entries are excluded.
63pub fn decompress_zip(data: &[u8]) -> Result<Vec<ZipEntry>, ZipError> {
64    if data.len() < SMALL_FILE_THRESHOLD {
65        parallel::decompress_parallel(data)
66    } else {
67        decompress_zip_stream(std::io::Cursor::new(data))
68    }
69}
70
71/// Decompress only entries whose name contains `needle`.
72///
73/// Filters at the central directory level — non-matching entries are never
74/// read or decompressed.  Fast prefix/suffix/substring match.
75///
76/// ```no_run
77/// // Extract only matching files from a ZIP
78/// let data = std::fs::read("archive.zip").unwrap();
79/// let entries = lzip_parallel::decompress_zip_filter(&data, "pom.xml").unwrap();
80/// ```
81pub fn decompress_zip_filter(data: &[u8], needle: &str) -> Result<Vec<ZipEntry>, ZipError> {
82    if data.len() < SMALL_FILE_THRESHOLD {
83        parallel::decompress_parallel_filter(data, needle)
84    } else {
85        let reader = StreamingZipRead::with_filter(std::io::Cursor::new(data), needle)?;
86        reader.collect()
87    }
88}
89
90/// Decompress only entries whose name exactly matches one of `names`.
91///
92/// Filters at the central directory level — non-matching entries are never
93/// read or decompressed. Use this when you need specific files from a wheel/zip.
94///
95/// ```no_run
96/// let data = std::fs::read("pkg.whl").unwrap();
97/// let entries = lzip_parallel::decompress_zip_filter_set(&data, &["METADATA", "RECORD"]).unwrap();
98/// ```
99pub fn decompress_zip_filter_set(data: &[u8], names: &[&str]) -> Result<Vec<ZipEntry>, ZipError> {
100    if data.len() < SMALL_FILE_THRESHOLD {
101        parallel::decompress_parallel_filter_set(data, names)
102    } else {
103        let reader = StreamingZipRead::with_filter_set(std::io::Cursor::new(data), names)?;
104        reader.collect()
105    }
106}
107
108/// Decompress only entries whose name ends with one of the given `suffixes`.
109///
110/// Useful for extracting all files of a certain type (e.g. all `.py` files from a wheel).
111///
112/// ```no_run
113/// let data = std::fs::read("pkg.whl").unwrap();
114/// let entries = lzip_parallel::decompress_zip_filter_suffixes(&data, &[".py", ".pyi"]).unwrap();
115/// ```
116pub fn decompress_zip_filter_suffixes(data: &[u8], suffixes: &[&str]) -> Result<Vec<ZipEntry>, ZipError> {
117    if data.len() < SMALL_FILE_THRESHOLD {
118        parallel::decompress_parallel_filter_suffixes(data, suffixes)
119    } else {
120        let reader = StreamingZipRead::with_filter_suffixes(std::io::Cursor::new(data), suffixes)?;
121        reader.collect()
122    }
123}
124
125/// Decompress all file entries from any `Read + Seek` source without loading
126/// the full file into memory.
127///
128/// Parses the Central Directory by seeking to the file tail, then reads and
129/// decodes entries in parallel batches of 64.  Directory entries are excluded.
130/// Entries are returned sorted by their position in the file.
131pub fn decompress_zip_stream<R: std::io::Read + std::io::Seek>(
132    source: R,
133) -> Result<Vec<ZipEntry>, ZipError> {
134    StreamingZipRead::new(source)?.collect()
135}
lzip_parallel/lib.rs

lzip_parallel/
lib.rs
