lzip_parallel/
lib.rs

1//! lzip — Pure Rust parallel ZIP decompressor.
2//!
3//! Architecture:
4//!   Parse the ZIP Central Directory (at file tail) → Vec<EntryLocation>.
5//!   Dispatch every file entry to a rayon worker that DEFLATE-decodes it
6//!   independently.  Results reassembled in Central Directory order.
7//!
8//! In-memory slice (< 10 MB): all entries decoded in one parallel pass.
9//! Streaming (Read + Seek): Central Directory parsed by seeking to tail;
10//!   entries read and decoded in parallel batches of 64 — the file is
11//!   never fully loaded into RAM.
12
13pub mod batch;
14pub mod central_dir;
15pub mod chunk;
16pub mod deflate_scan;
17pub mod entry;
18pub mod parallel;
19pub mod reader;
20
21pub use entry::{ZipEntry, ZipError};
22pub use reader::StreamingZipRead;
23
24/// Files below this threshold are decompressed in a single parallel pass.
25/// Files at or above use batched parallel decode.
26const SMALL_FILE_THRESHOLD: usize = 10 * 1024 * 1024; // 10 MB
27
28// ── Dedicated thread pool ─────────────────────────────────────────────────────
29
30use std::sync::OnceLock;
31
32static POOL: OnceLock<rayon::ThreadPool> = OnceLock::new();
33
34/// Get (or create) lzip's dedicated rayon thread pool.
35///
36/// Sized to available parallelism, separate from any caller-owned pool.
37/// Override with the `LZIP_THREADS` env var.
38pub fn thread_pool() -> &'static rayon::ThreadPool {
39    POOL.get_or_init(|| {
40        let n = std::env::var("LZIP_THREADS")
41            .ok()
42            .and_then(|v| v.parse().ok())
43            .unwrap_or_else(|| {
44                std::thread::available_parallelism()
45                    .map(|n| n.get())
46                    .unwrap_or(4)
47            });
48        rayon::ThreadPoolBuilder::new()
49            .num_threads(n)
50            .thread_name(|i| format!("lzip-{i}"))
51            .build()
52            .expect("failed to create lzip thread pool")
53    })
54}
55
56// ── Public API ────────────────────────────────────────────────────────────────
57
58/// Decompress all file entries in a ZIP byte slice.
59///
60/// For small slices (< 10 MB) decodes all entries in one parallel pass.
61/// For larger slices uses the streaming reader with a `Cursor` wrapper.
62/// Returns entries in Central Directory order; directory entries are excluded.
63pub fn decompress_zip(data: &[u8]) -> Result<Vec<ZipEntry>, ZipError> {
64    if data.len() < SMALL_FILE_THRESHOLD {
65        parallel::decompress_parallel(data)
66    } else {
67        decompress_zip_stream(std::io::Cursor::new(data))
68    }
69}
70
71/// Decompress only entries whose name contains `needle`.
72///
73/// Filters at the central directory level — non-matching entries are never
74/// read or decompressed.  Fast prefix/suffix/substring match.
75///
76/// ```no_run
77/// // Extract only .txt files from a ZIP
78/// let entries = lzip::decompress_zip_filter(data, "pom.xml").unwrap();
79/// ```
80pub fn decompress_zip_filter(data: &[u8], needle: &str) -> Result<Vec<ZipEntry>, ZipError> {
81    if data.len() < SMALL_FILE_THRESHOLD {
82        parallel::decompress_parallel_filter(data, needle)
83    } else {
84        let reader = StreamingZipRead::with_filter(std::io::Cursor::new(data), needle)?;
85        reader.collect()
86    }
87}
88
89/// Decompress only entries whose name exactly matches one of `names`.
90///
91/// Filters at the central directory level — non-matching entries are never
92/// read or decompressed. Use this when you need specific files from a wheel/zip.
93///
94/// ```no_run
95/// let entries = lzip::decompress_zip_filter_set(data, &["METADATA", "RECORD"]).unwrap();
96/// ```
97pub fn decompress_zip_filter_set(data: &[u8], names: &[&str]) -> Result<Vec<ZipEntry>, ZipError> {
98    if data.len() < SMALL_FILE_THRESHOLD {
99        parallel::decompress_parallel_filter_set(data, names)
100    } else {
101        let reader = StreamingZipRead::with_filter_set(std::io::Cursor::new(data), names)?;
102        reader.collect()
103    }
104}
105
106/// Decompress only entries whose name ends with one of the given `suffixes`.
107///
108/// Useful for extracting all files of a certain type (e.g. all `.py` files from a wheel).
109///
110/// ```no_run
111/// let entries = lzip::decompress_zip_filter_suffixes(data, &[".py", ".pyi"]).unwrap();
112/// ```
113pub fn decompress_zip_filter_suffixes(data: &[u8], suffixes: &[&str]) -> Result<Vec<ZipEntry>, ZipError> {
114    if data.len() < SMALL_FILE_THRESHOLD {
115        parallel::decompress_parallel_filter_suffixes(data, suffixes)
116    } else {
117        let reader = StreamingZipRead::with_filter_suffixes(std::io::Cursor::new(data), suffixes)?;
118        reader.collect()
119    }
120}
121
122/// Decompress all file entries from any `Read + Seek` source without loading
123/// the full file into memory.
124///
125/// Parses the Central Directory by seeking to the file tail, then reads and
126/// decodes entries in parallel batches of 64.  Directory entries are excluded.
127/// Entries are returned sorted by their position in the file.
128pub fn decompress_zip_stream<R: std::io::Read + std::io::Seek>(
129    source: R,
130) -> Result<Vec<ZipEntry>, ZipError> {
131    StreamingZipRead::new(source)?.collect()
132}
lzip_parallel/lib.rs

lzip_parallel/
lib.rs
