use std::collections::BTreeMap;
use std::fs::{self, File, OpenOptions};
use std::io::{self, BufRead, BufReader, Read, Seek, SeekFrom, Write};
use std::path::{Path, PathBuf};
use std::time::Instant;
use flate2::{
Compression, CrcReader, Decompress, FlushDecompress, Status, bufread::GzDecoder,
write::GzEncoder,
};
use tempfile::{NamedTempFile, TempDir};
use zip::{CompressionMethod, ZipArchive, ZipWriter, write::SimpleFileOptions};
use crate::{
Adapter, Artifact, ContentKind, ContentProfile, Conversion, ConversionFuture, ConversionKind,
Error, Format, Limits, MediaKind, Result, StepRequest, Stream,
};
const FORMATS: [&str; 5] = ["zip", "tar", "tar.gz", "tgz", "gz"];
const MAX_NAME: usize = 1024;
const BUFFER_SIZE: usize = 32 * 1024;
const CODEC_MEMORY: u64 = 8 * 1024 * 1024;
pub struct ArchivesAdapter;
impl Adapter for ArchivesAdapter {
fn name(&self) -> &str {
"archives"
}
fn formats(&self) -> Vec<Format> {
[
("zip", "ZIP", "zip", &["application/zip"][..], None),
("tar", "TAR", "tar", &["application/x-tar"], None),
(
"tar.gz",
"Gzip-compressed TAR",
"tar.gz",
&["application/gzip", "application/x-tar"],
Some("gz"),
),
(
"tgz",
"Gzip-compressed TAR (TGZ)",
"tgz",
&["application/gzip", "application/x-tar"],
Some("gz"),
),
("gz", "Gzip", "gz", &["application/gzip"], None),
]
.into_iter()
.map(|(id, name, extension, mime, container)| {
let format = Format::new(id, name, MediaKind::Archive)
.with_extensions(&[extension])
.with_mime(mime)
.with_payloads(&[ContentKind::new(MediaKind::Archive)]);
match container {
Some(container) => format.in_container(container),
None => format,
}
})
.collect()
}
fn inspect<'a>(
&'a self,
path: &'a std::path::Path,
format: &'a str,
limits: &'a Limits,
) -> crate::InspectFuture<'a> {
if !FORMATS.contains(&format) {
return Box::pin(async { None });
}
let path = path.to_path_buf();
let format = format.to_owned();
let limits = limits.clone();
Box::pin(async move {
let outcome =
tokio::task::spawn_blocking(move || profile_archive(&path, &format, &limits)).await;
Some(outcome.unwrap_or_else(|err| {
Err(Error::Conversion(format!("archive probe panicked: {err}")))
}))
})
}
fn conversions(&self) -> Vec<Conversion> {
let mut conversions = Vec::new();
for target in FORMATS {
conversions.push(
Conversion::new(
"*",
target,
"archives",
"package",
ConversionKind::Package,
20,
)
.with_note("files are placed in a container without touching their bytes"),
);
}
for source in FORMATS {
conversions.push(
Conversion::new(
source,
"bin",
"archives",
"extract",
ConversionKind::Extract,
10,
)
.with_note("every member is returned as one artifact, paths are discarded"),
);
for target in FORMATS {
if source != target {
conversions.push(
Conversion::new(
source,
target,
"archives",
"repack",
ConversionKind::Lossless,
5,
)
.with_note(
"members are re-compressed; stored bytes survive, container layout may change",
),
);
}
}
}
conversions
}
fn convert<'a>(&'a self, request: &'a StepRequest) -> ConversionFuture<'a> {
Box::pin(async move {
let request = request.clone();
let started = Instant::now();
tokio::task::spawn_blocking(move || execute(&request, started))
.await
.map_err(|error| Error::Security(format!("archive worker failed: {error}")))?
})
}
}
fn archive_timeout() -> Error {
Error::Timeout {
backend: "archives".into(),
}
}
fn security(error: impl std::fmt::Display) -> Error {
Error::Security(format!("invalid or unsupported archive: {error}"))
}
#[derive(Debug, thiserror::Error)]
#[error("archive output byte limit")]
struct OutputLimit;
fn output_error(error: io::Error) -> Error {
if error.kind() == io::ErrorKind::TimedOut {
archive_timeout()
} else if error
.get_ref()
.is_some_and(|inner| inner.is::<OutputLimit>())
{
Error::Limit("archive output bytes".into())
} else {
Error::Io(error)
}
}
fn zip_error(error: zip::result::ZipError) -> Error {
match error {
zip::result::ZipError::Io(error) => output_error(error),
error => security(error),
}
}
struct BoundedWriter<W> {
inner: W,
position: u64,
limit: u64,
started: Instant,
timeout: std::time::Duration,
}
impl<W> BoundedWriter<W> {
fn new(inner: W, budget: &Budget<'_>) -> Self {
Self {
inner,
position: 0,
limit: budget.limits.max_output_bytes,
started: budget.started,
timeout: budget.limits.timeout,
}
}
}
impl<W: Write> Write for BoundedWriter<W> {
fn write(&mut self, bytes: &[u8]) -> io::Result<usize> {
if self.started.elapsed() >= self.timeout {
return Err(io::Error::new(io::ErrorKind::TimedOut, "archive deadline"));
}
if bytes.len() as u64 > self.limit.saturating_sub(self.position) {
return Err(io::Error::other(OutputLimit));
}
let written = self.inner.write(bytes)?;
self.position += written as u64;
Ok(written)
}
fn flush(&mut self) -> io::Result<()> {
self.inner.flush()
}
}
impl<W: Seek> Seek for BoundedWriter<W> {
fn seek(&mut self, position: SeekFrom) -> io::Result<u64> {
let next = match position {
SeekFrom::Start(next) => Some(next),
SeekFrom::Current(offset) => self.position.checked_add_signed(offset),
SeekFrom::End(offset) => self
.inner
.seek(SeekFrom::End(0))?
.checked_add_signed(offset),
};
let Some(next) = next.filter(|next| *next <= self.limit) else {
return Err(io::Error::other(OutputLimit));
};
self.position = self.inner.seek(SeekFrom::Start(next))?;
Ok(self.position)
}
}
struct Budget<'a> {
limits: &'a Limits,
started: Instant,
entries: usize,
files: usize,
staged: usize,
expanded: u64,
payload: u64,
names: BTreeMap<String, bool>,
}
impl Budget<'_> {
fn check_time(&self) -> Result<()> {
if self.started.elapsed() >= self.limits.timeout {
Err(archive_timeout())
} else {
Ok(())
}
}
fn remaining(&self) -> u64 {
self.limits.max_output_bytes.saturating_sub(self.expanded)
}
fn entry(&mut self, name: &[u8], directory: bool) -> Result<String> {
self.check_time()?;
if self.entries >= self.limits.max_archive_entries {
return Err(Error::Limit(
"archive entry count (including directories)".into(),
));
}
if !directory && self.files >= self.limits.max_artifacts {
return Err(Error::Limit("archive file count".into()));
}
if (self.entries as u64).saturating_add(1).saturating_mul(8192)
> self.limits.max_memory_bytes.saturating_sub(CODEC_MEMORY)
{
return Err(Error::Limit("archive metadata memory".into()));
}
let name = valid_name(name, directory)?;
let folded = name.to_ascii_lowercase();
if self.names.contains_key(&folded) {
return Err(security("duplicate or case-folded entry name"));
}
for (position, _) in folded.match_indices('/') {
if self.names.get(&folded[..position]) == Some(&false) {
return Err(security("file is also an ancestor directory"));
}
}
if !directory {
let prefix = format!("{folded}/");
if self
.names
.range(prefix.clone()..)
.next()
.is_some_and(|(name, _)| name.starts_with(&prefix))
{
return Err(security("file conflicts with existing descendants"));
}
}
self.names.insert(folded, directory);
self.entries += 1;
self.files += usize::from(!directory);
Ok(name)
}
}
fn valid_name(bytes: &[u8], directory: bool) -> Result<String> {
if bytes.is_empty()
|| bytes.len() > MAX_NAME
|| bytes
.iter()
.any(|byte| !byte.is_ascii() || byte.is_ascii_control())
{
return Err(security("entry names must be bounded, printable ASCII"));
}
let mut name = String::from_utf8(bytes.to_vec())
.map_err(security)?
.replace('\\', "/");
if directory && name.ends_with('/') {
name.pop();
}
for component in name.split('/') {
if component.is_empty()
|| component == "."
|| component == ".."
|| component.ends_with(['.', ' '])
|| component.contains([':', '<', '>', '"', '|', '?', '*'])
{
return Err(security("absolute, ambiguous, or traversing entry path"));
}
let stem = component
.split('.')
.next()
.unwrap_or_default()
.to_ascii_uppercase();
if matches!(stem.as_str(), "CON" | "PRN" | "AUX" | "NUL")
|| (stem.len() == 4
&& (stem.starts_with("COM") || stem.starts_with("LPT"))
&& matches!(stem.as_bytes()[3], b'1'..=b'9'))
{
return Err(security("reserved Windows device name"));
}
}
Ok(name)
}
struct Entry {
name: String,
path: Option<PathBuf>,
size: u64,
}
fn private_file(path: &Path) -> Result<File> {
let mut options = OpenOptions::new();
options.write(true).read(true).create_new(true);
#[cfg(unix)]
{
use std::os::unix::fs::OpenOptionsExt;
options.mode(0o600);
}
Ok(options.open(path)?)
}
fn regular_input(path: &Path) -> Result<(File, u64)> {
if !fs::symlink_metadata(path)?.file_type().is_file() {
return Err(security("input is not a regular file"));
}
let file = File::open(path)?;
let metadata = file.metadata()?;
if !metadata.is_file() {
return Err(security("input is not a regular file"));
}
Ok((file, metadata.len()))
}
fn copy_bounded(
reader: &mut (impl Read + ?Sized),
writer: &mut impl Write,
limit: u64,
claimed: Option<u64>,
budget: &Budget<'_>,
) -> Result<u64> {
let mut total = 0u64;
let mut buffer = [0u8; BUFFER_SIZE];
loop {
budget.check_time()?;
let length = limit
.saturating_sub(total)
.saturating_add(1)
.min(BUFFER_SIZE as u64);
let count = reader
.read(&mut buffer[..length as usize])
.map_err(security)?;
if count == 0 {
break;
}
if count as u64 > limit.saturating_sub(total) {
return Err(Error::Limit(
"archive decompressed or input bytes/expansion ratio".into(),
));
}
total += count as u64;
if claimed.is_some_and(|claimed| total > claimed) {
return Err(security("entry is larger than its claimed size"));
}
writer.write_all(&buffer[..count]).map_err(output_error)?;
}
if claimed.is_some_and(|claimed| total != claimed) {
return Err(security("entry is shorter than its claimed size"));
}
Ok(total)
}
fn stage_entry(
reader: &mut (impl Read + ?Sized),
name: String,
claimed: u64,
cap: u64,
work: &Path,
budget: &mut Budget<'_>,
) -> Result<Entry> {
let basename = name.rsplit('/').next().unwrap_or("data.bin");
let basename: String = basename
.chars()
.take(80)
.map(|character| {
if character.is_ascii_alphanumeric() || matches!(character, '.' | '-' | '_') {
character
} else {
'_'
}
})
.collect();
budget.staged += 1;
let path = work.join(format!("{:06}-{basename}", budget.staged));
let mut output = private_file(&path)?;
let cap = cap.min(
budget
.limits
.max_output_bytes
.saturating_sub(budget.payload),
);
if claimed > cap {
return Err(Error::Limit(
"archive claimed decompressed size or expansion ratio".into(),
));
}
let size = copy_bounded(reader, &mut output, cap, Some(claimed), budget)?;
if output.metadata()?.len() != size {
return Err(security("staged entry length changed"));
}
budget.payload += size;
Ok(Entry {
name,
path: Some(path),
size,
})
}
fn execute(request: &StepRequest, started: Instant) -> Result<Vec<Artifact>> {
if request.inputs.is_empty() {
return Err(Error::Invalid(
"archive operation requires input files".into(),
));
}
if request.inputs.len() > request.limits.max_artifacts {
return Err(Error::Limit("archive input artifact count".into()));
}
if request
.inputs
.iter()
.any(|input| input.format != request.inputs[0].format)
{
return Err(Error::Invalid(
"archive inputs must have the same format".into(),
));
}
if request.from != "*"
&& request
.inputs
.iter()
.any(|input| input.format != request.from)
{
return Err(Error::Invalid(
"archive input format does not match the request".into(),
));
}
let packaging = request.operation == "package" && FORMATS.contains(&request.to.as_str());
let extracting = request.operation == "extract" && request.to == "bin";
let repacking = request.operation == "repack" && FORMATS.contains(&request.to.as_str());
if !packaging && (!(extracting || repacking) || !FORMATS.contains(&request.from.as_str())) {
return Err(Error::Invalid("unsupported archive operation".into()));
}
if request.limits.max_memory_bytes < CODEC_MEMORY {
return Err(Error::Limit("archive codec memory".into()));
}
let mut budget = Budget {
limits: &request.limits,
started,
entries: 0,
files: 0,
staged: 0,
expanded: 0,
payload: 0,
names: BTreeMap::new(),
};
budget.check_time()?;
if !fs::symlink_metadata(&request.output_dir)?
.file_type()
.is_dir()
{
return Err(security("output directory is not a real directory"));
}
let work = TempDir::new_in(&request.output_dir)?;
let mut entries = Vec::new();
let mut input_bytes = 0u64;
for input in &request.inputs {
let (mut source, length) = regular_input(&input.path)?;
let remaining = request.limits.max_input_bytes.saturating_sub(input_bytes);
if length > remaining {
return Err(Error::Limit("archive input bytes".into()));
}
let mut snapshot = NamedTempFile::new_in(work.path())?;
input_bytes += copy_bounded(
&mut source,
snapshot.as_file_mut(),
remaining,
Some(length),
&budget,
)?;
snapshot.as_file_mut().rewind()?;
if packaging {
let name = input
.path
.file_name()
.and_then(|name| name.to_str())
.ok_or_else(|| security("input filename is not portable"))?;
let name = budget.entry(name.as_bytes(), false)?;
let (_, path) = snapshot.keep().map_err(|error| Error::Io(error.error))?;
entries.push(Entry {
name,
path: Some(path),
size: length,
});
} else {
let source_cap = length.saturating_mul(request.limits.max_expansion_ratio);
match request.from.as_str() {
"zip" => entries.extend(read_zip(
snapshot.as_file_mut(),
length,
source_cap,
work.path(),
&mut budget,
)?),
"tar" => {
if length > budget.remaining().min(source_cap) {
return Err(Error::Limit("TAR expanded bytes".into()));
}
budget.expanded += length;
entries.extend(read_tar(
snapshot.as_file_mut(),
length,
work.path(),
&mut budget,
)?);
}
"gz" | "tar.gz" | "tgz" => {
let (mut expanded, name, size) =
read_gzip(snapshot.as_file_mut(), source_cap, work.path(), &budget)?;
budget.expanded += size;
if request.from == "gz" {
let name = budget.entry(name.as_bytes(), false)?;
entries.push(stage_entry(
expanded.as_file_mut(),
name,
size,
size,
work.path(),
&mut budget,
)?);
} else {
entries.extend(read_tar(
expanded.as_file_mut(),
size,
work.path(),
&mut budget,
)?);
}
}
_ => return Err(Error::Invalid("unsupported archive source".into())),
}
}
}
if entries.iter().all(|entry| entry.path.is_none()) {
return Err(Error::Invalid("archive contains no regular files".into()));
}
let artifacts = if extracting {
entries
.into_iter()
.filter_map(|entry| match entry.path {
Some(path) => {
let mut artifact = Artifact::new(path, "bin");
artifact.logical_name = Some(entry.name);
Some(artifact)
}
None => None,
})
.collect()
} else {
let path = work.path().join(format!("archive.{}", request.to));
write_archive(&entries, &request.to, &path, &budget)?;
for entry in entries {
if let Some(path) = entry.path {
fs::remove_file(path)?;
}
}
vec![Artifact::new(path, request.to.clone())]
};
budget.check_time()?;
let _ = work.keep();
Ok(artifacts)
}
fn read_exact(reader: &mut impl Read, bytes: &mut [u8]) -> Result<()> {
reader.read_exact(bytes).map_err(security)
}
fn little16(bytes: &[u8], offset: usize) -> u16 {
u16::from_le_bytes([bytes[offset], bytes[offset + 1]])
}
fn little32(bytes: &[u8], offset: usize) -> u32 {
u32::from_le_bytes([
bytes[offset],
bytes[offset + 1],
bytes[offset + 2],
bytes[offset + 3],
])
}
struct ZipEntry {
raw_name: Vec<u8>,
name: String,
directory: bool,
size: u64,
compressed: u64,
crc: u32,
method: u16,
start: u64,
end: u64,
}
fn zip_extra(reader: &mut File, length: u16) -> Result<()> {
if length > 1024 {
return Err(Error::Limit("ZIP extra field metadata bytes".into()));
}
let mut left = u64::from(length);
while left != 0 {
if left < 4 {
return Err(security("truncated ZIP extra field"));
}
let mut header = [0u8; 4];
read_exact(reader, &mut header)?;
let size = u64::from(little16(&header, 2));
if !matches!(little16(&header, 0), 0x5455 | 0x000a | 0x7875) || size > left - 4 {
return Err(security("unsupported ZIP extra field"));
}
reader
.seek(SeekFrom::Current(size as i64))
.map_err(security)?;
left -= 4 + size;
}
Ok(())
}
fn zip_headers(
file: &mut File,
length: u64,
source_cap: u64,
budget: &mut Budget<'_>,
) -> Result<Vec<ZipEntry>> {
let tail_length = length.min(65_557) as usize;
let mut tail = vec![0u8; tail_length];
file.seek(SeekFrom::Start(length - tail_length as u64))
.map_err(security)?;
read_exact(file, &mut tail)?;
let end = (0..tail_length.saturating_sub(21))
.rev()
.find(|&offset| {
tail[offset..].starts_with(b"PK\x05\x06")
&& offset + 22 + usize::from(little16(&tail, offset + 20)) == tail_length
})
.ok_or_else(|| security("missing ZIP end record"))?;
let footer = &tail[end..end + 22];
let count = usize::from(little16(footer, 10));
let central_size = u64::from(little32(footer, 12));
let central_start = u64::from(little32(footer, 16));
let central_end = length - tail_length as u64 + end as u64;
if little16(footer, 4) != 0
|| little16(footer, 6) != 0
|| usize::from(little16(footer, 8)) != count
|| count == usize::from(u16::MAX)
|| central_size == u64::from(u32::MAX)
|| central_start == u64::from(u32::MAX)
|| central_start + central_size != central_end
{
return Err(security("ZIP64, multi-disk, or inconsistent ZIP directory"));
}
if count
> budget
.limits
.max_archive_entries
.saturating_sub(budget.entries)
{
return Err(Error::Limit(
"ZIP entry count (including directories)".into(),
));
}
let mut entries = Vec::new();
let mut cursor = central_start;
let mut claimed_total = 0u64;
for _ in 0..count {
budget.check_time()?;
if cursor.saturating_add(46) > central_end {
return Err(security("truncated ZIP central header"));
}
file.seek(SeekFrom::Start(cursor)).map_err(security)?;
let mut central = [0u8; 46];
read_exact(file, &mut central)?;
if ¢ral[..4] != b"PK\x01\x02" {
return Err(security("invalid ZIP central signature"));
}
let flags = little16(¢ral, 8);
let method = little16(¢ral, 10);
let compressed = u64::from(little32(¢ral, 20));
let size = u64::from(little32(¢ral, 24));
let name_len = usize::from(little16(¢ral, 28));
let extra_len = little16(¢ral, 30);
let comment_len = u64::from(little16(¢ral, 32));
let start = u64::from(little32(¢ral, 42));
cursor += 46 + name_len as u64 + u64::from(extra_len) + comment_len;
if comment_len > 1024 {
return Err(Error::Limit("ZIP entry comment bytes".into()));
}
if cursor > central_end
|| name_len == 0
|| name_len > MAX_NAME
|| !matches!(central[5], 0 | 3)
|| little16(¢ral, 6) > 20
|| flags & !0x080e != 0
|| !matches!(method, 0 | 8)
|| (method == 0 && flags & 6 != 0)
|| little16(¢ral, 34) != 0
|| compressed == u64::from(u32::MAX)
|| size == u64::from(u32::MAX)
{
return Err(security("encrypted, unsupported, or malformed ZIP entry"));
}
let mut raw_name = vec![0u8; name_len];
read_exact(file, &mut raw_name)?;
zip_extra(file, extra_len)?;
let directory = raw_name.ends_with(b"/") || raw_name.ends_with(b"\\");
let attributes = little32(¢ral, 38);
let file_type = (attributes >> 16) & 0o170000;
if !matches!(file_type, 0 | 0o100000 | 0o040000)
|| (file_type == 0o040000 && !directory)
|| (file_type == 0o100000 && directory)
|| (attributes & 0x10 != 0 && !directory)
|| attributes & 8 != 0
|| (directory && size != 0)
|| (method == 0 && compressed != size)
{
return Err(security("ZIP entry is not a regular file or directory"));
}
let name = budget.entry(&raw_name, directory)?;
let cap = compressed.saturating_mul(budget.limits.max_expansion_ratio);
claimed_total = claimed_total
.checked_add(size)
.ok_or_else(|| Error::Limit("ZIP size overflow".into()))?;
if size > cap || claimed_total > source_cap.min(budget.remaining()) {
return Err(Error::Limit(
"ZIP claimed bytes or compression ratio".into(),
));
}
if start.saturating_add(30) > central_start {
return Err(security("ZIP local header outside payload region"));
}
file.seek(SeekFrom::Start(start)).map_err(security)?;
let mut local = [0u8; 30];
read_exact(file, &mut local)?;
let local_extra = little16(&local, 28);
let data_start = start + 30 + name_len as u64 + u64::from(local_extra);
let mut end = data_start.saturating_add(compressed);
if &local[..4] != b"PK\x03\x04"
|| little16(&local, 4) != little16(¢ral, 6)
|| little16(&local, 6) != flags
|| little16(&local, 8) != method
|| usize::from(little16(&local, 26)) != name_len
|| end > central_start
{
return Err(security("inconsistent ZIP local header"));
}
for (local_offset, central_offset) in [(14, 16), (18, 20), (22, 24)] {
let value = little32(&local, local_offset);
if value != little32(¢ral, central_offset) && !(flags & 8 != 0 && value == 0) {
return Err(security("ZIP local and central sizes/checksum disagree"));
}
}
let mut local_name = vec![0u8; name_len];
read_exact(file, &mut local_name)?;
if local_name != raw_name {
return Err(security("ZIP local and central names disagree"));
}
zip_extra(file, local_extra)?;
if flags & 8 != 0 {
file.seek(SeekFrom::Start(end)).map_err(security)?;
let mut descriptor = [0u8; 12];
read_exact(file, &mut descriptor[..4])?;
if &descriptor[..4] == b"PK\x07\x08" {
end += 4;
read_exact(file, &mut descriptor[..4])?;
}
read_exact(file, &mut descriptor[4..])?;
end += 12;
if end > central_start
|| little32(&descriptor, 0) != little32(¢ral, 16)
|| u64::from(little32(&descriptor, 4)) != compressed
|| u64::from(little32(&descriptor, 8)) != size
{
return Err(security("invalid ZIP data descriptor"));
}
}
entries.push(ZipEntry {
raw_name,
name,
directory,
size,
compressed,
crc: little32(¢ral, 16),
method,
start,
end,
});
}
if cursor != central_end {
return Err(security("unaccounted ZIP central records"));
}
let mut ranges: Vec<_> = entries
.iter()
.map(|entry| (entry.start, entry.end))
.collect();
ranges.sort_unstable();
let mut previous = 0;
for (start, end) in ranges {
if start != previous {
return Err(security("overlapping, hidden, or prefixed ZIP entries"));
}
previous = end;
}
if previous != central_start {
return Err(security("unaccounted ZIP payload bytes"));
}
Ok(entries)
}
struct StrictDeflate<R> {
reader: R,
decoder: Decompress,
ended: bool,
}
impl<R: BufRead> Read for StrictDeflate<R> {
fn read(&mut self, output: &mut [u8]) -> io::Result<usize> {
if output.is_empty() || self.ended {
return Ok(0);
}
loop {
let input = self.reader.fill_buf()?;
let before_in = self.decoder.total_in();
let before_out = self.decoder.total_out();
let status = self
.decoder
.decompress(input, output, FlushDecompress::None)
.map_err(|error| io::Error::new(io::ErrorKind::InvalidData, error))?;
let consumed = (self.decoder.total_in() - before_in) as usize;
let produced = (self.decoder.total_out() - before_out) as usize;
self.reader.consume(consumed);
self.ended = status == Status::StreamEnd;
if produced != 0 || self.ended {
return Ok(produced);
}
if consumed == 0 {
return Err(io::Error::new(
io::ErrorKind::UnexpectedEof,
"incomplete DEFLATE stream",
));
}
}
}
}
fn read_zip(
file: &mut File,
length: u64,
source_cap: u64,
work: &Path,
budget: &mut Budget<'_>,
) -> Result<Vec<Entry>> {
let headers = zip_headers(file, length, source_cap, budget)?;
let mut archive = ZipArchive::new(file).map_err(security)?;
if archive.len() != headers.len() {
return Err(security("ZIP directory entry count changed"));
}
let mut entries = Vec::new();
let mut source_total = 0u64;
for (index, header) in headers.into_iter().enumerate() {
let file = archive.by_index_raw(index).map_err(security)?;
if file.name_raw() != header.raw_name
|| file.size() != header.size
|| file.compressed_size() != header.compressed
|| file.encrypted()
{
return Err(security("ZIP metadata interpretation disagrees"));
}
let cap = header
.compressed
.saturating_mul(budget.limits.max_expansion_ratio)
.min(source_cap.saturating_sub(source_total))
.min(budget.remaining());
let mut stage = |reader: &mut dyn Read| -> Result<Entry> {
if header.directory {
copy_bounded(reader, &mut io::sink(), cap, Some(0), budget)?;
Ok(Entry {
name: header.name.clone(),
path: None,
size: 0,
})
} else {
stage_entry(reader, header.name.clone(), header.size, cap, work, budget)
}
};
let entry = if header.method == 8 {
let decoder = StrictDeflate {
reader: BufReader::new(file),
decoder: Decompress::new(false),
ended: false,
};
let mut reader = CrcReader::new(decoder);
let entry = stage(&mut reader)?;
if reader.crc().sum() != header.crc
|| !reader.get_ref().ended
|| reader.get_ref().decoder.total_in() != header.compressed
{
return Err(security("ZIP checksum or compressed byte count mismatch"));
}
entry
} else {
let mut reader = CrcReader::new(file);
let entry = stage(&mut reader)?;
if reader.crc().sum() != header.crc {
return Err(security("ZIP checksum mismatch"));
}
entry
};
if !header.directory {
source_total += entry.size;
budget.expanded += entry.size;
}
entries.push(entry);
}
Ok(entries)
}
fn c_string(bytes: &[u8]) -> Result<&[u8]> {
let end = bytes
.iter()
.position(|byte| *byte == 0)
.unwrap_or(bytes.len());
if bytes[end..].iter().any(|byte| *byte != 0) {
return Err(security("hidden bytes after TAR name terminator"));
}
Ok(&bytes[..end])
}
fn read_tar(
file: &mut File,
length: u64,
work: &Path,
budget: &mut Budget<'_>,
) -> Result<Vec<Entry>> {
if length < 1024 || !length.is_multiple_of(512) {
return Err(security(
"TAR must contain complete blocks and an end marker",
));
}
file.rewind().map_err(security)?;
let mut position = 0u64;
let mut entries = Vec::new();
let mut ended = false;
while position < length {
budget.check_time()?;
let mut block = [0u8; 512];
read_exact(file, &mut block)?;
position += 512;
if block.iter().all(|byte| *byte == 0) {
if !ended {
if position == length {
return Err(security("TAR requires two zero end blocks"));
}
read_exact(file, &mut block)?;
position += 512;
if block.iter().any(|byte| *byte != 0) {
return Err(security("invalid TAR end marker"));
}
ended = true;
}
continue;
}
if ended {
return Err(security("hidden entries or nonzero bytes after TAR end"));
}
let mut header = tar::Header::new_old();
header.as_mut_bytes().copy_from_slice(&block);
let checksum: u32 = block
.iter()
.enumerate()
.map(|(index, byte)| {
if (148..156).contains(&index) {
32
} else {
u32::from(*byte)
}
})
.sum();
if header.cksum().map_err(security)? != checksum {
return Err(security("TAR header checksum mismatch"));
}
header.uid().map_err(security)?;
header.gid().map_err(security)?;
header.mtime().map_err(security)?;
for field in [
&block[100..108],
&block[108..116],
&block[116..124],
&block[124..136],
&block[136..148],
&block[148..156],
] {
if field[0] & 0x80 == 0
&& field
.iter()
.position(|byte| *byte == 0)
.is_some_and(|end| field[end..].iter().any(|byte| !matches!(byte, 0 | b' ')))
{
return Err(security("hidden bytes in TAR numeric field"));
}
}
let directory = block[156] == b'5';
if !matches!(block[156], 0 | b'0' | b'5') {
return Err(security(
"TAR links, sparse files, devices, and extension records are unsupported",
));
}
let mode = header.mode().map_err(security)? & 0o170000;
if !matches!(mode, 0 | 0o100000 | 0o040000)
|| (directory && mode == 0o100000)
|| (!directory && mode == 0o040000)
|| block[157..257].iter().any(|byte| *byte != 0)
{
return Err(security("TAR entry has link or special-file metadata"));
}
let mut raw_name = Vec::new();
if (header.as_ustar().is_some() || header.as_gnu().is_some())
&& block[329..345]
.iter()
.any(|byte| !matches!(byte, 0 | b'0' | b' '))
{
return Err(security("TAR regular entry contains device numbers"));
}
if header.as_ustar().is_some() {
let prefix = c_string(&block[345..500])?;
if !prefix.is_empty() {
raw_name.extend_from_slice(prefix);
raw_name.push(b'/');
}
if block[500..].iter().any(|byte| *byte != 0) {
return Err(security("unsupported TAR header extension"));
}
} else if header.as_gnu().is_some() {
if block[345..].iter().any(|byte| *byte != 0) {
return Err(security("GNU TAR sparse or extension metadata"));
}
} else if block[257..].iter().any(|byte| *byte != 0) {
return Err(security("unrecognized TAR header format"));
}
raw_name.extend_from_slice(c_string(&block[..100])?);
let name = budget.entry(&raw_name, directory)?;
let size = header.size().map_err(security)?;
if directory && size != 0 {
return Err(security("TAR directory contains a payload"));
}
let padded = size
.checked_add(511)
.map(|size| size / 512 * 512)
.ok_or_else(|| Error::Limit("TAR entry size overflow".into()))?;
if padded > length.saturating_sub(position) {
return Err(security("TAR entry exceeds the available bytes"));
}
if directory {
entries.push(Entry {
name,
path: None,
size: 0,
});
} else {
entries.push(stage_entry(
&mut (&mut *file).take(size),
name,
size,
budget.limits.max_output_bytes,
work,
budget,
)?);
}
let padding = (padded - size) as usize;
read_exact(file, &mut block[..padding])?;
if block[..padding].iter().any(|byte| *byte != 0) {
return Err(security("nonzero TAR entry padding"));
}
position += padded;
}
if !ended {
return Err(security("missing TAR end marker"));
}
Ok(entries)
}
fn gzip_header(file: &mut File) -> Result<String> {
let mut header = [0u8; 10];
read_exact(file, &mut header)?;
if header[..3] != [0x1f, 0x8b, 8] || header[3] & 0xe4 != 0 {
return Err(security("invalid gzip header or unsupported extra fields"));
}
let mut name = "data.bin".to_owned();
for flag in [8, 16] {
if header[3] & flag != 0 {
let mut bytes = Vec::new();
loop {
let mut byte = [0u8; 1];
read_exact(file, &mut byte)?;
if byte[0] == 0 {
break;
}
if bytes.len() >= MAX_NAME {
return Err(Error::Limit("gzip header string length".into()));
}
bytes.push(byte[0]);
}
if flag == 8 {
name = valid_name(&bytes, false)?;
} else if bytes
.iter()
.any(|byte| !byte.is_ascii() || byte.is_ascii_control())
{
return Err(security("invalid gzip comment"));
}
}
}
if header[3] & 2 != 0 {
let mut crc = [0u8; 2];
read_exact(file, &mut crc)?;
}
file.rewind().map_err(security)?;
Ok(name)
}
fn read_gzip(
file: &mut File,
source_cap: u64,
work: &Path,
budget: &Budget<'_>,
) -> Result<(NamedTempFile, String, u64)> {
let name = gzip_header(file)?;
let mut decoder = GzDecoder::new(BufReader::new(file));
let mut expanded = NamedTempFile::new_in(work)?;
let size = copy_bounded(
&mut decoder,
expanded.as_file_mut(),
source_cap.min(budget.remaining()),
None,
budget,
)?;
if !decoder
.into_inner()
.fill_buf()
.map_err(security)?
.is_empty()
{
return Err(security("concatenated gzip members or trailing bytes"));
}
if expanded.as_file().metadata()?.len() != size {
return Err(security("gzip output length changed"));
}
expanded.as_file_mut().rewind()?;
Ok((expanded, name, size))
}
struct TimedReader<R> {
inner: R,
started: Instant,
timeout: std::time::Duration,
count: u64,
}
impl<R: Read> Read for TimedReader<R> {
fn read(&mut self, buffer: &mut [u8]) -> io::Result<usize> {
if self.started.elapsed() >= self.timeout {
return Err(io::Error::new(io::ErrorKind::TimedOut, "archive deadline"));
}
let read = self.inner.read(buffer)?;
self.count += read as u64;
Ok(read)
}
}
fn write_tar<W: Write>(writer: W, entries: &[Entry], budget: &Budget<'_>) -> Result<W> {
let mut archive = tar::Builder::new(writer);
for entry in entries {
budget.check_time()?;
let mut header = tar::Header::new_ustar();
header.set_path(&entry.name).map_err(security)?;
header.set_size(entry.size);
header.set_mode(if entry.path.is_some() { 0o600 } else { 0o700 });
header.set_uid(0);
header.set_gid(0);
header.set_mtime(0);
header.set_entry_type(if entry.path.is_some() {
tar::EntryType::Regular
} else {
tar::EntryType::Directory
});
header.set_cksum();
if let Some(path) = &entry.path {
let (file, size) = regular_input(path)?;
if size != entry.size {
return Err(security("staged input length changed"));
}
let mut reader = TimedReader {
inner: file.take(size),
started: budget.started,
timeout: budget.limits.timeout,
count: 0,
};
archive.append(&header, &mut reader).map_err(output_error)?;
if reader.count != size {
return Err(security("staged TAR input was truncated"));
}
} else {
archive.append(&header, io::empty()).map_err(output_error)?;
}
}
archive.into_inner().map_err(output_error)
}
fn write_archive(entries: &[Entry], format: &str, path: &Path, budget: &Budget<'_>) -> Result<()> {
let file = private_file(path)?;
let writer = BoundedWriter::new(file, budget);
let mut writer = match format {
"zip" => {
let mut archive = ZipWriter::new(writer);
let options = SimpleFileOptions::default()
.compression_method(CompressionMethod::Deflated)
.compression_level(Some(6))
.unix_permissions(0o600);
for entry in entries {
budget.check_time()?;
if let Some(path) = &entry.path {
archive
.start_file(&entry.name, options)
.map_err(zip_error)?;
let (mut input, size) = regular_input(path)?;
if size != entry.size {
return Err(security("staged input length changed"));
}
copy_bounded(&mut input, &mut archive, size, Some(size), budget)?;
} else {
archive
.add_directory(&entry.name, options)
.map_err(zip_error)?;
}
}
archive.finish().map_err(zip_error)?
}
"tar" => write_tar(writer, entries, budget)?,
"tar.gz" | "tgz" => write_tar(
GzEncoder::new(writer, Compression::default()),
entries,
budget,
)?
.finish()
.map_err(output_error)?,
"gz" => {
let mut files = entries.iter().filter(|entry| entry.path.is_some());
let entry = files
.next()
.ok_or_else(|| Error::Invalid("gzip requires one file".into()))?;
if files.next().is_some() || entries.iter().any(|entry| entry.path.is_none()) {
return Err(Error::Invalid(
"gzip cannot package multiple files; use tar.gz".into(),
));
}
let path = entry
.path
.as_ref()
.ok_or_else(|| Error::Invalid("gzip requires one regular file".into()))?;
let (mut input, size) = regular_input(path)?;
if size != entry.size {
return Err(security("staged input length changed"));
}
let mut gzip = GzEncoder::new(writer, Compression::default());
copy_bounded(&mut input, &mut gzip, size, Some(size), budget)?;
gzip.finish().map_err(output_error)?
}
_ => return Err(Error::Invalid("unsupported archive output".into())),
};
writer.flush().map_err(output_error)?;
if writer.inner.metadata()?.len() > budget.limits.max_output_bytes {
return Err(Error::Limit("final archive output bytes".into()));
}
budget.check_time()
}
fn profile_archive(path: &Path, id: &str, limits: &Limits) -> Result<ContentProfile> {
let file = File::open(path)?;
let mut profile = ContentProfile::new(id);
match id {
"zip" => {
let mut archive = ZipArchive::new(BufReader::new(file)).map_err(zip_error)?;
for index in 0..archive.len() {
let entry = archive.by_index(index).map_err(zip_error)?;
let kind = if entry.is_dir() {
MediaKind::Archive
} else {
MediaKind::Binary
};
profile.add_stream(Stream::new(index, ContentKind::new(kind)));
}
}
"tar" => {
let mut archive = tar::Archive::new(BufReader::new(file));
let entries = archive.entries().map_err(output_error)?;
for (index, entry) in entries.enumerate() {
let entry = entry.map_err(output_error)?;
let kind = if entry.header().entry_type().is_dir() {
MediaKind::Archive
} else {
MediaKind::Binary
};
profile.add_stream(Stream::new(index, ContentKind::new(kind)));
}
}
"gz" | "tar.gz" | "tgz" => {
let mut reader = GzDecoder::new(BufReader::new(&file));
let mut block = [0u8; 512];
let mut filled = 0usize;
loop {
match reader.read(&mut block[filled..]) {
Ok(0) => break,
Ok(n) => {
filled += n;
if filled == block.len() {
break;
}
}
Err(err) if err.kind() == io::ErrorKind::UnexpectedEof => break,
Err(err) => return Err(Error::Io(err)),
}
if filled as u64 > limits.max_input_bytes {
return Err(Error::Limit("gzip probe block".into()));
}
}
let kind = match id {
"gz" => MediaKind::Binary,
_ if filled >= 262 && &block[257..262] == b"ustar" => MediaKind::Archive,
_ => MediaKind::Binary,
};
profile.add_stream(Stream::new(0, ContentKind::new(kind)));
}
_ => unreachable!("formats checked by the caller"),
}
profile.deduplicate();
Ok(profile)
}
#[cfg(test)]
#[allow(clippy::unwrap_used, clippy::expect_used)]
mod tests {
use super::*;
use std::io::Cursor;
fn zip_bytes(entries: &[(&str, &[u8])]) -> Vec<u8> {
let mut writer = ZipWriter::new(Cursor::new(Vec::new()));
let options = SimpleFileOptions::default().compression_method(CompressionMethod::Deflated);
for (name, data) in entries {
writer.start_file(*name, options).unwrap();
writer.write_all(data).unwrap();
}
writer.finish().unwrap().into_inner()
}
fn gzip_bytes(data: &[u8]) -> Vec<u8> {
let mut gzip = GzEncoder::new(Vec::new(), Compression::default());
gzip.write_all(data).unwrap();
gzip.finish().unwrap()
}
fn request(root: &Path, bytes: &[u8], from: &str, to: &str, operation: &str) -> StepRequest {
let path = root.join(format!("input.{from}"));
fs::write(&path, bytes).unwrap();
let output_dir = root.join("output");
fs::create_dir_all(&output_dir).unwrap();
StepRequest::new(
vec![Artifact::new(path, from.to_owned())],
output_dir,
from.to_owned(),
to.to_owned(),
operation.to_owned(),
Limits::default(),
)
}
fn rejected(bytes: &[u8], from: &str, limits: Limits) -> Error {
let root = tempfile::tempdir().unwrap();
let mut request = request(root.path(), bytes, from, "bin", "extract");
request.limits = limits;
let error = execute(&request, Instant::now()).unwrap_err();
assert!(
matches!(error, Error::Security(_) | Error::Limit(_)),
"{error}"
);
assert_eq!(fs::read_dir(&request.output_dir).unwrap().count(), 0);
error
}
fn central_offsets(bytes: &[u8]) -> Vec<usize> {
bytes
.windows(4)
.enumerate()
.filter_map(|(index, bytes)| (bytes == b"PK\x01\x02").then_some(index))
.collect()
}
#[test]
fn registration_uses_wildcard_package_and_extract_edges() {
let adapter = ArchivesAdapter;
assert_eq!(adapter.name(), "archives");
assert_eq!(adapter.formats().len(), 5);
let edges = adapter.conversions();
assert_eq!(edges.iter().filter(|edge| edge.from == "*").count(), 5);
assert!(
edges
.iter()
.filter(|edge| edge.from == "*")
.all(|edge| edge.kind == ConversionKind::Package)
);
assert_eq!(
edges
.iter()
.filter(|edge| edge.to == "bin" && edge.kind == ConversionKind::Extract)
.count(),
5
);
}
#[test]
fn zip_traversal_and_portability_rejections() {
for name in [
"../escape",
"a/../../escape",
"/absolute",
"C:/absolute",
"C:relative",
"\\\\server\\share",
"a\\..\\escape",
"a\nname",
"a//b",
"NUL.txt",
"trailing.",
"a/./b",
] {
let bytes = zip_bytes(&[("safe.txt", b"safe"), (name, b"bad")]);
assert!(
matches!(
rejected(&bytes, "zip", Limits::default()),
Error::Security(_)
),
"{name}"
);
}
}
#[test]
fn zip_rejects_duplicate_and_case_folded_names_before_library_dedup() {
let mut bytes = zip_bytes(&[("a.txt", b"one"), ("b.txt", b"two")]);
let positions: Vec<_> = bytes
.windows(5)
.enumerate()
.filter_map(|(index, bytes)| (bytes == b"b.txt").then_some(index))
.collect();
for position in positions {
bytes[position] = b'a';
}
assert!(matches!(
rejected(&bytes, "zip", Limits::default()),
Error::Security(_)
));
rejected(
&zip_bytes(&[("File.txt", b"a"), ("file.txt", b"b")]),
"zip",
Limits::default(),
);
}
#[test]
fn zip_rejects_prefix_conflicts_in_both_orders() {
for names in [["node", "node/file"], ["node/file", "NODE"]] {
rejected(
&zip_bytes(&[(names[0], b"a"), (names[1], b"b")]),
"zip",
Limits::default(),
);
}
}
#[test]
fn zip_rejects_symlink_device_fifo_and_encryption_headers() {
for mode in [0o120777u32, 0o020600, 0o060600, 0o010600, 0o140600] {
let mut bytes = zip_bytes(&[("link", b"target")]);
let central = central_offsets(&bytes)[0];
bytes[central + 5] = 3;
bytes[central + 38..central + 42].copy_from_slice(&(mode << 16).to_le_bytes());
assert!(matches!(
rejected(&bytes, "zip", Limits::default()),
Error::Security(_)
));
}
for central_flag in [true, false] {
let mut bytes = zip_bytes(&[("secret", b"secret")]);
let central = central_offsets(&bytes)[0];
let offset = if central_flag { central + 8 } else { 6 };
bytes[offset] |= 1;
assert!(matches!(
rejected(&bytes, "zip", Limits::default()),
Error::Security(_)
));
}
}
#[test]
fn zip_ratio_and_forged_actual_sizes_are_bounded() {
let bytes = zip_bytes(&[("bomb", &vec![0u8; 256 * 1024])]);
assert!(matches!(
rejected(&bytes, "zip", Limits::default()),
Error::Limit(_)
));
let mut forged = bytes;
let central = central_offsets(&forged)[0];
forged[22..26].copy_from_slice(&1u32.to_le_bytes());
forged[central + 24..central + 28].copy_from_slice(&1u32.to_le_bytes());
let error = rejected(&forged, "zip", Limits::default());
assert!(
matches!(error, Error::Security(_) | Error::Limit(_)),
"forged sizes must be rejected before producing output, got {error:?}"
);
}
#[test]
fn zip_accepts_stored_entries_and_signed_data_descriptors() {
let root = tempfile::tempdir().unwrap();
let mut writer = ZipWriter::new(Cursor::new(Vec::new()));
writer
.start_file(
"stored",
SimpleFileOptions::default().compression_method(CompressionMethod::Stored),
)
.unwrap();
writer.write_all(b"stored content").unwrap();
let bytes = writer.finish().unwrap().into_inner();
let step = request(root.path(), &bytes, "zip", "bin", "extract");
let artifacts = execute(&step, Instant::now()).unwrap();
assert_eq!(fs::read(&artifacts[0].path).unwrap(), b"stored content");
let root = tempfile::tempdir().unwrap();
let mut bytes = zip_bytes(&[("streamed", b"streamed content")]);
let central = central_offsets(&bytes)[0];
let mut descriptor = b"PK\x07\x08".to_vec();
descriptor.extend_from_slice(&bytes[central + 16..central + 28]);
bytes.splice(central..central, descriptor);
let central = central + 16;
bytes[6] |= 8;
bytes[central + 8] |= 8;
bytes[14..26].fill(0);
let footer = bytes.len() - 22;
bytes[footer + 16..footer + 20].copy_from_slice(&(central as u32).to_le_bytes());
let step = request(root.path(), &bytes, "zip", "bin", "extract");
let artifacts = execute(&step, Instant::now()).unwrap();
assert_eq!(fs::read(&artifacts[0].path).unwrap(), b"streamed content");
}
#[test]
fn zip_rejects_compressed_padding_that_dilutes_ratio() {
let mut bytes = zip_bytes(&[("bomb", &vec![0u8; 64 * 1024])]);
let central = central_offsets(&bytes)[0];
let compressed = little32(&bytes, central + 20);
let extra = 4096usize;
bytes.splice(central..central, vec![0u8; extra]);
let central = central + extra;
bytes[18..22].copy_from_slice(&(compressed + extra as u32).to_le_bytes());
bytes[central + 20..central + 24]
.copy_from_slice(&(compressed + extra as u32).to_le_bytes());
let footer = bytes.len() - 22;
bytes[footer + 16..footer + 20].copy_from_slice(&(central as u32).to_le_bytes());
assert!(matches!(
rejected(&bytes, "zip", Limits::default()),
Error::Security(_)
));
}
#[test]
fn zip_crc_failure_after_valid_entry_cleans_everything() {
let mut bytes = zip_bytes(&[("good", b"good"), ("bad", b"bad")]);
let central = central_offsets(&bytes)[1];
let local = little32(&bytes, central + 42) as usize;
bytes[central + 16..central + 20].fill(0);
bytes[local + 14..local + 18].fill(0);
rejected(&bytes, "zip", Limits::default());
}
#[tokio::test]
async fn proper_zip_to_tar_preserves_names_and_payloads() {
let root = tempfile::tempdir().unwrap();
let bytes = zip_bytes(&[("nested/a.txt", b"alpha"), ("other/a.txt", b"beta")]);
let request = request(root.path(), &bytes, "zip", "tar", "repack");
let artifacts = ArchivesAdapter.convert(&request).await.unwrap();
assert_eq!(artifacts.len(), 1);
let mut tar = tar::Archive::new(File::open(&artifacts[0].path).unwrap());
let contents: Vec<_> = tar
.entries()
.unwrap()
.map(|entry| {
let mut entry = entry.unwrap();
let name = entry.path().unwrap().to_string_lossy().into_owned();
let mut bytes = Vec::new();
entry.read_to_end(&mut bytes).unwrap();
(name, bytes)
})
.collect();
assert_eq!(
contents,
vec![
("nested/a.txt".into(), b"alpha".to_vec()),
("other/a.txt".into(), b"beta".to_vec())
]
);
}
#[test]
fn wildcard_packaging_round_trips_all_formats() {
for format in FORMATS {
let root = tempfile::tempdir().unwrap();
let mut request = request(
root.path(),
b"portable payload",
"unknown",
format,
"package",
);
request.from = "*".into();
let packaged = execute(&request, Instant::now()).unwrap();
request.inputs = packaged;
request.from = format.into();
request.to = "bin".into();
request.operation = "extract".into();
let extracted = execute(&request, Instant::now()).unwrap();
assert_eq!(extracted.len(), 1, "{format}");
assert_eq!(extracted[0].format, "bin");
assert_eq!(fs::read(&extracted[0].path).unwrap(), b"portable payload");
}
}
#[test]
fn nested_archives_remain_opaque_binary_files() {
let root = tempfile::tempdir().unwrap();
let nested = zip_bytes(&[("../malicious", b"not expanded")]);
let outer = zip_bytes(&[("nested.zip", &nested), ("readme.txt", b"text")]);
let request = request(root.path(), &outer, "zip", "bin", "extract");
let artifacts = execute(&request, Instant::now()).unwrap();
assert_eq!(artifacts.len(), 2);
assert!(artifacts.iter().all(|artifact| artifact.format == "bin"));
assert_eq!(fs::read(&artifacts[0].path).unwrap(), nested);
assert!(
artifacts[0]
.path
.file_name()
.unwrap()
.to_string_lossy()
.ends_with("nested.zip")
);
}
#[test]
fn gzip_bounded_decompression_and_corrupt_trailer() {
let bytes = gzip_bytes(&vec![0u8; 256 * 1024]);
assert!(matches!(
rejected(&bytes, "gz", Limits::default()),
Error::Limit(_)
));
let limits = Limits {
max_expansion_ratio: u64::MAX,
max_output_bytes: 32,
..Limits::default()
};
assert!(matches!(rejected(&bytes, "gz", limits), Error::Limit(_)));
let mut bytes = gzip_bytes(b"valid contents");
let last = bytes.len() - 1;
bytes[last] ^= 1;
assert!(matches!(
rejected(&bytes, "gz", Limits::default()),
Error::Security(_)
));
}
#[test]
fn gzip_rejects_traversal_names_and_concatenated_members() {
let mut gzip = flate2::GzBuilder::new()
.filename("../escape")
.write(Vec::new(), Compression::default());
gzip.write_all(b"payload").unwrap();
rejected(&gzip.finish().unwrap(), "gz", Limits::default());
let mut bytes = gzip_bytes(b"one");
bytes.extend(gzip_bytes(b"two"));
rejected(&bytes, "gz", Limits::default());
}
#[test]
fn gzip_rejects_multiple_files_instead_of_concatenating() {
let root = tempfile::tempdir().unwrap();
let bytes = zip_bytes(&[("one", b"one"), ("two", b"two")]);
let request = request(root.path(), &bytes, "zip", "gz", "repack");
assert!(matches!(
execute(&request, Instant::now()),
Err(Error::Invalid(_))
));
assert_eq!(fs::read_dir(&request.output_dir).unwrap().count(), 0);
}
fn tar_bytes(entries: &[(&str, u8, &[u8])]) -> Vec<u8> {
let mut archive = tar::Builder::new(Vec::new());
for (name, kind, data) in entries {
let mut header = tar::Header::new_ustar();
header.as_mut_bytes()[..name.len()].copy_from_slice(name.as_bytes());
header.set_mode(0o600);
header.set_uid(0);
header.set_gid(0);
header.set_mtime(0);
header.set_size(data.len() as u64);
header.set_entry_type(tar::EntryType::new(*kind));
header.set_cksum();
archive.append(&header, *data).unwrap();
}
archive.into_inner().unwrap()
}
#[test]
fn tar_rejects_links_devices_fifo_sparse_and_extensions() {
for kind in *b"123467SxgLK" {
let bytes = tar_bytes(&[("safe", b'0', b"safe"), ("special", kind, b"")]);
assert!(matches!(
rejected(&bytes, "tar", Limits::default()),
Error::Security(_)
));
}
for name in ["../escape", "C:\\escape", "a\\..\\b"] {
rejected(
&tar_bytes(&[(name, b'0', b"payload")]),
"tar",
Limits::default(),
);
}
rejected(
&tar_bytes(&[("a", b'0', b"one"), ("A", b'0', b"two")]),
"tar",
Limits::default(),
);
}
#[test]
fn directory_entries_count_toward_limits() {
let bytes = tar_bytes(&[("a/", b'5', b""), ("b/", b'5', b""), ("c", b'0', b"c")]);
let limits = Limits {
max_archive_entries: 2,
..Limits::default()
};
assert!(matches!(rejected(&bytes, "tar", limits), Error::Limit(_)));
let mut writer = ZipWriter::new(Cursor::new(Vec::new()));
writer
.add_directory("a/", SimpleFileOptions::default())
.unwrap();
writer
.add_directory("b/", SimpleFileOptions::default())
.unwrap();
let bytes = writer.finish().unwrap().into_inner();
let limits = Limits {
max_archive_entries: 1,
..Limits::default()
};
assert!(matches!(rejected(&bytes, "zip", limits), Error::Limit(_)));
}
#[test]
fn tar_gzip_and_tgz_round_trip_multiple_entries_and_directories() {
for format in ["tar.gz", "tgz", "zip"] {
let root = tempfile::tempdir().unwrap();
let bytes = tar_bytes(&[
("nested/", b'5', b""),
("nested/a.txt", b'0', b"alpha"),
("b.txt", b'0', b"beta"),
]);
let mut step = request(root.path(), &bytes, "tar", format, "repack");
step.inputs = execute(&step, Instant::now()).unwrap();
step.from = format.into();
step.to = "bin".into();
step.operation = "extract".into();
let artifacts = execute(&step, Instant::now()).unwrap();
assert_eq!(artifacts.len(), 2);
assert_eq!(fs::read(&artifacts[0].path).unwrap(), b"alpha");
assert_eq!(fs::read(&artifacts[1].path).unwrap(), b"beta");
}
}
#[test]
fn tar_rejects_hidden_entries_after_end_and_truncation() {
let mut bytes = tar_bytes(&[("one", b'0', b"one")]);
bytes.extend(tar_bytes(&[("hidden", b'0', b"hidden")]));
rejected(&bytes, "tar", Limits::default());
let bytes = tar_bytes(&[("one", b'0', b"one")]);
rejected(&bytes[..bytes.len() - 512], "tar", Limits::default());
}
fn tamper_footer(bytes: &mut [u8], at: usize, value: u16) {
let footer = bytes.len() - 22;
bytes[footer + at..footer + at + 2].copy_from_slice(&value.to_le_bytes());
}
#[test]
fn zip_rejects_footer_inconsistencies_and_oversized_extras() {
type Tamper = fn(&mut [u8]);
let cases: [(&str, Tamper); 6] = [
("first disk differs", |b| tamper_footer(b, 4, 1)),
("entry disk differs", |b| tamper_footer(b, 6, 1)),
("total count disagrees", |b| tamper_footer(b, 8, 0)),
("sentinel count", |b| tamper_footer(b, 10, u16::MAX)),
("ZIP64 directory offset", |b| {
let footer = b.len() - 22;
b[footer + 12..footer + 16].copy_from_slice(&u32::MAX.to_le_bytes());
}),
("directory end mismatch", |b| {
let footer = b.len() - 22;
b[footer + 12..footer + 16].copy_from_slice(&1u32.to_le_bytes());
}),
];
for (label, tamper) in cases {
let mut bytes = zip_bytes(&[("a.txt", b"alpha")]);
tamper(&mut bytes);
assert!(
matches!(
rejected(&bytes, "zip", Limits::default()),
Error::Security(_)
),
"{label}"
);
}
let mut bytes = zip_bytes(&[("a.txt", b"alpha")]);
let central = central_offsets(&bytes)[0];
let name_len = usize::from(little16(&bytes, central + 28));
bytes.splice(
central + 46 + name_len..central + 46 + name_len,
[0u8; 2000],
);
bytes[central + 30..central + 32].copy_from_slice(&2000u16.to_le_bytes());
let footer = bytes.len() - 22;
let size = little32(&bytes, footer + 12) + 2000;
bytes[footer + 12..footer + 16].copy_from_slice(&size.to_le_bytes());
assert!(matches!(
rejected(&bytes, "zip", Limits::default()),
Error::Limit(_)
));
let mut bytes = zip_bytes(&[("a.txt", b"alpha")]);
let central = central_offsets(&bytes)[0];
bytes[central + 28..central + 30].fill(0);
assert!(matches!(
rejected(&bytes, "zip", Limits::default()),
Error::Security(_)
));
}
#[test]
fn zip_rejects_mismatched_descriptors_and_bogus_central_signatures() {
let mut bytes = zip_bytes(&[("a.txt", b"alpha")]);
let central = central_offsets(&bytes)[0];
bytes[central..central + 2].copy_from_slice(b"XX");
assert!(matches!(
rejected(&bytes, "zip", Limits::default()),
Error::Security(_)
));
let mut bytes = zip_bytes(&[("a.txt", b"alpha")]);
let central = central_offsets(&bytes)[0];
let local = little32(&bytes, central + 42) as usize;
bytes[central + 16..central + 20].fill(9);
let _ = local;
assert!(matches!(
rejected(&bytes, "zip", Limits::default()),
Error::Security(_)
));
let mut bytes = zip_bytes(&[("streamed", b"streamed content")]);
let central = central_offsets(&bytes)[0];
let mut descriptor = b"PK\x07\x08".to_vec();
descriptor.extend_from_slice(&bytes[central + 16..central + 28]);
descriptor[4] ^= 0xff;
bytes.splice(central..central, descriptor);
let central = central + 16;
bytes[6] |= 8;
bytes[central + 8] |= 8;
bytes[14..26].fill(0);
let footer = bytes.len() - 22;
bytes[footer + 16..footer + 20].copy_from_slice(&(central as u32).to_le_bytes());
assert!(matches!(
rejected(&bytes, "zip", Limits::default()),
Error::Security(_)
));
}
fn raw_tar_block(name: &[u8], kind: u8, size: u64, mutate: impl FnOnce(&mut [u8])) -> Vec<u8> {
let mut block = vec![0u8; 512];
block[..name.len()].copy_from_slice(name);
let write_octal = |block: &mut [u8], at: usize, width: usize, value: u64| {
let digits = format!("{value:0width$o}");
block[at..at + digits.len()].copy_from_slice(digits.as_bytes());
block[at + digits.len()] = if width == 12 { b' ' } else { 0 };
};
write_octal(&mut block, 100, 8, 0o600);
write_octal(&mut block, 108, 8, 0);
write_octal(&mut block, 116, 8, 0);
write_octal(&mut block, 124, 12, size);
write_octal(&mut block, 136, 8, 0);
block[156] = kind;
block[257..263].copy_from_slice(b"ustar\0");
block[263..265].copy_from_slice(b"00");
mutate(&mut block);
let checksum: u32 = block
.iter()
.enumerate()
.map(|(index, byte)| {
if (148..156).contains(&index) {
32
} else {
u32::from(*byte)
}
})
.sum();
block[148..155].copy_from_slice(format!("{checksum:07o}").as_bytes());
block[155] = 0;
block
}
fn raw_tar(entries: &[Vec<u8>], trailing: usize) -> Vec<u8> {
let mut bytes = Vec::new();
for entry in entries {
bytes.extend_from_slice(entry);
let rem = (entry.len() - 512) % 512;
if rem != 0 {
bytes.resize(bytes.len() + (512 - rem), 0);
}
}
bytes.extend(std::iter::repeat_n(0u8, trailing));
bytes
}
#[test]
fn tar_rejects_forged_header_fields() {
let root = tempfile::tempdir().unwrap();
fn entry(
name: &[u8],
kind: u8,
size: u64,
payload: &[u8],
mutate: impl FnOnce(&mut [u8]),
) -> Vec<u8> {
let mut bytes = raw_tar_block(name, kind, size, mutate);
bytes.extend_from_slice(payload);
bytes.extend(std::iter::repeat_n(0u8, 512 - payload.len()));
bytes.extend(std::iter::repeat_n(0u8, 1024));
bytes
}
let valid = entry(b"note.txt", b'0', 3, b"abc", |_| {});
assert_eq!(
execute(
&request(root.path(), &valid, "tar", "bin", "extract"),
Instant::now(),
)
.unwrap()
.len(),
1
);
for (label, bytes) in [
(
"directory payload",
entry(b"dir/", b'5', 4, b"xxxx", |_| {}),
),
(
"device numbers",
entry(b"x.txt", b'0', 3, b"abc", |b| b[329] = b'1'),
),
(
"hidden bytes in size field",
entry(b"y.txt", b'0', 3, b"abc", |b| {
b[127] = 0;
b[128] = b'9';
}),
),
(
"linkname on regular entry",
entry(b"z.txt", b'0', 3, b"abc", |b| b[157] = b'l'),
),
(
"trailing header garbage",
entry(b"g.txt", b'0', 3, b"abc", |b| b[501] = b'x'),
),
(
"claimed size past the end",
entry(b"huge.txt", b'0', 4096, b"abc", |_| {}),
),
("nonzero entry padding", {
let mut bytes = entry(b"p.txt", b'0', 3, b"abc", |_| {});
bytes[600] = 1;
bytes
}),
("single end block", {
let mut bytes = entry(b"one.txt", b'0', 3, b"abc", |_| {});
bytes.truncate(bytes.len() - 512);
bytes
}),
("broken end marker", {
let mut bytes = entry(b"one.txt", b'0', 3, b"abc", |_| {});
let last = bytes.len() - 512;
bytes[last] = 1;
bytes
}),
] {
assert!(
matches!(
rejected(&bytes, "tar", Limits::default()),
Error::Security(_)
),
"{label}"
);
}
}
#[test]
fn tar_ustar_prefix_names_round_trip_and_gnu_streams_are_rejected() {
let prefix = format!("directory-{}", "x".repeat(90));
let mut block = raw_tar_block(b"file.txt", b'0', 3, |b| {
b[345..345 + prefix.len()].copy_from_slice(prefix.as_bytes());
});
block.extend_from_slice(b"abc");
let bytes = raw_tar(&[block], 1024);
let root = tempfile::tempdir().unwrap();
let artifacts = execute(
&request(root.path(), &bytes, "tar", "bin", "extract"),
Instant::now(),
)
.unwrap();
assert_eq!(artifacts.len(), 1);
assert_eq!(fs::read(&artifacts[0].path).unwrap(), b"abc");
let mut block = raw_tar_block(b"gnu.txt", b'0', 3, |_| {});
block[257..263].copy_from_slice(b"ustar ");
block[263..265].copy_from_slice(b" \0");
block[345] = b'!';
block.extend_from_slice(b"abc");
assert!(matches!(
rejected(&raw_tar(&[block], 1024), "tar", Limits::default()),
Error::Security(_)
));
}
#[test]
fn gzip_rejects_extra_flags_and_invalid_comment_bytes() {
let mut bytes = gzip_bytes(b"payload");
bytes[3] = 4;
assert!(matches!(
rejected(&bytes, "gz", Limits::default()),
Error::Security(_)
));
let mut gzip = flate2::GzBuilder::new()
.comment("bad\ncontrol")
.write(Vec::new(), Compression::default());
gzip.write_all(b"payload").unwrap();
rejected(&gzip.finish().unwrap(), "gz", Limits::default());
let mut gzip = flate2::GzBuilder::new()
.filename("r".repeat(MAX_NAME + 1))
.write(Vec::new(), Compression::default());
gzip.write_all(b"payload").unwrap();
assert!(matches!(
rejected(&gzip.finish().unwrap(), "gz", Limits::default()),
Error::Limit(_)
));
}
#[test]
fn archive_output_writes_are_bounded_for_every_format() {
for format in FORMATS {
let root = tempfile::tempdir().unwrap();
let mut request = request(root.path(), b"payload", "bin", format, "package");
request.limits.max_output_bytes = 8;
assert!(
matches!(execute(&request, Instant::now()), Err(Error::Limit(_))),
"{format}"
);
assert_eq!(fs::read_dir(&request.output_dir).unwrap().count(), 0);
}
}
}