use std::io::{Read, Write};
use crate::archive::{
ArchiveRead, ArchiveWrite, Codec, Container, CreateOpts, DecodeOpts, EncodeOpts, Entry,
EntryMeta, OpenOpts, Sink,
};
use crate::error::{Error, Result};
use crate::fidelity::{Fidelity, FidelityReport, Rung};
use crate::format::{CodecCaps, ContainerCaps, FormatId, FormatMeta, MagicRule};
use crate::ladder::Resolved;
use crate::source::Source;
pub use crate::conformance::{
assert_codec_conforms, assert_codec_conforms_with, compressible, incompressible,
};
pub use crate::container_conformance::{
ContainerFixture, ExpectedEntry, assert_container_conforms, assert_container_conforms_skipping,
assert_container_conforms_with, assert_container_conforms_with_skipping, open_forward_only,
};
pub use crate::honesty::{
Attestation, check_display_has_no_raw_controls, check_entries_carried, check_entry_count,
check_entry_size, check_error_is_classified, check_extraction_contained, check_fidelity_claim,
check_salvage_claim,
};
pub const CODEC_SLOTS: &[&str] = &[
"gzip", "zlib", "deflate", "bzip2", "brotli", "lz4", "snappy", "zstd", "xz", "lzma", "lzip",
"compress",
];
pub const CONTAINER_SLOTS: &[&str] = &[
"tar", "ar", "cpio", "zip",
"lha", "arj",
"arc", "zoo",
];
pub const SALVAGE_SLOTS: &[&str] = &["zip", "arc", "zoo", "lha", "arj", "tar", "cpio", "ar"];
pub const SALVAGE_FUZZ_MAX_ENTRY: u64 = 256 * 1024;
pub const COMPAT_NAMES: &[&str] = &[
"bzip2",
"bunzip2",
"bzcat",
"bzip2recover",
"gzip",
"gunzip",
"zcat",
"gzcat",
"xz",
"unxz",
"xzcat",
"lzma",
"unlzma",
"lzcat",
];
pub const COMPAT_FILES: &[&str] = &[
"a", "a.bz2", "a.tbz2", "a.out", "a.gz", "a.tgz", "a.z", "a.xz", "a.txz", "a.lzma", "a.tlz", "a.lz", ];
pub const COMPAT_TOKENS: &[&str] = &[
"-c",
"-d",
"-z",
"-f",
"-t",
"-k",
"-s",
"-q",
"-v",
"-V",
"-L",
"-h",
"-1",
"-2",
"-3",
"-4",
"-5",
"-6",
"-7",
"-8",
"-9",
"-x",
"-dc",
"-cd",
"-tv",
"-kf",
"-dfk",
"-qt",
"-vvv",
"-zc",
"-s1",
"-dq",
"--stdout",
"--decompress",
"--compress",
"--force",
"--test",
"--keep",
"--small",
"--quiet",
"--version",
"--license",
"--exponential",
"--repetitive-best",
"--repetitive-fast",
"--fast",
"--best",
"--verbose",
"--help",
"--bogus",
"--",
"-",
"a",
"a.bz2",
"a.tbz2",
"a.out",
"-l",
"-n",
"-N",
"-r",
"-S",
".x",
"--list",
"--name",
"--no-name",
"--recursive",
"--suffix",
"--rsyncable",
"--synchronous",
"-a",
"--ascii",
"a.gz",
"a.tgz",
"a.z",
"-T0",
"-T1",
"-e",
"-0",
"-F",
"xz",
"lzma",
"raw",
"lzip",
"--format",
"--check",
"crc32",
"sha256",
"none",
"--single-stream",
"--no-sparse",
"--robot",
"--memlimit-decompress=1MiB",
"--lzma2=dict=4KiB",
"--x86",
"--delta=dist=2",
"--block-size=1KiB",
"--files",
"--files0",
"--ignore-check",
"a.xz",
"a.txz",
"a.lzma",
"a.tlz",
"a.lz",
"--files=a",
"--files0=a",
];
pub const COMPAT_FUZZ_MAX_INPUT: usize = 64 * 1024;
pub const COMPAT_MAX_ARGS: usize = 8;
pub const COMPAT_STDIN_BIT: u8 = 1 << 4;
fn file_bit(i: usize) -> u16 {
1 << if i < 4 { i } else { i + 1 }
}
const FIRST_WIDE_FILE: usize = 7;
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct CompatCase<'a> {
pub name: &'static str,
pub flavour: u8,
pub args: Vec<&'static str>,
pub files: Vec<&'static str>,
pub stdin: bool,
pub payload: &'a [u8],
}
pub fn decode_compat_case(data: &[u8]) -> Option<CompatCase<'_>> {
let (&name, rest) = data.split_first()?;
let (&argc_byte, rest) = rest.split_first()?;
let argc = argc_byte as usize % (COMPAT_MAX_ARGS + 1);
let wide = argc_byte as usize > COMPAT_MAX_ARGS;
if rest.len() < argc + 1 {
return None;
}
let (tokens, rest) = rest.split_at(argc);
let (&low, rest) = rest.split_first()?;
let (mask, payload) = if wide {
let (&high, rest) = rest.split_first()?;
(u16::from_le_bytes([low, high]), rest)
} else {
(u16::from(low), rest)
};
Some(CompatCase {
name: COMPAT_NAMES[name as usize % COMPAT_NAMES.len()],
flavour: ((name as usize / COMPAT_NAMES.len()) & 1) as u8,
args: tokens
.iter()
.map(|&t| COMPAT_TOKENS[t as usize % COMPAT_TOKENS.len()])
.collect(),
files: COMPAT_FILES
.iter()
.enumerate()
.filter(|&(i, _)| mask & file_bit(i) != 0)
.map(|(_, &f)| f)
.collect(),
stdin: mask & u16::from(COMPAT_STDIN_BIT) != 0,
payload,
})
}
pub fn encode_compat_case(
name: &str,
flavour: u8,
args: &[&str],
files: &[&str],
stdin: bool,
payload: &[u8],
) -> Vec<u8> {
let index = |table: &[&str], s: &str| {
table
.iter()
.position(|t| *t == s)
.unwrap_or_else(|| panic!("{s:?} is not in its compat table"))
};
assert!(args.len() <= COMPAT_MAX_ARGS, "too many compat arguments");
assert!(flavour <= 1, "a gzip flavour is 0 (GNU) or 1 (BSD)");
let selector = index(COMPAT_NAMES, name) + usize::from(flavour) * COMPAT_NAMES.len();
let selector = u8::try_from(selector).expect("a compat name selector fits a byte");
let files: Vec<usize> = files.iter().map(|f| index(COMPAT_FILES, f)).collect();
let wide = files.iter().any(|&i| i >= FIRST_WIDE_FILE);
let argc = args.len() + if wide { COMPAT_MAX_ARGS + 1 } else { 0 };
let mut v = vec![selector, argc as u8];
v.extend(args.iter().map(|a| index(COMPAT_TOKENS, a) as u8));
let mut mask = if stdin {
u16::from(COMPAT_STDIN_BIT)
} else {
0
};
for i in files {
mask |= file_bit(i);
}
let [low, high] = mask.to_le_bytes();
v.push(low);
if wide {
v.push(high);
}
v.extend_from_slice(payload);
v
}
pub const MOCK_CODEC: FormatId = FormatId::new("mock-codec");
pub const MOCK_CONTAINER: FormatId = FormatId::new("mock-container");
pub struct MockCodec;
struct XorReader(Box<dyn Read + Send>);
impl Read for XorReader {
fn read(&mut self, buf: &mut [u8]) -> std::io::Result<usize> {
let n = self.0.read(buf)?;
for b in &mut buf[..n] {
*b ^= 0xFF;
}
Ok(n)
}
}
struct XorSink(Box<dyn Write + Send>);
impl Write for XorSink {
fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {
let flipped: Vec<u8> = buf.iter().map(|b| b ^ 0xFF).collect();
self.0.write_all(&flipped)?;
Ok(buf.len())
}
fn flush(&mut self) -> std::io::Result<()> {
self.0.flush()
}
}
impl Sink for XorSink {
fn finish(mut self: Box<Self>) -> Result<()> {
self.0.flush()?;
Ok(())
}
}
impl Codec for MockCodec {
fn id(&self) -> FormatId {
MOCK_CODEC
}
fn caps(&self) -> CodecCaps {
CodecCaps {
encode: true,
decode: true,
..Default::default()
}
}
fn decoder(&self, src: Box<dyn Source>, _o: &DecodeOpts) -> Result<Box<dyn Source>> {
Ok(Box::new(crate::source::StreamOnly::new(XorReader(
Box::new(src),
))))
}
fn encoder(&self, dst: Box<dyn Write + Send>, _o: &EncodeOpts) -> Result<Box<dyn Sink>> {
Ok(Box::new(XorSink(dst)))
}
}
impl MockCodec {
pub fn workers_for_opts(&self, o: &DecodeOpts) -> usize {
match (&o.governor, o.threads) {
(Some(g), want) => g.acquire_many(want.unwrap_or(1), 0).workers(),
(None, _) => 1,
}
}
}
pub struct MockContainer;
pub fn mock_archive_bytes(entries: &[(&str, &[u8])]) -> Vec<u8> {
let mut out = Vec::new();
let mut offsets = Vec::new();
for (name, data) in entries {
offsets.push(out.len() as u64);
out.extend_from_slice(b"ME");
out.extend_from_slice(&(name.len() as u16).to_le_bytes());
out.extend_from_slice(name.as_bytes());
out.extend_from_slice(&(data.len() as u64).to_le_bytes());
out.extend_from_slice(data);
}
out.extend_from_slice(b"MI");
out.extend_from_slice(&(entries.len() as u32).to_le_bytes());
for o in &offsets {
out.extend_from_slice(&o.to_le_bytes());
}
out.extend_from_slice(b"MEND");
out
}
pub struct MockArchiveRead {
buf: Vec<u8>,
pos: usize,
report: FidelityReport,
index: Option<Vec<u64>>,
}
impl MockContainer {
fn parse_index(buf: &[u8]) -> Option<Vec<u64>> {
if !buf.ends_with(b"MEND") {
return None;
}
let end = buf.len() - 4;
for count in 0..=((end.saturating_sub(6)) / 8) {
let start = end.checked_sub(8 * count + 4 + 2)?;
if &buf[start..start + 2] == b"MI" {
let declared =
u32::from_le_bytes(buf[start + 2..start + 6].try_into().ok()?) as usize;
if declared != count {
continue;
}
let mut offsets = Vec::with_capacity(count);
for i in 0..count {
let o = start + 6 + i * 8;
offsets.push(u64::from_le_bytes(buf[o..o + 8].try_into().ok()?));
}
return Some(offsets);
}
}
None
}
fn find_first_record(buf: &[u8]) -> Option<usize> {
buf.windows(2).position(|w| w == b"ME")
}
}
impl Container for MockContainer {
fn id(&self) -> FormatId {
MOCK_CONTAINER
}
fn caps(&self) -> ContainerCaps {
ContainerCaps {
read: true,
write: true,
forward_parse: true,
trailing_index: true,
..Default::default()
}
}
fn open(&self, resolved: Resolved, _o: &OpenOpts) -> Result<Box<dyn ArchiveRead>> {
let Resolved {
mut source,
rung,
report,
..
} = resolved;
let seekable = source.caps().seekable;
let mut buf = Vec::new();
source.read_to_end(&mut buf)?;
let pos = if rung == Rung::Degraded {
Self::find_first_record(&buf).unwrap_or(buf.len())
} else {
0
};
let index = if seekable {
Self::parse_index(&buf)
} else {
None
};
let mut report = report;
if seekable && index.is_none() {
report.warn(Fidelity::TruncatedStream {
at: buf.len() as u64,
});
}
Ok(Box::new(MockArchiveRead {
buf,
pos,
report,
index,
}))
}
fn create(&self, dst: Box<dyn Sink>, _o: &CreateOpts) -> Result<Box<dyn ArchiveWrite>> {
Ok(Box::new(MockArchiveWrite {
dst,
offsets: Vec::new(),
written: 0,
}))
}
}
impl MockArchiveRead {
fn read_record_at(&self, at: usize) -> Result<Option<(EntryMeta, usize, usize)>> {
let b = &self.buf;
if at + 2 > b.len() || &b[at..at + 2] != b"ME" {
return Ok(None);
}
let name_len = u16::from_le_bytes(
b[at + 2..at + 4]
.try_into()
.map_err(|_| Error::Corrupt("short name_len".into()))?,
) as usize;
let name_at = at + 4;
let size_at = name_at + name_len;
if size_at + 8 > b.len() {
return Err(Error::Corrupt("truncated entry header".into()));
}
let name = String::from_utf8_lossy(&b[name_at..size_at]).into_owned();
let data_len = u64::from_le_bytes(
b[size_at..size_at + 8]
.try_into()
.map_err(|_| Error::Corrupt("short len".into()))?,
) as usize;
let data_at = size_at + 8;
if data_at + data_len > b.len() {
return Err(Error::Corrupt("truncated entry data".into()));
}
let mut meta = EntryMeta::file(name);
meta.size = Some(data_len as u64);
Ok(Some((meta, data_at, data_len)))
}
}
impl ArchiveRead for MockArchiveRead {
fn next_entry(&mut self) -> Result<Option<Entry<'_>>> {
let Some((meta, data_at, data_len)) = self.read_record_at(self.pos)? else {
return Ok(None);
};
self.pos = data_at + data_len;
let slice = &self.buf[data_at..data_at + data_len];
Ok(Some(Entry::new(
meta,
Box::new(std::io::Cursor::new(slice)),
)))
}
fn by_index(&mut self, index: usize) -> Result<Entry<'_>> {
let offsets = self
.index
.as_ref()
.ok_or(Error::NotSeekable {
format: MOCK_CONTAINER,
})?
.clone();
let at = *offsets
.get(index)
.ok_or_else(|| Error::EntryNotFound(index.to_string()))?;
let (meta, data_at, data_len) = self
.read_record_at(at as usize)?
.ok_or_else(|| Error::Corrupt("index points at a non-record".into()))?;
let slice = &self.buf[data_at..data_at + data_len];
Ok(Entry::new(meta, Box::new(std::io::Cursor::new(slice))))
}
fn fidelity(&self) -> &FidelityReport {
&self.report
}
}
pub struct MockArchiveWrite {
dst: Box<dyn Sink>,
offsets: Vec<u64>,
written: u64,
}
impl ArchiveWrite for MockArchiveWrite {
fn add(&mut self, meta: &EntryMeta, data: &mut dyn Read) -> Result<()> {
let mut payload = Vec::new();
data.read_to_end(&mut payload)?;
self.offsets.push(self.written);
let name = meta.name.as_bytes();
self.dst.write_all(b"ME")?;
self.dst.write_all(&(name.len() as u16).to_le_bytes())?;
self.dst.write_all(name)?;
self.dst.write_all(&(payload.len() as u64).to_le_bytes())?;
self.dst.write_all(&payload)?;
self.written += 2 + 2 + name.len() as u64 + 8 + payload.len() as u64;
Ok(())
}
fn finish(mut self: Box<Self>) -> Result<Box<dyn Sink>> {
self.dst.write_all(b"MI")?;
self.dst
.write_all(&(self.offsets.len() as u32).to_le_bytes())?;
for o in &self.offsets {
self.dst.write_all(&o.to_le_bytes())?;
}
self.dst.write_all(b"MEND")?;
Ok(self.dst)
}
}
pub struct FramedMockContainer;
pub const FRAMED_CONTAINER: FormatId = FormatId::new("framed-mock-container");
const FRAMED_MAGICS: &[MagicRule] = &[MagicRule {
offset: 0,
bytes: b"FE",
format: FRAMED_CONTAINER,
}];
pub fn framed_container_meta() -> FormatMeta {
FormatMeta::container(FRAMED_CONTAINER, &["fmc"], FRAMED_MAGICS)
}
impl Container for FramedMockContainer {
fn id(&self) -> FormatId {
FRAMED_CONTAINER
}
fn caps(&self) -> ContainerCaps {
ContainerCaps {
read: true,
write: true,
forward_parse: true,
..Default::default()
}
}
fn open(&self, resolved: Resolved, _o: &OpenOpts) -> Result<Box<dyn ArchiveRead>> {
let Resolved { source, report, .. } = resolved;
Ok(Box::new(FramedArchiveRead {
source,
pending: 0,
report,
}))
}
fn create(&self, dst: Box<dyn Sink>, _o: &CreateOpts) -> Result<Box<dyn ArchiveWrite>> {
Ok(Box::new(FramedArchiveWrite { dst }))
}
}
struct FramedArchiveWrite {
dst: Box<dyn Sink>,
}
impl ArchiveWrite for FramedArchiveWrite {
fn add(&mut self, meta: &EntryMeta, data: &mut dyn Read) -> Result<()> {
let mut payload = Vec::new();
data.read_to_end(&mut payload)?;
let name = meta.name.as_bytes();
self.dst.write_all(b"FE")?;
self.dst.write_all(&(name.len() as u32).to_le_bytes())?;
self.dst.write_all(name)?;
self.dst.write_all(&(payload.len() as u64).to_le_bytes())?;
self.dst.write_all(&payload)?;
Ok(())
}
fn finish(mut self: Box<Self>) -> Result<Box<dyn Sink>> {
self.dst.write_all(b"FZ")?;
Ok(self.dst)
}
}
struct BoundedReader<'a> {
source: &'a mut Box<dyn Source>,
remaining: &'a mut u64,
}
impl Read for BoundedReader<'_> {
fn read(&mut self, buf: &mut [u8]) -> std::io::Result<usize> {
if *self.remaining == 0 {
return Ok(0);
}
let cap = (buf.len() as u64).min(*self.remaining) as usize;
let n = self.source.read(&mut buf[..cap])?;
*self.remaining -= n as u64;
Ok(n)
}
}
struct FramedArchiveRead {
source: Box<dyn Source>,
pending: u64,
report: FidelityReport,
}
impl FramedArchiveRead {
fn skip_pending(&mut self) -> Result<()> {
if self.pending > 0 {
let mut reader = BoundedReader {
source: &mut self.source,
remaining: &mut self.pending,
};
std::io::copy(&mut reader, &mut std::io::sink())?;
}
Ok(())
}
}
fn read_exact_or_corrupt(source: &mut Box<dyn Source>, buf: &mut [u8]) -> Result<()> {
source.read_exact(buf).map_err(|e| {
if e.kind() == std::io::ErrorKind::UnexpectedEof {
Error::Corrupt("framed mock container: truncated stream".into())
} else {
Error::Io(e)
}
})
}
impl ArchiveRead for FramedArchiveRead {
fn next_entry(&mut self) -> Result<Option<Entry<'_>>> {
self.skip_pending()?;
let mut tag = [0u8; 2];
read_exact_or_corrupt(&mut self.source, &mut tag)?;
match &tag {
b"FZ" => Ok(None),
b"FE" => {
let mut len_buf = [0u8; 4];
read_exact_or_corrupt(&mut self.source, &mut len_buf)?;
let name_len = u32::from_le_bytes(len_buf) as usize;
let mut name_buf = vec![0u8; name_len];
read_exact_or_corrupt(&mut self.source, &mut name_buf)?;
let name = String::from_utf8_lossy(&name_buf).into_owned();
let mut dlen_buf = [0u8; 8];
read_exact_or_corrupt(&mut self.source, &mut dlen_buf)?;
let data_len = u64::from_le_bytes(dlen_buf);
self.pending = data_len;
let mut meta = EntryMeta::file(name);
meta.size = Some(data_len);
let reader = BoundedReader {
source: &mut self.source,
remaining: &mut self.pending,
};
Ok(Some(Entry::new(meta, Box::new(reader))))
}
_ => Err(Error::Corrupt(
"framed mock container: unrecognised record tag".into(),
)),
}
}
fn by_index(&mut self, _index: usize) -> Result<Entry<'_>> {
Err(Error::NotSeekable {
format: FRAMED_CONTAINER,
})
}
fn fidelity(&self) -> &FidelityReport {
&self.report
}
}
#[derive(Clone, Default)]
pub struct SharedBuf(std::sync::Arc<std::sync::Mutex<Vec<u8>>>);
impl SharedBuf {
pub fn new() -> Self {
Self::default()
}
pub fn contents(&self) -> Vec<u8> {
self.0.lock().expect("SharedBuf mutex poisoned").clone()
}
}
impl Write for SharedBuf {
fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {
self.0
.lock()
.expect("SharedBuf mutex poisoned")
.extend_from_slice(buf);
Ok(buf.len())
}
fn flush(&mut self) -> std::io::Result<()> {
Ok(())
}
}
pub struct FailsAfterPrefix {
prefix: Vec<u8>,
pos: usize,
kind: std::io::ErrorKind,
message: &'static str,
}
impl Read for FailsAfterPrefix {
fn read(&mut self, buf: &mut [u8]) -> std::io::Result<usize> {
if self.pos < self.prefix.len() {
let n = (self.prefix.len() - self.pos).min(buf.len());
buf[..n].copy_from_slice(&self.prefix[self.pos..self.pos + n]);
self.pos += n;
return Ok(n);
}
Err(std::io::Error::new(self.kind, self.message))
}
}
impl Source for FailsAfterPrefix {
fn caps(&self) -> crate::source::SourceCaps {
crate::source::SourceCaps {
seekable: false,
len: None,
}
}
fn as_seek(&mut self) -> Option<&mut dyn crate::source::SeekRead> {
None
}
}
pub const FAILING_SOURCE_MESSAGE: &str = "the stream is malformed from here on";
pub fn decode_side_failing_source(prefix: &[u8], kind: std::io::ErrorKind) -> Box<dyn Source> {
crate::source::DecodeSideSource::wrap(raw_failing_source(prefix, kind))
}
pub fn raw_failing_source(prefix: &[u8], kind: std::io::ErrorKind) -> Box<dyn Source> {
Box::new(FailsAfterPrefix {
prefix: prefix.to_vec(),
pos: 0,
kind,
message: FAILING_SOURCE_MESSAGE,
})
}
pub struct ChunkedReader {
data: Vec<u8>,
pos: usize,
chunk: usize,
}
impl ChunkedReader {
pub fn new(data: Vec<u8>, chunk: usize) -> Self {
assert!(chunk > 0, "chunk must be non-zero");
Self {
data,
pos: 0,
chunk,
}
}
}
impl Read for ChunkedReader {
fn read(&mut self, buf: &mut [u8]) -> std::io::Result<usize> {
let n = self.chunk.min(buf.len()).min(self.data.len() - self.pos);
buf[..n].copy_from_slice(&self.data[self.pos..self.pos + n]);
self.pos += n;
Ok(n)
}
}
#[derive(Debug)]
pub struct CountingSource<R> {
inner: R,
read: u64,
}
impl<R> CountingSource<R> {
pub fn new(inner: R) -> Self {
Self { inner, read: 0 }
}
pub fn bytes_read(&self) -> u64 {
self.read
}
pub fn into_inner(self) -> R {
self.inner
}
}
impl<R: std::io::Read> std::io::Read for CountingSource<R> {
fn read(&mut self, out: &mut [u8]) -> std::io::Result<usize> {
let n = self.inner.read(out)?;
self.read += n as u64;
Ok(n)
}
}
impl<R: std::io::Seek> std::io::Seek for CountingSource<R> {
fn seek(&mut self, to: std::io::SeekFrom) -> std::io::Result<u64> {
self.inner.seek(to)
}
}
pub const SCAN_READ_FACTOR: u64 = 8;
pub const SCAN_READ_SLACK: u64 = 64 * 1024;
pub fn check_scan_is_linear(bytes_read: u64, file_len: u64) -> std::result::Result<(), String> {
let bound = SCAN_READ_FACTOR
.saturating_mul(file_len)
.saturating_add(SCAN_READ_SLACK);
if bytes_read > bound {
return Err(format!(
"salvage scan read {bytes_read} bytes of a {file_len}-byte input, past the \
{SCAN_READ_FACTOR}x + {SCAN_READ_SLACK} bound: its work grows faster than its input"
));
}
Ok(())
}
pub fn check_sightings_bounded(
sightings: &[crate::salvage::Sighting],
total: u64,
) -> std::result::Result<(), String> {
use crate::salvage::{SightingKind, SightingLog};
let mut groups: Vec<((FormatId, &'static str, SightingKind), usize, u64)> = Vec::new();
for s in sightings {
let key = (s.format, s.shape, s.kind);
match groups.iter_mut().find(|(k, _, _)| *k == key) {
Some((_, stored, last)) => {
if s.offset <= *last {
return Err(format!(
"{} {:?} stored offset {} after {last}: not ascending",
s.format, s.shape, s.offset
));
}
*stored += 1;
*last = s.offset;
}
None => groups.push((key, 1, s.offset)),
}
}
if let Some(((format, shape, _), stored, _)) = groups
.iter()
.find(|(_, stored, _)| *stored > SightingLog::DEFAULT_CAP)
{
return Err(format!(
"{stored} {format} {shape:?} sightings stored, past the cap of {}",
SightingLog::DEFAULT_CAP
));
}
let counted: u64 = sightings.iter().map(|s| 1 + s.more).sum();
if counted != total {
return Err(format!("the sightings stand for {counted}, not {total}"));
}
Ok(())
}
#[cfg(test)]
mod tests {
use super::*;
use crate::source::ReaderSource;
use std::io::Read;
fn resolve_default(src: Box<dyn Source>) -> Resolved {
crate::resolve(
src,
MOCK_CONTAINER,
MockContainer.caps(),
&Default::default(),
)
.unwrap()
}
#[test]
fn a_decoder_returns_a_source_so_capabilities_survive_the_layer() {
let encoded: Vec<u8> = b"the quick brown fox".iter().map(|b| b ^ 0xFF).collect();
let src: Box<dyn Source> = Box::new(ReaderSource::new(std::io::Cursor::new(encoded)));
let dec = MockCodec.decoder(src, &DecodeOpts::default()).unwrap();
assert!(!dec.caps().seekable);
}
#[test]
fn mock_codec_round_trips() {
let plain = b"the quick brown fox".to_vec();
let codec = MockCodec;
let sink = SharedBuf::new();
{
let mut s = codec
.encoder(Box::new(sink.clone()), &EncodeOpts::default())
.unwrap();
std::io::Write::write_all(&mut s, &plain).unwrap();
s.finish().unwrap();
}
let encoded = sink.contents();
assert_ne!(encoded, plain, "the mock must actually transform the bytes");
let src: Box<dyn Source> = Box::new(ReaderSource::new(std::io::Cursor::new(encoded)));
let mut dec = codec.decoder(src, &DecodeOpts::default()).unwrap();
let mut out = Vec::new();
dec.read_to_end(&mut out).unwrap();
assert_eq!(out, plain);
}
#[test]
fn mock_codec_declares_both_directions() {
let c = MockCodec.caps();
assert!(c.encode && c.decode);
}
#[test]
fn mock_archive_reads_back_every_entry_in_order() {
let bytes = mock_archive_bytes(&[("a.txt", b"alpha"), ("b.bin", b"bravo!!")]);
let src: Box<dyn Source> = Box::new(ReaderSource::new(std::io::Cursor::new(bytes)));
let mut ar = MockContainer
.open(resolve_default(src), &OpenOpts::default())
.unwrap();
let mut seen = Vec::new();
while let Some(mut e) = ar.next_entry().unwrap() {
let name = e.meta().name.clone();
let mut data = Vec::new();
e.reader().read_to_end(&mut data).unwrap();
seen.push((name, data));
}
assert_eq!(seen.len(), 2);
assert_eq!(seen[0], ("a.txt".to_string(), b"alpha".to_vec()));
assert_eq!(seen[1], ("b.bin".to_string(), b"bravo!!".to_vec()));
}
#[test]
fn mock_container_declares_zip_shaped_capabilities() {
let c = MockContainer.caps();
assert!(c.read && c.write);
assert!(c.forward_parse, "must exercise the ForwardOnly rung");
assert!(c.trailing_index, "must exercise the index-loss warning");
assert!(!c.needs_seek);
}
#[test]
fn mock_archive_writer_output_is_readable_by_the_reader() {
let sink = SharedBuf::new();
{
let mut w = MockContainer
.create(
crate::archive::PlainSink::new(Box::new(sink.clone())),
&CreateOpts::default(),
)
.unwrap();
w.add(&EntryMeta::file("only.txt"), &mut &b"hello"[..])
.unwrap();
w.finish().unwrap().finish().unwrap();
}
let buf = sink.contents();
let src: Box<dyn Source> = Box::new(ReaderSource::new(std::io::Cursor::new(buf)));
let mut ar = MockContainer
.open(resolve_default(src), &OpenOpts::default())
.unwrap();
let mut e = ar.next_entry().unwrap().expect("one entry");
assert_eq!(e.meta().name, "only.txt");
let mut data = Vec::new();
e.reader().read_to_end(&mut data).unwrap();
assert_eq!(data, b"hello");
}
#[test]
fn traits_are_object_safe() {
let _c: &dyn Codec = &MockCodec;
let _k: &dyn Container = &MockContainer;
}
#[test]
fn by_index_reaches_entries_out_of_order_and_agrees_with_sequential_reads() {
let bytes =
mock_archive_bytes(&[("a.txt", b"alpha"), ("b.bin", b"bravo!!"), ("c.log", b"")]);
let seekable = || -> Box<dyn Source> {
let mut f = tempfile::NamedTempFile::new().unwrap();
std::io::Write::write_all(&mut f, &bytes).unwrap();
let (_, path) = f.keep().unwrap();
Box::new(crate::source::FileSource::open(&path).unwrap())
};
let mut ar = MockContainer
.open(resolve_default(seekable()), &OpenOpts::default())
.unwrap();
let mut sequential = Vec::new();
while let Some(mut e) = ar.next_entry().unwrap() {
let name = e.meta().name.clone();
let mut data = Vec::new();
e.reader().read_to_end(&mut data).unwrap();
sequential.push((name, data));
}
let mut ar = MockContainer
.open(resolve_default(seekable()), &OpenOpts::default())
.unwrap();
for i in [2usize, 0, 1] {
let mut e = ar.by_index(i).unwrap();
let name = e.meta().name.clone();
let mut data = Vec::new();
e.reader().read_to_end(&mut data).unwrap();
assert_eq!(
(name, data),
sequential[i].clone(),
"by_index({i}) disagreed with the sequential read"
);
}
let mut ar = MockContainer
.open(resolve_default(seekable()), &OpenOpts::default())
.unwrap();
assert!(matches!(
ar.by_index(99),
Err(crate::Error::EntryNotFound(_))
));
}
#[test]
fn open_reports_lossless_fidelity_on_a_seekable_source() {
let bytes = mock_archive_bytes(&[("a.txt", b"alpha")]);
let mut f = tempfile::NamedTempFile::new().unwrap();
std::io::Write::write_all(&mut f, &bytes).unwrap();
let src: Box<dyn Source> = Box::new(crate::source::FileSource::open(f.path()).unwrap());
let ar = MockContainer
.open(resolve_default(src), &OpenOpts::default())
.unwrap();
let report = ar.fidelity();
assert_eq!(report.rung, crate::Rung::Exact);
assert!(
report.is_lossless(),
"a seekable source must yield an authoritative, warning-free report"
);
}
#[test]
fn open_reports_index_and_count_loss_on_a_non_seekable_source() {
let bytes = mock_archive_bytes(&[("a.txt", b"alpha")]);
let src: Box<dyn Source> = Box::new(ReaderSource::new(std::io::Cursor::new(bytes)));
let ar = MockContainer
.open(resolve_default(src), &OpenOpts::default())
.unwrap();
let report = ar.fidelity();
assert_eq!(report.rung, crate::Rung::ForwardOnly);
assert!(
report
.warnings
.contains(&crate::Fidelity::TrailingIndexUnread {
format: MOCK_CONTAINER
})
);
assert!(
report
.warnings
.contains(&crate::Fidelity::EntryCountUnknown)
);
}
#[test]
fn a_container_can_see_the_rung_it_was_opened_at() {
let bytes = mock_archive_bytes(&[("a.txt", b"alpha")]);
let caps = ContainerCaps {
degraded_parse: true,
..MockContainer.caps()
};
let policy = crate::StreamPolicy::Adaptive {
allow_forward_only: false,
spill: crate::SpillPolicy::Off,
allow_degraded: true,
};
let src: Box<dyn Source> = Box::new(ReaderSource::new(std::io::Cursor::new(bytes)));
let resolved = crate::resolve(src, MOCK_CONTAINER, caps, &policy).unwrap();
assert_eq!(resolved.rung, crate::Rung::Degraded);
let ar = MockContainer.open(resolved, &OpenOpts::default()).unwrap();
assert_eq!(
ar.fidelity().rung,
crate::Rung::Degraded,
"the rung must reach the container"
);
}
#[test]
fn a_spilled_read_reports_spilled_without_a_manual_merge() {
let bytes = mock_archive_bytes(&[("a.txt", b"alpha")]);
let policy = crate::StreamPolicy::Adaptive {
allow_forward_only: false,
spill: crate::SpillPolicy::default(),
allow_degraded: true,
};
let src: Box<dyn Source> = Box::new(ReaderSource::new(std::io::Cursor::new(bytes)));
let resolved = crate::resolve(src, MOCK_CONTAINER, MockContainer.caps(), &policy).unwrap();
assert_eq!(resolved.rung, crate::Rung::Spilled);
let ar = MockContainer.open(resolved, &OpenOpts::default()).unwrap();
assert_eq!(ar.fidelity().rung, crate::Rung::Spilled);
assert!(
ar.fidelity().is_lossless(),
"spilling costs disk, not accuracy"
);
}
#[test]
fn the_container_inherits_the_ladders_seed_rather_than_rederiving_it() {
let bytes = mock_archive_bytes(&[("a.txt", b"alpha")]);
let src: Box<dyn Source> = Box::new(ReaderSource::new(std::io::Cursor::new(bytes)));
let resolved = crate::resolve(
src,
MOCK_CONTAINER,
MockContainer.caps(),
&Default::default(),
)
.unwrap();
let ar = MockContainer.open(resolved, &OpenOpts::default()).unwrap();
let n = ar
.fidelity()
.warnings
.iter()
.filter(|w| matches!(w, crate::Fidelity::EntryCountUnknown))
.count();
assert_eq!(n, 1, "EntryCountUnknown must appear exactly once");
}
#[test]
fn degraded_salvages_a_stream_that_forward_only_cannot_parse() {
let mut bytes = b"JUNKJUNK".to_vec();
bytes.extend_from_slice(&mock_archive_bytes(&[("a.txt", b"alpha")]));
let forward = crate::resolve(
Box::new(ReaderSource::new(std::io::Cursor::new(bytes.clone()))),
MOCK_CONTAINER,
MockContainer.caps(),
&crate::StreamPolicy::default(),
)
.unwrap();
assert_eq!(forward.rung, crate::Rung::ForwardOnly);
let mut ar = MockContainer.open(forward, &OpenOpts::default()).unwrap();
assert!(
ar.next_entry().unwrap().is_none(),
"a forward parse cannot start mid-junk"
);
let caps = ContainerCaps {
degraded_parse: true,
..MockContainer.caps()
};
let policy = crate::StreamPolicy::Adaptive {
allow_forward_only: false,
spill: crate::SpillPolicy::Off,
allow_degraded: true,
};
let salvage = crate::resolve(
Box::new(ReaderSource::new(std::io::Cursor::new(bytes))),
MOCK_CONTAINER,
caps,
&policy,
)
.unwrap();
assert_eq!(salvage.rung, crate::Rung::Degraded);
let mut ar = MockContainer.open(salvage, &OpenOpts::default()).unwrap();
let mut e = ar
.next_entry()
.unwrap()
.expect("salvage must recover the record");
assert_eq!(e.meta().name, "a.txt");
let mut data = Vec::new();
e.reader().read_to_end(&mut data).unwrap();
assert_eq!(data, b"alpha");
}
#[test]
fn a_codec_parallelises_only_when_handed_a_governor() {
let plain = b"the quick brown fox".to_vec();
let solo = DecodeOpts::default();
assert!(solo.governor.is_none());
assert_eq!(MockCodec.workers_for_opts(&solo), 1);
let gov = crate::Governor::new(4, 1024 * 1024 * 1024);
let opts = DecodeOpts {
threads: Some(4),
governor: Some(std::sync::Arc::clone(&gov)),
..Default::default()
};
assert_eq!(MockCodec.workers_for_opts(&opts), 4);
assert_eq!(gov.outstanding(), 0, "the lease must be released again");
let encoded: Vec<u8> = plain.iter().map(|b| b ^ 0xFF).collect();
let src: Box<dyn Source> = Box::new(ReaderSource::new(std::io::Cursor::new(encoded)));
let mut dec = MockCodec.decoder(src, &opts).unwrap();
let mut out = Vec::new();
dec.read_to_end(&mut out).unwrap();
assert_eq!(out, plain);
}
#[test]
fn opts_debug_does_not_require_a_debug_bound_on_governor() {
let gov = crate::Governor::new(2, 1024);
let o = DecodeOpts {
threads: None,
governor: Some(gov),
..Default::default()
};
assert!(format!("{o:?}").contains("governor"));
}
#[test]
fn counting_source_counts_delivered_bytes() {
use std::io::{Cursor, Seek, SeekFrom};
let mut src = CountingSource::new(Cursor::new(vec![1u8; 100]));
let mut buf = [0u8; 30];
src.read_exact(&mut buf).unwrap();
src.seek(SeekFrom::Start(0)).unwrap();
src.read_exact(&mut buf).unwrap();
assert_eq!(src.bytes_read(), 60);
}
#[test]
fn a_compat_case_round_trips_through_its_wire_format() {
let seed = encode_compat_case(
"bunzip2",
0,
&["-k", "--", "a.bz2"],
&["a.bz2"],
true,
b"BZh9",
);
let case = decode_compat_case(&seed).unwrap();
assert_eq!(
case,
CompatCase {
name: "bunzip2",
flavour: 0,
args: vec!["-k", "--", "a.bz2"],
files: vec!["a.bz2"],
stdin: true,
payload: b"BZh9",
}
);
let seed = encode_compat_case(
"gzcat",
1,
&["-S", ".x", "a.tgz"],
&["a.gz", "a.tgz", "a.z"],
false,
b"\x1f\x8b",
);
let case = decode_compat_case(&seed).unwrap();
assert_eq!(
case,
CompatCase {
name: "gzcat",
flavour: 1,
args: vec!["-S", ".x", "a.tgz"],
files: vec!["a.gz", "a.tgz", "a.z"],
stdin: false,
payload: b"\x1f\x8b",
}
);
assert!(decode_compat_case(&seed[..5]).is_none());
assert!(COMPAT_FILES.iter().all(|f| COMPAT_TOKENS.contains(f)));
assert!(
COMPAT_TOKENS
.iter()
.all(|t| !t.contains('/') && !t.contains(".."))
);
}
#[test]
fn compat_mask_skips_the_stdin_bit() {
assert_eq!(&COMPAT_FILES[4..7], ["a.gz", "a.tgz", "a.z"]);
for (file, bit) in [("a.gz", 5), ("a.tgz", 6), ("a.z", 7)] {
let seed = encode_compat_case("gzip", 0, &[], &[file], false, b"");
assert_eq!(seed.last(), Some(&(1u8 << bit)), "{file}");
let case = decode_compat_case(&seed).unwrap();
assert_eq!(case.files, [file]);
assert!(!case.stdin, "{file} set the stdin bit");
}
let case = decode_compat_case(&[4, 0, COMPAT_STDIN_BIT]).unwrap();
assert!(case.stdin);
assert!(case.files.is_empty(), "{:?}", case.files);
let case = decode_compat_case(&[4, 0, 0xff]).unwrap();
assert_eq!(case.files, COMPAT_FILES[..7]);
assert!(case.stdin);
}
#[test]
fn old_seeds_decode_unchanged() {
const OLD_NAMES: [&str; 4] = ["bzip2", "bunzip2", "bzcat", "bzip2recover"];
const OLD_FILES: [&str; 4] = ["a", "a.bz2", "a.tbz2", "a.out"];
const OLD_TOKEN_TABLE: [&str; 56] = [
"-c",
"-d",
"-z",
"-f",
"-t",
"-k",
"-s",
"-q",
"-v",
"-V",
"-L",
"-h",
"-1",
"-2",
"-3",
"-4",
"-5",
"-6",
"-7",
"-8",
"-9",
"-x",
"-dc",
"-cd",
"-tv",
"-kf",
"-dfk",
"-qt",
"-vvv",
"-zc",
"-s1",
"-dq",
"--stdout",
"--decompress",
"--compress",
"--force",
"--test",
"--keep",
"--small",
"--quiet",
"--version",
"--license",
"--exponential",
"--repetitive-best",
"--repetitive-fast",
"--fast",
"--best",
"--verbose",
"--help",
"--bogus",
"--",
"-",
"a",
"a.bz2",
"a.tbz2",
"a.out",
];
const OLD_TOKENS: usize = OLD_TOKEN_TABLE.len();
assert_eq!(COMPAT_TOKENS[..OLD_TOKENS], OLD_TOKEN_TABLE);
assert_eq!(COMPAT_NAMES[..4], OLD_NAMES);
assert_eq!(COMPAT_FILES[..4], OLD_FILES);
let seed = [1, 3, 5, 50, 53, 0b1_0010, b'B', b'Z', b'h', b'9'];
let case = decode_compat_case(&seed).unwrap();
assert_eq!(
case,
CompatCase {
name: "bunzip2",
flavour: 0,
args: vec!["-k", "--", "a.bz2"],
files: vec!["a.bz2"],
stdin: true,
payload: b"BZh9",
}
);
for name in 0..4u8 {
for mask in 0..32u8 {
for token in 0..OLD_TOKENS as u8 {
let seed = [name, 1, token, mask];
let case = decode_compat_case(&seed).unwrap();
let old_files: Vec<&str> = (0..4)
.filter(|i| mask & (1 << i) != 0)
.map(|i| OLD_FILES[i])
.collect();
assert_eq!(case.name, OLD_NAMES[usize::from(name)]);
assert_eq!(case.flavour, 0);
assert_eq!(case.args, [OLD_TOKEN_TABLE[usize::from(token)]]);
assert_eq!(case.files, old_files);
assert_eq!(case.stdin, mask & 0x10 != 0);
}
}
}
seeds_of_0_12_0_decode_unchanged();
}
fn seeds_of_0_12_0_decode_unchanged() {
const OLD_NAMES: [&str; 8] = [
"bzip2",
"bunzip2",
"bzcat",
"bzip2recover",
"gzip",
"gunzip",
"zcat",
"gzcat",
];
const OLD_FILES: [&str; 7] = ["a", "a.bz2", "a.tbz2", "a.out", "a.gz", "a.tgz", "a.z"];
const OLD_TOKENS: usize = 74;
assert_eq!(COMPAT_NAMES[..8], OLD_NAMES);
assert_eq!(COMPAT_FILES[..7], OLD_FILES);
assert_eq!(COMPAT_TOKENS[OLD_TOKENS - 1], "a.z");
let old_bit = |i: usize| 1u8 << if i < 4 { i } else { i + 1 };
for name in 0..8u8 {
for argc in 0..=COMPAT_MAX_ARGS as u8 {
for mask in 0..=255u8 {
let token = (u32::from(mask) * 7 + u32::from(argc)) as u8 % OLD_TOKENS as u8;
let mut seed = vec![name, argc];
seed.extend(std::iter::repeat_n(token, usize::from(argc)));
seed.extend([mask, 0xA5, 0x5A]);
let case = decode_compat_case(&seed).unwrap();
let old_files: Vec<&str> = (0..7)
.filter(|&i| mask & old_bit(i) != 0)
.map(|i| OLD_FILES[i])
.collect();
assert_eq!(case.name, OLD_NAMES[usize::from(name)]);
assert_eq!(case.flavour, 0);
assert_eq!(case.args.len(), usize::from(argc));
assert!(
case.args
.iter()
.all(|a| *a == COMPAT_TOKENS[usize::from(token)])
);
assert_eq!(case.files, old_files, "mask {mask:#x}");
assert_eq!(case.stdin, mask & 0x10 != 0);
assert_eq!(case.payload, [0xA5, 0x5A], "argc {argc} mask {mask:#x}");
let again = encode_compat_case(
case.name,
0,
&case.args,
&case.files,
case.stdin,
case.payload,
);
assert_eq!(again, seed, "argc {argc} mask {mask:#x}");
}
}
}
}
#[test]
fn xz_files_take_bits_past_the_stdin_bit() {
assert_eq!(
&COMPAT_FILES[7..],
["a.xz", "a.txz", "a.lzma", "a.tlz", "a.lz"]
);
for (i, file) in COMPAT_FILES.iter().enumerate().skip(7) {
let bit = i + 1;
assert!((8..=12).contains(&bit));
let seed = encode_compat_case("xz", 0, &["-k", file], &[file], false, b"p");
let argc = COMPAT_MAX_ARGS as u8 + 1 + 2;
let k = COMPAT_TOKENS.iter().position(|t| *t == "-k").unwrap() as u8;
let f = COMPAT_TOKENS.iter().position(|t| t == file).unwrap() as u8;
assert_eq!(seed, [8, argc, k, f, 0, 1 << (bit - 8), b'p'], "{file}");
let case = decode_compat_case(&seed).unwrap();
assert_eq!(case.name, "xz");
assert_eq!(case.args, ["-k", *file]);
assert_eq!(case.files, [*file]);
assert!(!case.stdin, "{file} set the stdin bit");
assert_eq!(case.payload, b"p");
}
let seed = encode_compat_case("unxz", 0, &[], &["a", "a.z", "a.lz"], true, b"");
let case = decode_compat_case(&seed).unwrap();
assert_eq!(case.files, ["a", "a.z", "a.lz"]);
assert!(case.stdin);
let case = decode_compat_case(&[8, 9, 0xff, 0xff]).unwrap();
assert_eq!(case.files, COMPAT_FILES);
assert!(case.stdin);
assert!(case.args.is_empty());
let case = decode_compat_case(&[8, 9, 0, 0xe0, 1]).unwrap();
assert!(case.files.is_empty());
assert_eq!(case.payload, [1]);
assert!(decode_compat_case(&[8, 9, 0]).is_none());
let case = decode_compat_case(&[8, 11, 0, 1, 0, 0]).unwrap();
assert_eq!(case.args, [COMPAT_TOKENS[0], COMPAT_TOKENS[1]]);
assert!(COMPAT_FILES.iter().all(|f| COMPAT_TOKENS.contains(f)));
}
#[test]
fn flavour_from_name_byte() {
let len = COMPAT_NAMES.len() as u8;
assert_eq!(len, 14);
let at = |byte: u8| {
let seed = [byte, 0, 0];
let case = decode_compat_case(&seed).unwrap();
(case.name, case.flavour)
};
assert_eq!(at(0), ("bzip2", 0));
assert_eq!(at(4), ("gzip", 0));
assert_eq!(at(7), ("gzcat", 0));
assert_eq!(at(8), ("xz", 0));
assert_eq!(at(13), ("lzcat", 0));
assert_eq!(at(14), ("bzip2", 1));
assert_eq!(at(18), ("gzip", 1));
assert_eq!(at(21), ("gzcat", 1));
assert_eq!(at(28), ("bzip2", 0));
assert_eq!(at(255), ("bzip2recover", 0));
for (i, name) in COMPAT_NAMES.iter().enumerate() {
for flavour in 0..2u8 {
let seed = encode_compat_case(name, flavour, &[], &[], false, b"");
assert_eq!(
usize::from(seed[0]),
i + usize::from(flavour) * COMPAT_NAMES.len()
);
assert_eq!(at(seed[0]), (*name, flavour));
}
}
}
#[test]
fn the_linear_bound_is_factor_times_length_plus_slack() {
let len = 1_000_000;
assert!(check_scan_is_linear(SCAN_READ_FACTOR * len + SCAN_READ_SLACK, len).is_ok());
let err =
check_scan_is_linear(SCAN_READ_FACTOR * len + SCAN_READ_SLACK + 1, len).unwrap_err();
assert!(err.contains("read"), "{err}");
}
}