use crate::{CodecError, Result};
const CODEC: &str = "chunked";
fn bad(detail: impl Into<String>) -> CodecError {
CodecError::Decode {
codec: CODEC,
detail: detail.into(),
}
}
fn varint(d: &[u8], i: &mut usize) -> Result<u64> {
let mut v = 0u64;
for shift in (0..64).step_by(7) {
let b = *d
.get(*i)
.ok_or_else(|| bad("header ends inside a number"))?;
*i += 1;
v |= u64::from(b & 0x7F)
.checked_shl(shift)
.filter(|x| x >> shift == u64::from(b & 0x7F))
.ok_or_else(|| bad("number in the header overflows"))?;
if b & 0x80 == 0 {
return Ok(v);
}
}
Err(bad("number in the header is too long"))
}
fn varints(d: &[u8]) -> Result<Vec<u64>> {
let mut i = 0;
let mut out = Vec::new();
while i < d.len() {
out.push(varint(d, &mut i)?);
}
Ok(out)
}
pub fn chunked_decode(
data: &[u8],
expected_len: usize,
bytes_per_sample: usize,
) -> Result<Vec<u8>> {
let mut i = 0usize;
let mut sizes: Option<Vec<u64>> = None;
let mut raw_sizes: Option<Vec<u64>> = None;
let mut method = 0u8;
let mut hilo = false;
loop {
let id = varint(data, &mut i)?;
if id == 0 {
break;
}
let len =
usize::try_from(varint(data, &mut i)?).map_err(|_| bad("header entry too long"))?;
let payload = i
.checked_add(len)
.and_then(|e| data.get(i..e))
.ok_or_else(|| bad("header entry runs past the data"))?;
i += len;
match id {
1 => sizes = Some(varints(payload)?),
2 => {
method = *payload.first().ok_or_else(|| bad("empty codec entry"))?;
}
3 => raw_sizes = Some(varints(payload)?),
4 => match payload.first() {
Some(0) => hilo = false,
Some(1) => hilo = true,
Some(v) => {
return Err(CodecError::Unsupported {
codec: CODEC,
detail: format!("preprocessing {v}"),
});
}
None => return Err(bad("empty preprocessing entry")),
},
other => {
return Err(CodecError::Unsupported {
codec: CODEC,
detail: format!("header entry {other}"),
});
}
}
}
let sizes = sizes.ok_or_else(|| bad("no chunk sizes in the header"))?;
let raw_sizes = raw_sizes.ok_or_else(|| bad("no decompressed sizes in the header"))?;
if method > 1 {
return Err(CodecError::Unsupported {
codec: CODEC,
detail: format!("codec {method}"),
});
}
if hilo && bytes_per_sample != 2 {
return Err(CodecError::Unsupported {
codec: CODEC,
detail: format!("HiLo split of {bytes_per_sample}-byte samples"),
});
}
let n = sizes.len();
if hilo && n > 1 {
return Err(CodecError::Unsupported {
codec: CODEC,
detail: format!("HiLo split across {n} chunks"),
});
}
let raw_size = |k: usize| -> Result<u64> {
match raw_sizes.len() {
m if m == n => Ok(raw_sizes[k]),
1 => Ok(raw_sizes[0]),
2 => Ok(if k + 1 == n {
raw_sizes[1]
} else {
raw_sizes[0]
}),
m => Err(CodecError::Unsupported {
codec: CODEC,
detail: format!("{m} decompressed sizes for {n} chunks"),
}),
}
};
let limit = if expected_len == 0 {
crate::MAX_UNSIZED_OUTPUT
} else {
expected_len
};
let mut out = Vec::with_capacity(limit.min(crate::MAX_PREALLOC));
for (k, &csize) in sizes.iter().enumerate() {
let csize = usize::try_from(csize).map_err(|_| bad("chunk size overflows"))?;
let chunk = i
.checked_add(csize)
.and_then(|e| data.get(i..e))
.ok_or_else(|| bad(format!("chunk {k} runs past the data")))?;
i += csize;
let room = limit.saturating_sub(out.len());
let want = usize::try_from(raw_size(k)?).unwrap_or(usize::MAX);
if want > room {
return Err(bad(format!("chunk {k} decodes past the expected size")));
}
let mut d = match method {
0 => crate::zstd_decode(chunk, want)?,
_ => crate::lz4_block_decode(chunk, want)?,
};
if d.len() != want {
return Err(bad(format!(
"chunk {k} decodes to {} bytes, expected {want}",
d.len()
)));
}
if hilo {
if d.len() % 2 != 0 {
return Err(bad(format!("chunk {k} holds half a 16-bit sample")));
}
d = crate::unshuffle_hilo(&d);
}
out.extend_from_slice(&d);
}
if expected_len != 0 && out.len() != expected_len {
return Err(CodecError::SizeMismatch {
codec: CODEC,
got: out.len(),
expected: expected_len,
});
}
Ok(out)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn varints_are_seven_bit_groups() {
let mut i = 0;
assert_eq!(varint(&[0xCA, 0x27], &mut i).unwrap(), 5066);
assert_eq!(i, 2);
assert!(varint(&[0x80], &mut 0).is_err());
assert!(varint(&[0xFF; 11], &mut 0).is_err());
}
#[test]
fn malformed_headers_are_refused() {
assert!(chunked_decode(&[], 0, 1).is_err());
assert!(chunked_decode(&[0x00], 0, 1).is_err()); assert!(chunked_decode(&[0x09, 0x01, 0x00, 0x00], 0, 1).is_err()); assert!(chunked_decode(&[0x01, 0x05, 0x01], 0, 1).is_err()); let r = chunked_decode(
&[0x01, 0x01, 0x00, 0x03, 0x01, 0x00, 0x02, 0x01, 0x07, 0x00],
0,
1,
);
assert!(matches!(r, Err(CodecError::Unsupported { .. })));
let r = chunked_decode(
&[
0x01, 0x02, 0x00, 0x00, 0x03, 0x01, 0x00, 0x04, 0x01, 0x01, 0x00,
],
0,
2,
);
assert!(matches!(r, Err(CodecError::Unsupported { .. })));
}
#[cfg(feature = "lz4")]
#[test]
fn two_size_form_gives_the_last_chunk_its_own_size() {
let stream = [
0x01, 0x02, 0x05, 0x03, 0x03, 0x02, 0x04, 0x02, 0x02, 0x01, 0x01, 0x00, 0x40, 1, 2, 3, 4, 0x20, 5, 6,
];
assert_eq!(chunked_decode(&stream, 6, 1).unwrap(), [1, 2, 3, 4, 5, 6]);
assert!(chunked_decode(&stream, 7, 1).is_err());
}
}