use std::io::{Cursor, Write};
use bytes::Bytes;
use thiserror::Error;
use xet_core_structures::error::CoreError;
use xet_core_structures::merklehash::MerkleHash;
use xet_core_structures::xorb_object::{
XORB_CHUNK_HEADER_LENGTH, deserialize_chunks, deserialize_chunks_to_writer, parse_chunk_header,
};
use crate::hash::compute_chunk_hash;
#[derive(Debug, Error)]
pub enum XorbError {
#[error("failed to parse serialized xorb: {0}")]
Format(#[from] CoreError),
#[error("xorb chunk byte offsets are inconsistent with the decoded data")]
InconsistentChunkOffsets,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct DecodedChunk {
pub data: Vec<u8>,
pub hash: MerkleHash,
}
#[derive(Debug, Clone)]
pub struct DecodedChunkData {
pub data: Bytes,
pub chunk_offsets: Vec<usize>,
}
#[derive(Debug, Clone)]
pub struct XorbReader {
bytes: Vec<u8>,
}
impl XorbReader {
#[must_use]
pub const fn new(bytes: Vec<u8>) -> Self {
Self { bytes }
}
pub fn decode_chunk_data(&self) -> Result<DecodedChunkData, XorbError> {
let (data, offsets) = deserialize_chunks(&mut Cursor::new(self.bytes.as_slice()))?;
Ok(DecodedChunkData {
data: Bytes::from(data),
chunk_offsets: offsets
.into_iter()
.map(|offset| usize::try_from(offset).unwrap_or(usize::MAX))
.collect(),
})
}
pub(crate) fn decode_chunk_data_bounded(
&self,
limit: usize,
) -> Result<DecodedChunkData, XorbError> {
let mut position = 0usize;
let mut decoded_size = 0usize;
let mut chunk_count = 0usize;
while position < self.bytes.len() {
chunk_count = chunk_count.saturating_add(1);
if chunk_count > 8192 {
return Err(XorbError::InconsistentChunkOffsets);
}
let header_end = position
.checked_add(XORB_CHUNK_HEADER_LENGTH)
.ok_or(XorbError::InconsistentChunkOffsets)?;
let header_bytes = self
.bytes
.get(position..header_end)
.ok_or(XorbError::InconsistentChunkOffsets)?;
let header = parse_chunk_header(
header_bytes
.try_into()
.map_err(|_error| XorbError::InconsistentChunkOffsets)?,
)?;
decoded_size = decoded_size
.checked_add(header.get_uncompressed_length() as usize)
.filter(|size| *size <= limit)
.ok_or_else(|| {
XorbError::Format(CoreError::Io(std::io::Error::new(
std::io::ErrorKind::InvalidData,
"decoded xorb exceeds buffer reservation",
)))
})?;
position = header_end
.checked_add(header.get_compressed_length() as usize)
.filter(|end| *end <= self.bytes.len())
.ok_or(XorbError::InconsistentChunkOffsets)?;
}
let mut writer = BoundedChunkWriter {
data: Vec::with_capacity(decoded_size),
limit: decoded_size,
};
let (_, offsets) =
deserialize_chunks_to_writer(&mut Cursor::new(self.bytes.as_slice()), &mut writer)?;
Ok(DecodedChunkData {
data: Bytes::from(writer.data),
chunk_offsets: offsets
.into_iter()
.map(|offset| usize::try_from(offset).unwrap_or(usize::MAX))
.collect(),
})
}
pub fn decode_chunks(&self) -> Result<Vec<DecodedChunk>, XorbError> {
let (data, offsets) = deserialize_chunks(&mut Cursor::new(self.bytes.as_slice()))?;
let mut chunks = Vec::with_capacity(offsets.len().saturating_sub(1));
let mut previous_offset: Option<u32> = None;
for offset in offsets {
if let Some(start) = previous_offset {
let end = offset;
let chunk_bytes = data
.get(start as usize..end as usize)
.ok_or(XorbError::InconsistentChunkOffsets)?;
let hash = compute_chunk_hash(chunk_bytes);
chunks.push(DecodedChunk {
data: chunk_bytes.to_vec(),
hash,
});
}
previous_offset = Some(offset);
}
Ok(chunks)
}
}
struct BoundedChunkWriter {
data: Vec<u8>,
limit: usize,
}
impl Write for BoundedChunkWriter {
fn write(&mut self, bytes: &[u8]) -> std::io::Result<usize> {
if self
.data
.len()
.checked_add(bytes.len())
.is_none_or(|size| size > self.limit)
{
return Err(std::io::Error::other(
"decoded xorb exceeds buffer reservation",
));
}
self.data
.try_reserve_exact(bytes.len())
.map_err(std::io::Error::other)?;
self.data.extend_from_slice(bytes);
Ok(bytes.len())
}
fn flush(&mut self) -> std::io::Result<()> {
Ok(())
}
}
#[cfg(test)]
mod tests {
use xet_core_structures::merklehash::MerkleHash;
use xet_core_structures::xorb_object::{CompressionScheme, serialize_chunk};
#[test]
fn bounded_decode_rejects_declared_output_before_decompression() {
let mut payload = Vec::new();
serialize_chunk(&vec![7u8; 4096], &mut payload, CompressionScheme::LZ4).unwrap();
let reader = super::XorbReader::new(payload);
assert!(matches!(
reader.decode_chunk_data_bounded(8),
Err(super::XorbError::Format(_))
));
let decoded = reader.decode_chunk_data_bounded(4096).unwrap();
assert_eq!(decoded.data.as_ref(), vec![7u8; 4096]);
}
#[test]
fn bounded_decode_limits_zero_length_chunk_metadata() {
let one = [0u8; 8];
let payload = one.repeat(8193);
assert!(matches!(
super::XorbReader::new(payload).decode_chunk_data_bounded(1),
Err(super::XorbError::InconsistentChunkOffsets)
));
}
use super::{DecodedChunk, XorbError, XorbReader};
use crate::hash::compute_chunk_hash;
fn serialize_payload(chunks: &[&[u8]]) -> Vec<u8> {
let mut payload = Vec::new();
for chunk in chunks {
serialize_chunk(chunk, &mut payload, CompressionScheme::None).unwrap();
}
payload
}
fn expected_chunks(chunks: &[&[u8]]) -> Vec<DecodedChunk> {
chunks
.iter()
.map(|data| DecodedChunk {
data: data.to_vec(),
hash: compute_chunk_hash(data),
})
.collect()
}
#[test]
fn decode_chunks_round_trips_payload_without_footer() {
let chunk_a = b"hello xet chunk";
let chunk_b = b"second chunk payload";
let payload = serialize_payload(&[chunk_a, chunk_b]);
let decoded = XorbReader::new(payload).decode_chunks().unwrap();
assert_eq!(decoded, expected_chunks(&[chunk_a, chunk_b]));
}
#[test]
fn decode_chunks_single_chunk() {
let chunk = b"just one chunk";
let payload = serialize_payload(&[chunk]);
let decoded = XorbReader::new(payload).decode_chunks().unwrap();
assert_eq!(decoded, expected_chunks(&[chunk]));
}
#[test]
fn decode_chunks_empty_payload_yields_no_chunks() {
let decoded = XorbReader::new(Vec::new()).decode_chunks().unwrap();
assert!(decoded.is_empty());
}
#[test]
fn decode_chunks_rejects_truncated_chunk() {
let chunk = b"truncated-chunk-data";
let mut payload = serialize_payload(&[chunk]);
payload.truncate(payload.len() - 3);
let result = XorbReader::new(payload).decode_chunks();
assert!(matches!(result, Err(XorbError::Format(_))));
}
#[test]
fn decode_chunks_rejects_footer_appended_after_payload() {
let chunk = b"with-footer";
let mut payload = serialize_payload(&[chunk]);
payload.extend_from_slice(&[b'X', b'E', b'T', b'B', b'L', b'O', b'B', 1, 0, 0, 0]);
let result = XorbReader::new(payload).decode_chunks();
assert!(matches!(result, Err(XorbError::Format(_))));
}
#[test]
fn decode_chunks_computes_data_hashes() {
let chunk = b"hash me";
let payload = serialize_payload(&[chunk]);
let decoded = XorbReader::new(payload).decode_chunks().unwrap();
assert_eq!(decoded[0].hash, compute_chunk_hash(chunk));
assert_ne!(decoded[0].hash, compute_chunk_hash(b"hash different"));
}
#[test]
fn decode_chunks_handles_incompressible_data() {
let chunk: Vec<u8> = (0..512).map(|i| (i * 31 % 251) as u8).collect();
let payload = serialize_payload(&[&chunk]);
let decoded = XorbReader::new(payload).decode_chunks().unwrap();
assert_eq!(decoded[0].data, chunk);
}
#[test]
fn term_verification_hash_matches_xorb_footer_chunk_range_hash() {
use std::io::Cursor;
use xet_core_structures::merklehash::xorb_hash;
use xet_core_structures::xorb_object::{XorbObject, XorbObjectInfoV1, serialize_chunk};
let chunks: [Vec<u8>; 3] = [
b"first chunk of the xorb".to_vec(),
b"second chunk of the xorb".to_vec(),
b"third chunk of the xorb".to_vec(),
];
let chunk_hashes: Vec<MerkleHash> = chunks
.iter()
.map(|chunk| compute_chunk_hash(chunk))
.collect();
let mut payload = Vec::new();
let mut boundaries: Vec<u32> = Vec::new();
let mut unpacked: Vec<u32> = Vec::new();
let mut packed_total: u32 = 0;
let mut unpacked_total: u32 = 0;
for chunk in &chunks {
let written = serialize_chunk(chunk, &mut payload, CompressionScheme::None).unwrap();
packed_total = packed_total.saturating_add(u32::try_from(written).unwrap());
boundaries.push(packed_total);
unpacked_total = unpacked_total.saturating_add(u32::try_from(chunk.len()).unwrap());
unpacked.push(unpacked_total);
}
let mut info = XorbObjectInfoV1::default();
info.xorb_hash = xorb_hash(
&chunks
.iter()
.map(|chunk| {
(
compute_chunk_hash(chunk),
u64::try_from(chunk.len()).unwrap(),
)
})
.collect::<Vec<_>>(),
);
info.num_chunks = u32::try_from(chunks.len()).unwrap();
info.chunk_hashes = chunk_hashes.clone();
info.chunk_boundary_offsets = boundaries;
info.unpacked_chunk_offsets = unpacked;
info.fill_in_boundary_offsets();
let content_len = usize::try_from(packed_total).unwrap();
let (_xorb_object, _footer_len) =
XorbObject::serialize_given_info(&mut payload, info).unwrap();
let decoded = XorbReader::new(payload[..content_len].to_vec())
.decode_chunks()
.unwrap();
let decoded_hashes: Vec<MerkleHash> = decoded.iter().map(|chunk| chunk.hash).collect();
let computed = crate::hash::compute_term_verification_hash(&decoded_hashes);
let xorb = XorbObject::deserialize(&mut Cursor::new(payload.as_slice())).unwrap();
let authoritative = xorb
.generate_chunk_range_hash(0, u32::try_from(chunks.len()).unwrap())
.unwrap();
assert_eq!(computed, authoritative);
assert_eq!(decoded_hashes, chunk_hashes);
}
}