use tokio::io::{AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt};
use super::Frame;
use super::error::{ProtocolError, Result};
const MAX_LEN: u32 = 0x3FFF_FFFF;
const CHUNK_FLAG: u32 = 1 << 30;
const CHUNK_TAG: u8 = 0;
const BATCH_TAG: u8 = 1;
#[must_use]
fn len_ok(len: usize) -> bool {
len <= MAX_LEN as usize
}
#[expect(clippy::cast_possible_truncation)]
fn frame_prefix(len: usize, flag: u32) -> [u8; 4] {
(len as u32 | flag).to_be_bytes()
}
pub fn encode(frame: &Frame, compress: bool, threshold: usize) -> Result<Vec<u8>> {
let data = postcard::to_allocvec(frame)
.map_err(|e| ProtocolError::Protocol(format!("Failed to serialize frame: {e}")))?;
let (payload, flag): (Vec<u8>, u32) = if compress && data.len() > threshold {
(lz4_flex::block::compress_prepend_size(&data), 1 << 31)
} else {
(data, 0)
};
if !len_ok(payload.len()) {
return Err(ProtocolError::Protocol(
"Frame exceeds maximum length".to_string(),
));
}
let mut wire = Vec::with_capacity(4 + payload.len());
wire.extend_from_slice(&frame_prefix(payload.len(), flag));
wire.extend_from_slice(&payload);
Ok(wire)
}
pub fn decode(prefix: [u8; 4], payload: &[u8]) -> Result<Frame> {
let len = u32::from_be_bytes(prefix);
if len & CHUNK_FLAG != 0 {
return Err(ProtocolError::Protocol(
"chunk frame reached the postcard decode path".to_string(),
));
}
if len & (1 << 31) != 0 {
let claimed = payload
.get(0..4)
.ok_or_else(|| ProtocolError::Protocol("Truncated lz4 size header".to_string()))?;
let claimed = u32::from_le_bytes([claimed[0], claimed[1], claimed[2], claimed[3]]);
if claimed > MAX_LEN {
return Err(ProtocolError::Protocol(format!(
"Compressed frame claims {claimed} bytes, above the {MAX_LEN} cap"
)));
}
let data = lz4_flex::block::decompress_size_prepended(payload)
.map_err(|e| ProtocolError::Protocol(format!("Failed to decompress frame: {e}")))?;
postcard::from_bytes(&data)
.map_err(|e| ProtocolError::Protocol(format!("Failed to deserialize frame: {e}")))
} else {
postcard::from_bytes(payload)
.map_err(|e| ProtocolError::Protocol(format!("Failed to deserialize frame: {e}")))
}
}
pub async fn send_chunk_frame<W: AsyncWrite + Unpin>(
stream: &mut W,
file_id: u64,
data: &[u8],
) -> Result<()> {
let mut wire = Vec::with_capacity(4 + 1 + 8 + data.len());
chunk_frame_wire(file_id, data, &mut wire)?;
stream
.write_all(&wire)
.await
.map_err(|e| ProtocolError::Protocol(format!("Failed to write chunk frame: {e}")))?;
stream
.flush()
.await
.map_err(|e| ProtocolError::Protocol(format!("Failed to flush chunk frame: {e}")))
}
pub fn chunk_frame_wire(file_id: u64, data: &[u8], out: &mut Vec<u8>) -> Result<()> {
let total = 1 + 8 + data.len();
if !len_ok(total) {
return Err(ProtocolError::Protocol(
"Chunk frame exceeds maximum length".to_string(),
));
}
out.extend_from_slice(&frame_prefix(total, CHUNK_FLAG));
out.push(CHUNK_TAG);
out.extend_from_slice(&file_id.to_le_bytes());
out.extend_from_slice(data);
Ok(())
}
pub async fn send_batch_raw<W: AsyncWrite + Unpin>(
stream: &mut W,
items: &[crate::protocol::BatchItem],
) -> Result<()> {
let mut total: u64 = 1 + 4;
for item in items {
total += 8 + 4 + item.file_path.len() as u64 + 8 + 1 + item.data.len() as u64;
if item.checksum.is_some() {
total += 32;
}
}
if total > u64::from(MAX_LEN) {
return Err(ProtocolError::Protocol(
"Batch frame exceeds maximum length".to_string(),
));
}
let mut head = Vec::with_capacity(4 + 1 + 4);
#[expect(clippy::cast_possible_truncation)]
let total = total as u32;
head.extend_from_slice(&frame_prefix(total as usize, CHUNK_FLAG));
head.push(BATCH_TAG);
#[expect(clippy::cast_possible_truncation)]
head.extend_from_slice(&(items.len() as u32).to_le_bytes());
stream
.write_all(&head)
.await
.map_err(|e| ProtocolError::Protocol(format!("Failed to write batch frame: {e}")))?;
let header_size = items
.iter()
.map(|i| 8 + 4 + i.file_path.len() + 8 + 1 + usize::from(i.checksum.is_some()) * 32)
.sum::<usize>();
let mut headers = Vec::with_capacity(header_size);
for item in items {
headers.extend_from_slice(&item.file_id.to_le_bytes());
#[expect(clippy::cast_possible_truncation)]
headers.extend_from_slice(&(item.file_path.len() as u32).to_le_bytes());
headers.extend_from_slice(item.file_path.as_bytes());
headers.extend_from_slice(&(item.data.len() as u64).to_le_bytes());
match item.checksum {
Some(checksum) => {
headers.push(1);
headers.extend_from_slice(&checksum);
}
None => headers.push(0),
}
}
stream
.write_all(&headers)
.await
.map_err(|e| ProtocolError::Protocol(format!("Failed to write batch frame: {e}")))?;
for item in items {
stream
.write_all(&item.data)
.await
.map_err(|e| ProtocolError::Protocol(format!("Failed to write batch frame: {e}")))?;
}
stream
.flush()
.await
.map_err(|e| ProtocolError::Protocol(format!("Failed to flush batch frame: {e}")))
}
pub async fn send_frame<W: AsyncWrite + Unpin>(stream: &mut W, frame: &Frame) -> Result<()> {
send_frame_compressed(stream, frame, false, usize::MAX).await
}
pub async fn send_frame_compressed<W: AsyncWrite + Unpin>(
stream: &mut W,
frame: &Frame,
compress: bool,
threshold: usize,
) -> Result<()> {
let wire = encode(frame, compress, threshold)?;
stream
.write_all(&wire)
.await
.map_err(|e| ProtocolError::Protocol(format!("Failed to write frame: {e}")))?;
stream
.flush()
.await
.map_err(|e| ProtocolError::Protocol(format!("Failed to flush frame: {e}")))
}
pub async fn receive_frame<R: AsyncRead + Unpin>(stream: &mut R) -> Result<Frame> {
let mut len_buf = [0u8; 4];
stream
.read_exact(&mut len_buf)
.await
.map_err(|e| ProtocolError::Protocol(format!("Failed to read frame length: {e}")))?;
let len = u32::from_be_bytes(len_buf);
if len & CHUNK_FLAG != 0 {
let size = (len & MAX_LEN) as usize;
let mut tag_buf = [0u8; 1];
stream
.read_exact(&mut tag_buf)
.await
.map_err(|e| ProtocolError::Protocol(format!("Failed to read frame tag: {e}")))?;
match tag_buf[0] {
CHUNK_TAG => {
let data_len = size
.checked_sub(1 + 8)
.ok_or_else(|| ProtocolError::Protocol("Chunk frame shorter than its header".to_string()))?;
let mut head = [0u8; 8];
stream
.read_exact(&mut head)
.await
.map_err(|e| ProtocolError::Protocol(format!("Failed to read chunk header: {e}")))?;
let file_id = u64::from_le_bytes(head);
let mut data = vec![0u8; data_len];
stream
.read_exact(&mut data)
.await
.map_err(|e| ProtocolError::Protocol(format!("Failed to read chunk data: {e}")))?;
return Ok(Frame::FileChunk { file_id, data });
}
BATCH_TAG => {
use crate::delta::Delta;
let mut count_buf = [0u8; 4];
stream
.read_exact(&mut count_buf)
.await
.map_err(|e| ProtocolError::Protocol(format!("Failed to read batch header: {e}")))?;
let count = u32::from_le_bytes(count_buf) as usize;
if count > size {
return Err(ProtocolError::Protocol(
"Batch frame count exceeds its payload length".to_string(),
));
}
let mut recipes = Vec::with_capacity(count);
let mut data_lens: Vec<usize> = Vec::with_capacity(count);
let mut remaining = size.saturating_sub(1 + 4);
for _ in 0..count {
let mut head = [0u8; 8];
stream
.read_exact(&mut head)
.await
.map_err(|e| ProtocolError::Protocol(format!("Failed to read batch header: {e}")))?;
let file_id = u64::from_le_bytes(head);
let mut path_len_buf = [0u8; 4];
stream
.read_exact(&mut path_len_buf)
.await
.map_err(|e| ProtocolError::Protocol(format!("Failed to read batch header: {e}")))?;
let path_len = u32::from_le_bytes(path_len_buf) as usize;
if path_len > remaining {
return Err(ProtocolError::Protocol(
"Batch record path exceeds the frame length".to_string(),
));
}
remaining -= path_len;
let mut path_bytes = vec![0u8; path_len];
stream
.read_exact(&mut path_bytes)
.await
.map_err(|e| ProtocolError::Protocol(format!("Failed to read batch header: {e}")))?;
let file_path = String::from_utf8(path_bytes).map_err(|e| {
ProtocolError::Protocol(format!("Invalid batch path: {e}"))
})?;
let mut size_buf = [0u8; 8];
stream
.read_exact(&mut size_buf)
.await
.map_err(|e| ProtocolError::Protocol(format!("Failed to read batch header: {e}")))?;
let data_len = u64::from_le_bytes(size_buf);
if data_len > remaining as u64 {
return Err(ProtocolError::Protocol(
"Batch record data exceeds the frame length".to_string(),
));
}
#[expect(clippy::cast_possible_truncation, reason = "bounded by the remaining-budget check above")]
let data_len = data_len as usize;
remaining -= data_len;
let mut flag_buf = [0u8; 1];
stream
.read_exact(&mut flag_buf)
.await
.map_err(|e| ProtocolError::Protocol(format!("Failed to read batch header: {e}")))?;
let checksum = match flag_buf[0] {
0 => None,
1 => {
let mut checksum = [0u8; 32];
stream
.read_exact(&mut checksum)
.await
.map_err(|e| ProtocolError::Protocol(format!("Failed to read batch header: {e}")))?;
Some(checksum)
}
other => {
return Err(ProtocolError::Protocol(format!(
"Invalid batch checksum flag: {other}"
)))
}
};
let mut delta = Delta::new(data_len as u64, 0);
delta.checksum = checksum;
data_lens.push(data_len);
recipes.push(crate::protocol::BatchFile {
file_id,
file_path,
delta,
});
}
for (recipe, data_len) in recipes.iter_mut().zip(data_lens) {
let mut data = vec![0u8; data_len];
stream
.read_exact(&mut data)
.await
.map_err(|e| ProtocolError::Protocol(format!("Failed to read batch data: {e}")))?;
recipe.delta.push_literal_owned(data);
}
return Ok(Frame::Batch { recipes });
}
other => {
return Err(ProtocolError::Protocol(format!(
"Unknown raw-layout frame tag: {other}"
)))
}
}
}
let size = (len & MAX_LEN) as usize;
let mut data = vec![0u8; size];
stream
.read_exact(&mut data)
.await
.map_err(|e| ProtocolError::Protocol(format!("Failed to read frame data: {e}")))?;
decode(len_buf, &data)
}
#[cfg(test)]
mod tests {
use super::*;
use crate::delta::Delta;
use crate::protocol::{BatchFile, BatchItem, BUILD_FINGERPRINT};
use crate::test_fuzz::FuzzRng;
fn small_frame() -> Frame {
Frame::Hello {
fingerprint: BUILD_FINGERPRINT.to_string(),
}
}
fn big_frame() -> Frame {
let mut delta = Delta::new(1024 * 1024, 0);
delta.push_literal(&vec![0x42u8; 1024 * 1024]);
Frame::Batch {
recipes: vec![BatchFile {
file_id: 1,
file_path: "a.txt".into(),
delta,
}],
}
}
#[test]
fn wire_roundtrip_uncompressed() {
let frame = small_frame();
let wire = encode(&frame, false, 0).unwrap();
let (prefix, payload) = wire.split_at(4);
assert_eq!(u32::from_be_bytes(prefix.try_into().unwrap()) >> 31, 0);
let decoded = decode(prefix.try_into().unwrap(), payload).unwrap();
assert_eq!(format!("{frame:?}"), format!("{decoded:?}"));
}
#[test]
fn wire_roundtrip_compressed() {
let frame = big_frame();
let wire = encode(&frame, true, 1024).unwrap();
let (prefix, payload) = wire.split_at(4);
assert_ne!(u32::from_be_bytes(prefix.try_into().unwrap()) >> 31, 0);
let decoded = decode(prefix.try_into().unwrap(), payload).unwrap();
match decoded {
Frame::Batch { recipes } => {
assert_eq!(recipes.len(), 1);
assert_eq!(recipes[0].file_path, "a.txt");
}
other => panic!("expected Batch, got {other:?}"),
}
}
#[test]
fn small_frames_bypass_compression() {
let frame = small_frame();
let wire = encode(&frame, true, 1024).unwrap();
let prefix = u32::from_be_bytes(wire[0..4].try_into().unwrap());
assert_eq!(prefix >> 31, 0);
}
#[test]
fn compression_shrinks_repetitive_payload() {
let frame = big_frame();
let plain = encode(&frame, false, 0).unwrap();
let compressed = encode(&frame, true, 1024).unwrap();
assert!(
compressed.len() < plain.len(),
"compression should shrink repetitive data ({} vs {})",
compressed.len(),
plain.len()
);
}
#[tokio::test]
async fn codec_works_over_any_byte_stream() {
use tokio::io::duplex;
let (mut a, mut b) = duplex(1024 * 1024);
let frame = big_frame();
let writer = tokio::spawn(async move {
send_frame_compressed(&mut a, &frame, true, 1024)
.await
.unwrap();
});
let got = receive_frame(&mut b).await.unwrap();
writer.await.unwrap();
match got {
Frame::Batch { recipes } => {
assert_eq!(recipes.len(), 1);
assert_eq!(recipes[0].file_path, "a.txt");
}
other => panic!("expected Batch, got {other:?}"),
}
}
#[tokio::test]
async fn batch_raw_layout_roundtrip() {
use tokio::io::duplex;
let (mut a, mut b) = duplex(4 * 1024 * 1024);
let items = vec![
BatchItem {
file_id: 7,
file_path: "sub/a.txt".into(),
data: vec![0x11u8; 4096],
checksum: Some([0xAA; 32]),
},
BatchItem {
file_id: 8,
file_path: "b.bin".into(),
data: vec![0x22u8; 8192],
checksum: None,
},
];
let sent = items;
let writer = tokio::spawn(async move {
send_batch_raw(&mut a, &sent).await.unwrap();
});
let got = receive_frame(&mut b).await.unwrap();
writer.await.unwrap();
match got {
Frame::Batch { recipes } => {
assert_eq!(recipes.len(), 2);
assert_eq!(recipes[0].file_id, 7);
assert_eq!(recipes[0].file_path, "sub/a.txt");
assert_eq!(recipes[0].delta.source_size, 4096);
assert_eq!(recipes[0].delta.checksum, Some([0xAA; 32]));
assert_eq!(
recipes[0].delta.ops,
vec![crate::delta::DeltaOp::Literal(vec![0x11u8; 4096])]
);
assert_eq!(recipes[1].file_id, 8);
assert_eq!(recipes[1].file_path, "b.bin");
assert_eq!(recipes[1].delta.checksum, None);
assert_eq!(
recipes[1].delta.ops,
vec![crate::delta::DeltaOp::Literal(vec![0x22u8; 8192])]
);
}
other => panic!("expected Batch, got {other:?}"),
}
}
#[test]
fn batch_raw_layout_rejects_unknown_tag() {
let len = 1u32 | CHUNK_FLAG;
let mut wire = Vec::new();
wire.extend_from_slice(&len.to_be_bytes());
wire.push(0x7F);
let mut cursor = std::io::Cursor::new(wire);
let err = tokio::runtime::Builder::new_current_thread()
.enable_io()
.build()
.unwrap()
.block_on(async { receive_frame(&mut cursor).await })
.unwrap_err();
assert!(err.to_string().contains("Unknown raw-layout frame tag"));
}
#[test]
fn compressed_frame_size_claim_is_capped() {
let mut payload = vec![0u8; 16];
payload[0..4].copy_from_slice(&MAX_LEN.wrapping_add(1).to_le_bytes());
let prefix = (1u32 << 31).to_be_bytes();
let err = decode(prefix, &payload).unwrap_err();
assert!(err.to_string().contains("above the"), "{err}");
let mut payload = vec![0u8; 16];
payload[0..4].copy_from_slice(&MAX_LEN.to_le_bytes());
assert!(decode(prefix, &payload).is_err());
}
#[test]
fn fuzz_decode_never_panics_and_roundtrips() {
let mut rng = FuzzRng::new(0xD3C0_DE5E_CAFE_F00D);
for _ in 0..2_000 {
let mut wire = rng.bytes(4096);
if wire.len() < 4 {
continue;
}
let prefix: [u8; 4] = wire[0..4].try_into().unwrap();
if u32::from_be_bytes(prefix) & (1 << 31) != 0 && wire.len() >= 8 {
let claim = u32::try_from(rng.below(16 * 1024)).unwrap_or(u32::MAX);
wire[4..8].copy_from_slice(&claim.to_le_bytes());
}
let payload = &wire[4..];
if let Ok(frame) = decode(prefix, payload) {
let wire2 = encode(&frame, false, 0).unwrap();
let (prefix2, payload2) = wire2.split_at(4);
let again = decode(prefix2.try_into().unwrap(), payload2).unwrap();
assert_eq!(format!("{frame:?}"), format!("{again:?}"));
}
}
}
#[test]
fn fuzz_receive_frame_never_panics() {
let rt = tokio::runtime::Builder::new_current_thread()
.enable_io()
.build()
.unwrap();
let mut rng = FuzzRng::new(0xDEAD_BEEF_0BAD_F00D);
for _ in 0..1_000 {
let mut wire = rng.bytes(4096);
if wire.len() < 4 {
continue;
}
if u32::from_be_bytes(wire[0..4].try_into().unwrap()) & (1 << 31) != 0
&& wire.len() >= 8
{
let claim = u32::try_from(rng.below(16 * 1024)).unwrap_or(u32::MAX);
wire[4..8].copy_from_slice(&claim.to_le_bytes());
}
let mut cursor = std::io::Cursor::new(wire);
let _ = rt.block_on(async { receive_frame(&mut cursor).await });
}
}
#[tokio::test]
async fn chunk_frame_zero_copy_roundtrip() {
use tokio::io::duplex;
let (mut a, mut b) = duplex(2 * 1024 * 1024);
let data: Vec<u8> = (0..1024 * 1024)
.map(|i| u8::try_from(i % 251).unwrap())
.collect();
let sent_data = data.clone();
let writer = tokio::spawn(async move {
send_chunk_frame(&mut a, 42, &sent_data).await.unwrap();
});
let got = receive_frame(&mut b).await.unwrap();
writer.await.unwrap();
match got {
Frame::FileChunk { file_id, data: got } => {
assert_eq!(file_id, 42);
assert_eq!(got, data);
}
other => panic!("expected FileChunk, got {other:?}"),
}
}
}