use crate::format::{
check_name, FileBranch, FolderBranch, Reader, MAX_CHUNK_BYTES, MAX_NAME_BYTES, RECORD_MARK,
};
use crate::hash::{Hash, Root};
use crate::tree::MAX_LIST_ITEMS;
use crate::{Error, Result};
use std::io::{ErrorKind, Read, Write};
pub const FRAME_RECORD: u8 = RECORD_MARK;
pub const FRAME_FILE_LIST: u8 = 0x21;
pub const FRAME_FOLDER_LIST: u8 = 0x22;
pub const FRAME_FILE_BRANCH: u8 = 0x23;
pub const FRAME_FOLDER_BRANCH: u8 = 0x24;
pub const FRAME_CHUNK: u8 = 0x25;
pub const FRAME_HAVE: u8 = 0x26;
pub const FRAME_REFUSED: u8 = 0x27;
const ASK_RECORD: u8 = 0x30;
const ASK_FOLDER: u8 = 0x31;
const ASK_FOLDER_BRANCH: u8 = 0x32;
const ASK_FOLDER_NAME: u8 = 0x33;
const ASK_FILE: u8 = 0x34;
const ASK_FILE_BRANCH: u8 = 0x35;
const ASK_FILE_BYTES: u8 = 0x36;
const ASK_CHUNKS: u8 = 0x37;
const ASK_CHUNK: u8 = 0x38;
const ASK_HAVE: u8 = 0x39;
pub const BUNDLE_MAGIC: [u8; 8] = *b"SCIONIC\x01";
pub const MAX_REQUEST_BYTES: usize = 35 + MAX_NAME_BYTES;
const READ_STEP_BYTES: usize = 1 << 20;
fn invalid(detail: &str) -> Error {
Error::InvalidData {
detail: detail.to_owned(),
}
}
#[derive(Clone, Debug, Eq, PartialEq)]
pub enum Request {
Record(Root),
Folder(Hash),
FolderBranch {
folder: Hash,
start: u64,
count: u32,
},
FolderName { folder: Hash, name: String },
File(Hash),
FileBranch { file: Hash, start: u64, count: u32 },
FileBytes {
file: Hash,
offset: u64,
length: u64,
},
Chunks { file: Hash, start: u64, count: u32 },
Chunk(Hash),
Have(Hash),
}
impl Request {
pub fn encode(&self) -> Vec<u8> {
let mut out = Vec::with_capacity(49);
let ranged = |out: &mut Vec<u8>, tag: u8, hash: &Hash, start: u64, count: u32| {
out.push(tag);
out.extend_from_slice(hash.as_bytes());
out.extend_from_slice(&start.to_be_bytes());
out.extend_from_slice(&count.to_be_bytes());
};
let one = |out: &mut Vec<u8>, tag: u8, hash: &Hash| {
out.push(tag);
out.extend_from_slice(hash.as_bytes());
};
match self {
Self::Record(root) => one(&mut out, ASK_RECORD, root.as_hash()),
Self::Folder(folder) => one(&mut out, ASK_FOLDER, folder),
Self::FolderBranch {
folder,
start,
count,
} => ranged(&mut out, ASK_FOLDER_BRANCH, folder, *start, *count),
Self::FolderName { folder, name } => {
one(&mut out, ASK_FOLDER_NAME, folder);
out.extend_from_slice(&(name.len() as u16).to_be_bytes());
out.extend_from_slice(name.as_bytes());
}
Self::File(file) => one(&mut out, ASK_FILE, file),
Self::FileBranch { file, start, count } => {
ranged(&mut out, ASK_FILE_BRANCH, file, *start, *count);
}
Self::FileBytes {
file,
offset,
length,
} => {
one(&mut out, ASK_FILE_BYTES, file);
out.extend_from_slice(&offset.to_be_bytes());
out.extend_from_slice(&length.to_be_bytes());
}
Self::Chunks { file, start, count } => {
ranged(&mut out, ASK_CHUNKS, file, *start, *count);
}
Self::Chunk(chunk) => one(&mut out, ASK_CHUNK, chunk),
Self::Have(file) => one(&mut out, ASK_HAVE, file),
}
out
}
pub fn decode(bytes: &[u8]) -> Result<Self> {
let mut reader = Reader::new(bytes, "a request");
let tag = reader.u8()?;
let hash = reader.hash()?;
let request = match tag {
ASK_RECORD => Self::Record(Root::from_hash(hash)),
ASK_FOLDER => Self::Folder(hash),
ASK_FILE => Self::File(hash),
ASK_CHUNK => Self::Chunk(hash),
ASK_HAVE => Self::Have(hash),
ASK_FOLDER_BRANCH | ASK_FILE_BRANCH | ASK_CHUNKS => {
let start = reader.u64()?;
let count = reader.u32()?;
if count == 0 || start >= MAX_LIST_ITEMS {
return Err(invalid("a request asks for nothing, or beyond any list"));
}
match tag {
ASK_FOLDER_BRANCH => Self::FolderBranch {
folder: hash,
start,
count,
},
ASK_FILE_BRANCH => Self::FileBranch {
file: hash,
start,
count,
},
_ => Self::Chunks {
file: hash,
start,
count,
},
}
}
ASK_FOLDER_NAME => {
let length = usize::from(reader.u16()?);
let name = std::str::from_utf8(reader.take(length)?)
.map_err(|_| invalid("a name is not UTF-8"))?;
check_name(name)?;
Self::FolderName {
folder: hash,
name: name.to_owned(),
}
}
ASK_FILE_BYTES => {
let offset = reader.u64()?;
let length = reader.u64()?;
if length == 0 || offset.checked_add(length).is_none() {
return Err(invalid("a request asks for no bytes, or beyond any file"));
}
Self::FileBytes {
file: hash,
offset,
length,
}
}
_ => return Err(invalid("a request of an unknown kind")),
};
reader.finish()?;
Ok(request)
}
}
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub enum Refusal {
NotHeld,
OutOfRange,
}
impl Refusal {
fn byte(self) -> u8 {
match self {
Self::NotHeld => 1,
Self::OutOfRange => 2,
}
}
}
#[derive(Clone, Debug, Eq, PartialEq)]
pub struct HaveMap {
count: u64,
bits: Vec<u8>,
}
impl HaveMap {
pub fn new(count: u64) -> Result<Self> {
if count > MAX_LIST_ITEMS {
return Err(invalid("a map for more chunks than a file can have"));
}
let length = count.div_ceil(8) as usize;
let mut bits = Vec::new();
bits.try_reserve_exact(length)
.map_err(|_| Error::AllocationFailed {
resource: "a map of held chunks",
requested: length as u64,
})?;
bits.resize(length, 0);
Ok(Self { count, bits })
}
pub fn count(&self) -> u64 {
self.count
}
pub fn get(&self, index: u64) -> bool {
index < self.count && self.bits[(index / 8) as usize] & (0x80 >> (index % 8)) != 0
}
pub fn set(&mut self, index: u64) -> Result<()> {
if index >= self.count {
return Err(invalid("a chunk beyond the end of the file"));
}
self.bits[(index / 8) as usize] |= 0x80 >> (index % 8);
Ok(())
}
pub fn held(&self) -> u64 {
self.bits
.iter()
.map(|byte| u64::from(byte.count_ones()))
.sum()
}
pub fn encode(&self) -> Vec<u8> {
let mut out = Vec::with_capacity(8 + self.bits.len());
out.extend_from_slice(&self.count.to_be_bytes());
out.extend_from_slice(&self.bits);
out
}
pub fn decode(bytes: &[u8]) -> Result<Self> {
let mut reader = Reader::new(bytes, "a map of held chunks");
let count = reader.u64()?;
if count > MAX_LIST_ITEMS || reader.remaining() as u64 != count.div_ceil(8) {
return Err(invalid("a map of held chunks does not fit its count"));
}
let bits = reader.take(reader.remaining())?.to_vec();
let spare = (8 - count % 8) % 8;
if bits
.last()
.is_some_and(|last| last & ((1_u8 << spare) - 1) != 0)
{
return Err(invalid("a map of held chunks has bits past its count"));
}
Ok(Self { count, bits })
}
}
#[derive(Clone, Debug, Eq, PartialEq)]
pub enum Frame<'a> {
Record(&'a [u8]),
FileList(&'a [u8]),
FolderList(&'a [u8]),
FileBranch(FileBranch),
FolderBranch(FolderBranch),
Chunk(&'a [u8]),
Have(HaveMap),
Refused(Refusal),
}
impl<'a> Frame<'a> {
pub fn decode(bytes: &'a [u8]) -> Result<Self> {
let (&tag, body) = bytes
.split_first()
.ok_or_else(|| invalid("a frame is empty"))?;
Ok(match tag {
FRAME_RECORD => Self::Record(bytes),
FRAME_FILE_LIST => Self::FileList(body),
FRAME_FOLDER_LIST => Self::FolderList(body),
FRAME_FILE_BRANCH => Self::FileBranch(FileBranch::decode(body)?),
FRAME_FOLDER_BRANCH => Self::FolderBranch(FolderBranch::decode(body)?),
FRAME_CHUNK => {
if body.is_empty() || body.len() > MAX_CHUNK_BYTES {
return Err(invalid("a chunk does not hold 1 byte to 8 MiB"));
}
Self::Chunk(body)
}
FRAME_HAVE => Self::Have(HaveMap::decode(body)?),
FRAME_REFUSED => match body {
[1] => Self::Refused(Refusal::NotHeld),
[2] => Self::Refused(Refusal::OutOfRange),
_ => return Err(invalid("a refusal of an unknown kind")),
},
_ => return Err(invalid("a frame of an unknown kind")),
})
}
}
fn framed(tag: u8, body: &[u8]) -> Vec<u8> {
let mut frame = Vec::with_capacity(1 + body.len());
frame.push(tag);
frame.extend_from_slice(body);
frame
}
pub fn file_list_frame(list: &[u8]) -> Vec<u8> {
framed(FRAME_FILE_LIST, list)
}
pub fn folder_list_frame(list: &[u8]) -> Vec<u8> {
framed(FRAME_FOLDER_LIST, list)
}
pub fn file_branch_frame(branch: &FileBranch) -> Vec<u8> {
framed(FRAME_FILE_BRANCH, &branch.encode())
}
pub fn folder_branch_frame(branch: &FolderBranch) -> Vec<u8> {
framed(FRAME_FOLDER_BRANCH, &branch.encode())
}
pub fn chunk_frame(chunk: &[u8]) -> Vec<u8> {
framed(FRAME_CHUNK, chunk)
}
pub fn have_frame(map: &HaveMap) -> Vec<u8> {
framed(FRAME_HAVE, &map.encode())
}
pub fn refused_frame(why: Refusal) -> Vec<u8> {
vec![FRAME_REFUSED, why.byte()]
}
pub fn write_frame(out: &mut dyn Write, frame: &[u8]) -> Result<()> {
let length = u32::try_from(frame.len()).map_err(|_| Error::LimitExceeded {
what: "a frame",
limit: u64::from(u32::MAX),
requested: frame.len() as u64,
})?;
let failed = |source| Error::Stream {
operation: "write a frame",
source,
};
out.write_all(&length.to_be_bytes()).map_err(failed)?;
out.write_all(frame).map_err(failed)
}
pub(crate) fn write_framed(out: &mut dyn Write, tag: u8, body: &[u8]) -> Result<()> {
let length = u32::try_from(body.len() + 1).map_err(|_| Error::LimitExceeded {
what: "a frame",
limit: u64::from(u32::MAX),
requested: body.len() as u64 + 1,
})?;
let failed = |source| Error::Stream {
operation: "write a frame",
source,
};
let mut head = [0; 5];
head[..4].copy_from_slice(&length.to_be_bytes());
head[4] = tag;
out.write_all(&head).map_err(failed)?;
out.write_all(body).map_err(failed)
}
pub(crate) enum Next {
Frame,
End,
Closed,
}
pub(crate) fn write_end(out: &mut dyn Write) -> Result<()> {
out.write_all(&[0; 4]).map_err(|source| Error::Stream {
operation: "write a frame",
source,
})
}
pub(crate) fn read_next(input: &mut dyn Read, limit: usize, frame: &mut Vec<u8>) -> Result<Next> {
frame.clear();
read_next_onto(input, limit, frame)
}
pub(crate) fn read_next_onto(
input: &mut dyn Read,
limit: usize,
frames: &mut Vec<u8>,
) -> Result<Next> {
let start = frames.len();
let next = read_onto(input, limit, frames, start);
if !matches!(next, Ok(Next::Frame)) {
frames.truncate(start);
}
next
}
fn read_onto(
input: &mut dyn Read,
limit: usize,
frames: &mut Vec<u8>,
start: usize,
) -> Result<Next> {
let failed = |source| Error::Stream {
operation: "read a frame",
source,
};
let cut_short = |error: std::io::Error| {
if error.kind() == ErrorKind::UnexpectedEof {
invalid("a stream ends inside a frame")
} else {
failed(error)
}
};
let mut header = [0_u8; 4];
let mut filled = 0;
while filled < header.len() {
match input.read(&mut header[filled..]) {
Ok(0) if filled == 0 => return Ok(Next::Closed),
Ok(0) => return Err(invalid("a stream ends inside a frame")),
Ok(count) => filled += count,
Err(error) if error.kind() == ErrorKind::Interrupted => {}
Err(error) => return Err(failed(error)),
}
}
let length = u32::from_be_bytes(header) as usize;
if length == 0 {
return Ok(Next::End);
}
if length > limit {
return Err(Error::LimitExceeded {
what: "a frame",
limit: limit as u64,
requested: length as u64,
});
}
let end = start.checked_add(length).ok_or(Error::AllocationFailed {
resource: "a frame",
requested: length as u64,
})?;
while frames.len() < end {
let held = frames.len();
let more = (end - held).min(READ_STEP_BYTES);
frames
.try_reserve(more)
.map_err(|_| Error::AllocationFailed {
resource: "a frame",
requested: length as u64,
})?;
frames.resize(held + more, 0);
input.read_exact(&mut frames[held..]).map_err(cut_short)?;
}
Ok(Next::Frame)
}
pub fn read_frame(input: &mut dyn Read, limit: usize) -> Result<Option<Vec<u8>>> {
let mut frame = Vec::new();
match read_next(input, limit, &mut frame)? {
Next::Frame => Ok(Some(frame)),
Next::End => Err(invalid("a frame is empty")),
Next::Closed => Ok(None),
}
}
#[cfg(test)]
mod tests {
use super::*;
fn every_request() -> Vec<Request> {
let hash = Hash::of(b"x");
vec![
Request::Record(Root::from_hash(hash)),
Request::Folder(hash),
Request::FolderBranch {
folder: hash,
start: 3,
count: 9,
},
Request::FolderName {
folder: hash,
name: "model.safetensors".to_owned(),
},
Request::File(hash),
Request::FileBranch {
file: hash,
start: 100,
count: 100,
},
Request::FileBytes {
file: hash,
offset: 500 << 30,
length: 1 << 30,
},
Request::Chunks {
file: hash,
start: 100,
count: 101,
},
Request::Chunk(hash),
Request::Have(hash),
]
}
#[test]
fn every_request_reads_back_and_is_read_in_one_way_only() {
for request in every_request() {
let bytes = request.encode();
assert_eq!(Request::decode(&bytes).unwrap(), request);
for cut in 0..bytes.len() {
assert!(
Request::decode(&bytes[..cut]).is_err(),
"{request:?} cut at {cut}"
);
}
assert!(Request::decode(&[bytes.clone(), vec![0]].concat()).is_err());
}
let mut tags: Vec<u8> = every_request()
.iter()
.map(|request| request.encode()[0])
.collect();
tags.dedup();
assert_eq!(tags, (0x30..=0x39).collect::<Vec<u8>>());
assert_eq!(every_request()[7].encode().len(), 45);
}
#[test]
fn a_request_for_nothing_or_for_a_name_no_folder_holds_is_refused() {
let hash = Hash::of(b"x");
let ranged = |tag: u8, start: u64, count: u32| {
let mut bytes = vec![tag];
bytes.extend_from_slice(hash.as_bytes());
bytes.extend_from_slice(&start.to_be_bytes());
bytes.extend_from_slice(&count.to_be_bytes());
Request::decode(&bytes)
};
for tag in [0x32, 0x35, 0x37] {
assert!(ranged(tag, 0, 1).is_ok());
assert!(ranged(tag, 0, 0).is_err());
assert!(ranged(tag, MAX_LIST_ITEMS, 1).is_err());
}
let named = |name: &[u8]| {
let mut bytes = vec![0x33];
bytes.extend_from_slice(hash.as_bytes());
bytes.extend_from_slice(&(name.len() as u16).to_be_bytes());
bytes.extend_from_slice(name);
Request::decode(&bytes)
};
assert!(named(b"a").is_ok());
for name in [&b""[..], b".", b"..", b"a/b", b"a\0", &[0xff]] {
assert!(named(name).is_err(), "{name:?}");
}
let stretch = |offset: u64, length: u64| {
let mut bytes = vec![0x36];
bytes.extend_from_slice(hash.as_bytes());
bytes.extend_from_slice(&offset.to_be_bytes());
bytes.extend_from_slice(&length.to_be_bytes());
Request::decode(&bytes)
};
assert!(stretch(0, 1).is_ok());
assert!(stretch(0, 0).is_err());
assert!(stretch(u64::MAX, 1).is_err());
let mut unknown = vec![0x3a];
unknown.extend_from_slice(hash.as_bytes());
assert!(Request::decode(&unknown).is_err());
}
#[test]
fn a_map_of_held_chunks_has_one_bit_a_chunk() {
let mut map = HaveMap::new(11).unwrap();
assert_eq!(map.encode().len(), 8 + 2);
for index in [0, 3, 10] {
map.set(index).unwrap();
}
assert!(map.set(11).is_err());
assert_eq!(map.held(), 3);
let held: Vec<u64> = (0..16).filter(|index| map.get(*index)).collect();
assert_eq!(held, [0, 3, 10]);
let bytes = map.encode();
assert_eq!(&bytes[8..], [0b1001_0000, 0b0010_0000]);
assert_eq!(HaveMap::decode(&bytes).unwrap(), map);
let mut past = bytes.clone();
past[9] |= 0b0001_0000;
assert!(HaveMap::decode(&past).is_err());
assert!(HaveMap::decode(&[bytes.clone(), vec![0]].concat()).is_err());
assert!(HaveMap::decode(&bytes[..9]).is_err());
let big = HaveMap::new(500_000).unwrap();
assert_eq!(big.encode().len(), 8 + 62_500);
assert_eq!(
HaveMap::decode(&HaveMap::new(0).unwrap().encode())
.unwrap()
.count(),
0
);
assert_eq!(
HaveMap::decode(&HaveMap::new(8).unwrap().encode())
.unwrap()
.count(),
8
);
}
#[test]
fn frames_are_told_apart_by_their_first_byte() {
assert_eq!(
Frame::decode(&chunk_frame(b"abc")).unwrap(),
Frame::Chunk(b"abc")
);
assert_eq!(
Frame::decode(&file_list_frame(&[7; 36])).unwrap(),
Frame::FileList(&[7; 36])
);
assert_eq!(
Frame::decode(&folder_list_frame(&[])).unwrap(),
Frame::FolderList(&[])
);
assert_eq!(
Frame::decode(&refused_frame(Refusal::NotHeld)).unwrap(),
Frame::Refused(Refusal::NotHeld)
);
assert_eq!(
Frame::decode(&refused_frame(Refusal::OutOfRange)).unwrap(),
Frame::Refused(Refusal::OutOfRange)
);
let map = HaveMap::new(3).unwrap();
assert_eq!(Frame::decode(&have_frame(&map)).unwrap(), Frame::Have(map));
assert_eq!(
Frame::decode(&[0x20, 1]).unwrap(),
Frame::Record(&[0x20, 1])
);
for bad in [
&[][..],
&[0x25],
&[0x27],
&[0x27, 0],
&[0x27, 3],
&[0x27, 1, 1],
&[0x28],
&[0x1f],
] {
assert!(Frame::decode(bad).is_err(), "{bad:?}");
}
let too_long = chunk_frame(&vec![0; MAX_CHUNK_BYTES + 1]);
assert!(Frame::decode(&too_long).is_err());
assert!(Frame::decode(&chunk_frame(&vec![0; MAX_CHUNK_BYTES])).is_ok());
}
#[test]
fn frames_go_through_a_stream_whole_or_not_at_all() {
let mut stream = Vec::new();
write_frame(&mut stream, b"one").unwrap();
write_frame(&mut stream, b"three").unwrap();
let mut input = stream.as_slice();
assert_eq!(read_frame(&mut input, 16).unwrap().unwrap(), b"one");
assert_eq!(read_frame(&mut input, 16).unwrap().unwrap(), b"three");
assert!(read_frame(&mut input, 16).unwrap().is_none());
let mut input = stream.as_slice();
assert!(matches!(
read_frame(&mut input, 2),
Err(Error::LimitExceeded { .. })
));
for cut in [1, 3, 5, 6] {
let mut input = &stream[..cut];
assert!(read_frame(&mut input, 16).is_err(), "cut at {cut}");
}
let mut empty = &[0_u8, 0, 0, 0][..];
assert!(read_frame(&mut empty, 16).is_err());
}
}