use serde::{Serialize, de::DeserializeOwned};
use crate::error::{BlobError, Result};
use crate::hash::Hash;
pub const WIRE_VERSION: u16 = 3;
pub struct WireTag {
text: &'static str,
encoding: std::sync::LazyLock<zenoh::bytes::Encoding, fn() -> zenoh::bytes::Encoding>,
}
impl WireTag {
#[must_use]
pub const fn as_str(&self) -> &'static str {
self.text
}
#[must_use]
pub fn encoding(&'static self) -> &'static zenoh::bytes::Encoding {
&self.encoding
}
#[must_use]
pub fn matches(&'static self, encoding: &zenoh::bytes::Encoding) -> bool {
encoding == self.encoding()
}
}
impl std::fmt::Debug for WireTag {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_tuple("WireTag").field(&self.text).finish()
}
}
impl std::fmt::Display for WireTag {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.write_str(self.text)
}
}
impl From<&'static WireTag> for zenoh::bytes::Encoding {
fn from(t: &'static WireTag) -> Self {
t.encoding().clone()
}
}
macro_rules! wire_tags {
($($(#[$m:meta])* $name:ident = $text:literal;)*) => {
$(
$(#[$m])*
pub static $name: WireTag = WireTag {
text: $text,
encoding: std::sync::LazyLock::new(|| zenoh::bytes::Encoding::from($text)),
};
)*
};
}
wire_tags! {
ENC_MANIFEST = "zblob/manifest;v=3";
ENC_SLICE = "zblob/bao4;v=3";
ENC_INDEX = "zblob/index;v=3";
ENC_INDEX_DESC = "zblob/indexdesc;v=3";
ENC_CHUNK = "zblob/chunk;v=3";
ENC_PUSH = "zblob/push;v=3";
ENC_AVAIL = "zblob/have;v=3";
ENC_HAVEBITS = "zblob/havebits;v=3";
ENC_TREEPROBE = "zblob/treeprobe;v=3";
ENC_FANOUT = "zblob/fanout;v=3";
}
#[derive(Debug, Clone, Default, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
#[serde(try_from = "Vec<(u16, Vec<u8>)>", into = "Vec<(u16, Vec<u8>)>")]
pub struct Ext(Vec<(u16, Vec<u8>)>);
impl Ext {
pub const MAX_FIELDS: usize = 16;
pub const MAX_VALUE_LEN: usize = 256;
#[must_use]
pub fn new() -> Self {
Ext(Vec::new())
}
pub fn from_fields(fields: Vec<(u16, Vec<u8>)>) -> Result<Self> {
if fields.len() > Self::MAX_FIELDS {
return Err(BlobError::MalformedMessage(format!(
"extension list has {} fields, over the cap of {}",
fields.len(),
Self::MAX_FIELDS
)));
}
if let Some((id, v)) = fields.iter().find(|(_, v)| v.len() > Self::MAX_VALUE_LEN) {
return Err(BlobError::MalformedMessage(format!(
"extension {id} carries {} bytes, over the cap of {}",
v.len(),
Self::MAX_VALUE_LEN
)));
}
Ok(Ext(fields))
}
#[must_use]
pub fn get(&self, id: u16) -> Option<&[u8]> {
self.0
.iter()
.find(|(k, _)| *k == id)
.map(|(_, v)| v.as_slice())
}
#[must_use]
pub fn get_u32(&self, id: u16) -> Option<u32> {
Some(u32::from_le_bytes(self.get(id)?.try_into().ok()?))
}
#[must_use]
pub fn get_u64(&self, id: u16) -> Option<u64> {
Some(u64::from_le_bytes(self.get(id)?.try_into().ok()?))
}
pub fn set_u32(&mut self, id: u16, value: u32) -> Result<()> {
self.set(id, value.to_le_bytes().to_vec())
}
pub fn set_u64(&mut self, id: u16, value: u64) -> Result<()> {
self.set(id, value.to_le_bytes().to_vec())
}
pub fn set(&mut self, id: u16, value: Vec<u8>) -> Result<()> {
let mut fields = std::mem::take(&mut self.0);
fields.retain(|(k, _)| *k != id);
fields.push((id, value));
*self = Ext::from_fields(fields)?;
Ok(())
}
#[must_use]
pub fn len(&self) -> usize {
self.0.len()
}
#[must_use]
pub fn is_empty(&self) -> bool {
self.0.is_empty()
}
pub fn iter(&self) -> impl Iterator<Item = (u16, &[u8])> {
self.0.iter().map(|(k, v)| (*k, v.as_slice()))
}
}
impl TryFrom<Vec<(u16, Vec<u8>)>> for Ext {
type Error = BlobError;
fn try_from(v: Vec<(u16, Vec<u8>)>) -> Result<Self> {
Ext::from_fields(v)
}
}
impl From<Ext> for Vec<(u16, Vec<u8>)> {
fn from(e: Ext) -> Self {
e.0
}
}
pub const EXT_MAX_CHUNKS_PER_QUERY: u16 = 1;
pub const EXT_MAX_BLOB_SIZE: u16 = 2;
#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
pub struct WantList {
pub version: u16,
pub hashes: Vec<Hash>,
}
pub const MAX_WANT_LIST: usize = 512;
impl WantList {
pub fn new(hashes: Vec<Hash>) -> Self {
WantList {
version: WIRE_VERSION,
hashes,
}
}
pub fn validate(&self, max: usize) -> Result<()> {
if self.version != WIRE_VERSION {
return Err(BlobError::UnsupportedVersion(self.version));
}
if self.hashes.is_empty() {
return Err(BlobError::MalformedMessage("empty want list".into()));
}
if self.hashes.len() > max {
return Err(BlobError::MalformedMessage(format!(
"want list of {} exceeds the limit of {max}",
self.hashes.len()
)));
}
let mut seen = std::collections::HashSet::with_capacity(self.hashes.len());
if let Some(dup) = self.hashes.iter().find(|h| !seen.insert(**h)) {
return Err(BlobError::MalformedMessage(format!(
"want list repeats {dup}"
)));
}
Ok(())
}
}
#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
pub struct HaveBits {
pub version: u16,
pub count: u32,
pub bits: Vec<u8>,
}
impl HaveBits {
pub fn from_presence(present: impl IntoIterator<Item = bool>) -> Self {
let mut count = 0u32;
let mut bits: Vec<u8> = Vec::new();
for (i, yes) in present.into_iter().enumerate() {
if i % 8 == 0 {
bits.push(0);
}
if yes {
let last = bits.len() - 1;
bits[last] |= 1 << (i % 8);
}
count += 1;
}
HaveBits {
version: WIRE_VERSION,
count,
bits,
}
}
pub fn is_set(&self, i: u32) -> bool {
i < self.count
&& (i / 8) < self.bits.len() as u32
&& self.bits[(i / 8) as usize] & (1 << (i % 8)) != 0
}
pub fn count_set(&self) -> u32 {
count_bits(&self.bits, self.count)
}
pub fn validate(&self, asked: usize) -> Result<()> {
if self.version != WIRE_VERSION {
return Err(BlobError::UnsupportedVersion(self.version));
}
if self.count as usize != asked {
return Err(BlobError::MalformedMessage(format!(
"probe answered {} entries for a want-list of {asked}",
self.count
)));
}
let want = self.count.div_ceil(8) as usize;
if self.bits.len() != want {
return Err(BlobError::MalformedMessage(format!(
"probe bitfield is {} bytes, expected {want}",
self.bits.len()
)));
}
Ok(())
}
}
#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
pub struct IndexDescriptor {
pub version: u16,
pub root: Hash,
pub algo: crate::hash::HashAlgo,
pub index_chunks: Vec<crate::tree::ChunkRef>,
pub index_len: u64,
pub ext: Ext,
}
impl IndexDescriptor {
pub fn validate(&self, max_index_bytes: usize) -> Result<()> {
if self.version != WIRE_VERSION {
return Err(BlobError::UnsupportedVersion(self.version));
}
if self.index_chunks.is_empty() {
return Err(BlobError::MalformedMessage(
"index descriptor has no chunks".into(),
));
}
if self.index_len > max_index_bytes as u64 {
return Err(BlobError::InvalidManifest(format!(
"index of {} bytes exceeds the {max_index_bytes} byte limit",
self.index_len
)));
}
let summed: u64 = self.index_chunks.iter().map(|c| c.len as u64).sum();
if summed != self.index_len {
return Err(BlobError::MalformedMessage(format!(
"index chunks total {summed} bytes, declared {}",
self.index_len
)));
}
Ok(())
}
}
#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
pub struct TreeProbe {
pub version: u16,
pub have_index: bool,
pub chunks_present: u32,
pub chunks_total: u32,
}
impl TreeProbe {
pub fn validate(&self) -> Result<()> {
if self.version != WIRE_VERSION {
return Err(BlobError::UnsupportedVersion(self.version));
}
if self.chunks_present > self.chunks_total {
return Err(BlobError::MalformedMessage(format!(
"probe claims {} of {} chunks",
self.chunks_present, self.chunks_total
)));
}
Ok(())
}
pub fn is_complete(&self) -> bool {
self.have_index && self.chunks_present == self.chunks_total
}
}
#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
pub struct Availability {
pub version: u16,
pub chunk_count: u32,
pub bits: Vec<u8>,
}
impl Availability {
pub fn full(chunk_count: u32) -> Self {
let mut bits = vec![0xffu8; chunk_count.div_ceil(8) as usize];
if let Some(last) = bits.last_mut()
&& !chunk_count.is_multiple_of(8)
{
*last = (1u8 << (chunk_count % 8)) - 1;
}
Availability {
version: WIRE_VERSION,
chunk_count,
bits,
}
}
pub fn is_set(&self, i: u32) -> bool {
i < self.chunk_count
&& (i / 8) < self.bits.len() as u32
&& self.bits[(i / 8) as usize] & (1 << (i % 8)) != 0
}
pub fn count(&self) -> u32 {
count_bits(&self.bits, self.chunk_count)
}
pub fn validate(&self, max_chunks: u32) -> Result<()> {
if self.version != WIRE_VERSION {
return Err(BlobError::UnsupportedVersion(self.version));
}
if self.chunk_count > max_chunks {
return Err(BlobError::MalformedMessage(format!(
"availability claims {} chunks, over the limit of {max_chunks}",
self.chunk_count
)));
}
let want = self.chunk_count.div_ceil(8) as usize;
if self.bits.len() != want {
return Err(BlobError::MalformedMessage(format!(
"availability bitfield is {} bytes, expected {want} for {} chunks",
self.bits.len(),
self.chunk_count
)));
}
Ok(())
}
}
fn count_bits(bits: &[u8], limit: u32) -> u32 {
let full = (limit / 8) as usize;
let mut n: u32 = bits.iter().take(full).map(|b| b.count_ones()).sum();
if let Some(last) = bits.get(full) {
let rem = limit % 8;
if rem > 0 {
n += (last & ((1u8 << rem) - 1)).count_ones();
}
}
n
}
pub fn encode<T: Serialize>(value: &T) -> Result<Vec<u8>> {
Ok(postcard::to_stdvec(value)?)
}
pub fn decode<T: DeserializeOwned>(data: &[u8]) -> Result<T> {
Ok(postcard::from_bytes(data)?)
}
#[cfg(test)]
mod properties {
use super::*;
use proptest::prelude::*;
proptest! {
#[test]
fn count_never_exceeds_the_declared_chunk_count(
chunk_count in 0u32..5000,
bits in prop::collection::vec(any::<u8>(), 0..800),
) {
let avail = Availability { version: WIRE_VERSION, chunk_count, bits };
prop_assert!(avail.count() <= chunk_count);
let by_hand = (0..chunk_count).filter(|i| avail.is_set(*i)).count() as u32;
prop_assert_eq!(avail.count(), by_hand);
}
#[test]
fn validate_accepts_only_well_sized_bitfields(
chunk_count in 0u32..5000,
len in 0usize..800,
max_chunks in 1u32..5000,
) {
let avail = Availability {
version: WIRE_VERSION,
chunk_count,
bits: vec![0u8; len],
};
prop_assert_eq!(
avail.validate(max_chunks).is_ok(),
len == chunk_count.div_ceil(8) as usize && chunk_count <= max_chunks
);
}
}
#[test]
fn counting_is_bounded_by_the_bitfield_not_the_claim() {
let hostile = Availability {
version: WIRE_VERSION,
chunk_count: u32::MAX,
bits: vec![0xff; 4],
};
let started = std::time::Instant::now();
assert_eq!(hostile.count(), 32, "only the bits present may count");
assert!(
started.elapsed() < std::time::Duration::from_millis(50),
"counting took {:?} — it is iterating the claim, not the data",
started.elapsed()
);
let hostile = HaveBits {
version: WIRE_VERSION,
count: u32::MAX,
bits: vec![0xff; 4],
};
let started = std::time::Instant::now();
assert_eq!(hostile.count_set(), 32);
assert!(started.elapsed() < std::time::Duration::from_millis(50));
}
proptest! {
#[test]
fn accepted_want_lists_are_bounded_and_distinct(
n in 0usize..40,
max in 1usize..32,
version in prop_oneof![Just(WIRE_VERSION), 0u16..8],
repeat in any::<bool>(),
) {
let mut hashes: Vec<Hash> = (0..n).map(|i| Hash::of(&[i as u8])).collect();
if repeat && !hashes.is_empty() {
hashes.push(hashes[0]);
}
let w = WantList { version, hashes };
if w.validate(max).is_ok() {
prop_assert_eq!(w.version, WIRE_VERSION);
prop_assert!(!w.hashes.is_empty());
prop_assert!(w.hashes.len() <= max);
let distinct: std::collections::HashSet<_> = w.hashes.iter().collect();
prop_assert_eq!(distinct.len(), w.hashes.len());
}
}
#[test]
fn accepted_have_bits_answer_exactly_what_was_asked(
asked in 0usize..64,
count in 0u32..64,
bit_len in 0usize..16,
version in prop_oneof![Just(WIRE_VERSION), 0u16..8],
) {
let h = HaveBits { version, count, bits: vec![0xAA; bit_len] };
if h.validate(asked).is_ok() {
prop_assert_eq!(h.version, WIRE_VERSION);
prop_assert_eq!(h.count as usize, asked);
prop_assert_eq!(h.bits.len(), h.count.div_ceil(8) as usize);
for i in 0..h.count {
let _ = h.is_set(i);
}
prop_assert!(h.count_set() <= h.count);
}
}
#[test]
fn accepted_index_descriptors_add_up(
lens in prop::collection::vec(0u32..4096, 0..8),
declared in 0u64..40_000,
max in 1usize..30_000,
version in prop_oneof![Just(WIRE_VERSION), 0u16..8],
) {
let d = IndexDescriptor {
version,
root: Hash::of(b"root"),
algo: crate::hash::HashAlgo::Blake3,
index_chunks: lens
.iter()
.enumerate()
.map(|(i, len)| crate::tree::ChunkRef {
hash: Hash::of(&[i as u8]),
len: *len,
})
.collect(),
index_len: declared,
ext: Ext::new(),
};
if d.validate(max).is_ok() {
prop_assert_eq!(d.version, WIRE_VERSION);
prop_assert!(!d.index_chunks.is_empty());
prop_assert!(d.index_len <= max as u64);
let summed: u64 = d.index_chunks.iter().map(|c| u64::from(c.len)).sum();
prop_assert_eq!(summed, d.index_len);
}
}
#[test]
fn accepted_tree_probes_cannot_claim_more_than_the_total(
present in 0u32..1000,
total in 0u32..1000,
have_index in any::<bool>(),
version in prop_oneof![Just(WIRE_VERSION), 0u16..8],
) {
let p = TreeProbe {
version,
have_index,
chunks_present: present,
chunks_total: total,
};
if p.validate().is_ok() {
prop_assert_eq!(p.version, WIRE_VERSION);
prop_assert!(p.chunks_present <= p.chunks_total);
}
}
}
#[test]
fn the_new_validators_refuse_what_they_claim_to() {
let h = Hash::of(b"a");
assert!(
WantList {
version: WIRE_VERSION + 1,
hashes: vec![h]
}
.validate(8)
.is_err()
);
assert!(WantList::new(vec![]).validate(8).is_err());
assert!(
WantList::new(vec![h, Hash::of(b"b"), Hash::of(b"c")])
.validate(2)
.is_err()
);
assert!(WantList::new(vec![h, h]).validate(8).is_err());
assert!(
WantList::new(vec![h]).validate(8).is_ok(),
"the honest case must pass"
);
assert!(
HaveBits {
version: WIRE_VERSION + 1,
count: 8,
bits: vec![0]
}
.validate(8)
.is_err()
);
assert!(
HaveBits {
version: WIRE_VERSION,
count: 9,
bits: vec![0, 0]
}
.validate(8)
.is_err()
);
assert!(
HaveBits {
version: WIRE_VERSION,
count: 8,
bits: vec![0, 0]
}
.validate(8)
.is_err()
);
assert!(
HaveBits {
version: WIRE_VERSION,
count: 8,
bits: vec![0]
}
.validate(8)
.is_ok()
);
let desc = |chunks: Vec<u32>, len: u64| IndexDescriptor {
version: WIRE_VERSION,
root: h,
algo: crate::hash::HashAlgo::Blake3,
index_chunks: chunks
.iter()
.enumerate()
.map(|(i, l)| crate::tree::ChunkRef {
hash: Hash::of(&[i as u8]),
len: *l,
})
.collect(),
index_len: len,
ext: Ext::new(),
};
assert!(desc(vec![], 0).validate(1000).is_err());
assert!(desc(vec![100, 100], 200).validate(100).is_err());
assert!(desc(vec![100, 100], 300).validate(1000).is_err());
assert!(desc(vec![100, 100], 200).validate(1000).is_ok());
assert!(
TreeProbe {
version: WIRE_VERSION + 1,
have_index: true,
chunks_present: 1,
chunks_total: 1
}
.validate()
.is_err()
);
assert!(
TreeProbe {
version: WIRE_VERSION,
have_index: true,
chunks_present: 2,
chunks_total: 1
}
.validate()
.is_err()
);
assert!(
TreeProbe {
version: WIRE_VERSION,
have_index: true,
chunks_present: 1,
chunks_total: 1
}
.validate()
.is_ok()
);
}
#[test]
fn full_availability_is_valid() {
for n in [0u32, 1, 7, 8, 9, 4095] {
let a = Availability::full(n);
a.validate(u32::MAX).expect("full() must validate");
assert_eq!(a.count(), n, "full() must report every chunk present");
}
}
}
#[cfg(test)]
mod tests {
use super::*;
#[derive(Debug, PartialEq, serde::Serialize, serde::Deserialize)]
struct Sample {
version: u16,
a: u32,
b: String,
}
#[test]
fn postcard_roundtrip() {
let s = Sample {
version: 2,
a: 7,
b: "hi".into(),
};
let bytes = encode(&s).unwrap();
assert_eq!(decode::<Sample>(&bytes).unwrap(), s);
}
#[test]
fn truncated_input_rejected() {
let s = Sample {
version: 2,
a: 7,
b: "hello world".into(),
};
let bytes = encode(&s).unwrap();
assert!(decode::<Sample>(&bytes[..bytes.len() - 3]).is_err());
}
}