use bytes::Bytes;
use serde::{Deserialize, Deserializer, Serialize, Serializer};
use thiserror::Error;
pub const BULK_PROTOCOL_VERSION: u8 = 8;
pub const BULK_FORMAT_RAW_V1: u8 = 1;
pub const BULK_HEADER_SIZE: usize = 12;
pub const DEFAULT_BULK_RECORD_PAYLOAD: u32 = 256 * 1024;
pub const DEFAULT_FILESYSTEM_BULK_RECORD_PAYLOAD: u32 = 3 * 1024 * 1024;
pub const MIN_BULK_RECORD_PAYLOAD: u32 = 16 * 1024;
pub const MAX_BULK_RECORD_PAYLOAD: u32 = DEFAULT_FILESYSTEM_BULK_RECORD_PAYLOAD;
pub const DEFAULT_BULK_WINDOW: u64 = 8 * 1024 * 1024;
pub const MAX_BULK_WINDOW: u64 = 32 * 1024 * 1024;
pub const BULK_FLOW_MASK_HOST_TO_GUEST: u8 = 0b01;
pub const BULK_FLOW_MASK_GUEST_TO_HOST: u8 = 0b10;
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
#[repr(u8)]
pub enum BulkKind {
Filesystem = 1,
Tcp = 2,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
#[repr(u8)]
pub enum BulkFlow {
HostToGuest = 1,
GuestToHost = 2,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct BulkRecord {
pub id: u32,
pub kind: BulkKind,
pub flow: BulkFlow,
pub offset: u64,
pub payload: Bytes,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
pub struct BulkOffer {
pub format: u8,
pub max_record_payload: u32,
pub guest_to_host_credit_limit: u64,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
pub struct BulkAccepted {
pub kind: BulkKind,
pub flows: u8,
pub format: u8,
pub max_record_payload: u32,
pub host_to_guest_credit_limit: u64,
pub guest_to_host_credit_limit: u64,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
pub struct BulkCredit {
pub kind: BulkKind,
pub flow: BulkFlow,
pub consumed_offset: u64,
pub credit_limit: u64,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
pub struct BulkFinish {
pub kind: BulkKind,
pub flow: BulkFlow,
pub final_offset: u64,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
#[repr(u8)]
pub enum BulkCancelReason {
CallerCancelled = 1,
DestinationIo = 2,
ResourceLimit = 3,
TransportFailure = 4,
ProtocolState = 5,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct BulkCancel {
pub kind: BulkKind,
pub reason: BulkCancelReason,
pub message: String,
}
#[derive(Debug, Clone)]
pub struct BulkSendState {
kind: BulkKind,
flow: BulkFlow,
max_record_payload: u32,
next_offset: u64,
consumed_offset: u64,
credit_limit: u64,
finished: bool,
}
#[derive(Debug, Clone)]
pub struct BulkReceiveState {
kind: BulkKind,
flow: BulkFlow,
max_record_payload: u32,
window: u64,
next_expected_offset: u64,
consumed_offset: u64,
credit_limit: u64,
finished: bool,
}
#[derive(Debug, Clone, PartialEq, Eq, Error)]
pub enum BulkStateError {
#[error("unsupported bulk format {0}")]
UnsupportedFormat(u8),
#[error("invalid maximum bulk record payload {0}")]
InvalidRecordLimit(u32),
#[error("invalid bulk record payload length {length} (max {max})")]
InvalidPayloadLength {
length: usize,
max: u32,
},
#[error("bulk kind or flow does not match the correlation")]
FlowMismatch,
#[error("bulk offset overflow")]
OffsetOverflow,
#[error("bulk record end {end} exceeds credit limit {limit}")]
CreditExceeded {
end: u64,
limit: u64,
},
#[error("invalid bulk credit: {0}")]
InvalidCredit(String),
#[error("bulk record offset {actual} does not match expected {expected}")]
OffsetMismatch {
expected: u64,
actual: u64,
},
#[error("bulk finish offset {actual} does not match expected {expected}")]
FinishMismatch {
expected: u64,
actual: u64,
},
#[error("bulk flow is already finished")]
AlreadyFinished,
}
impl BulkKind {
pub fn from_wire(value: u8) -> Option<Self> {
match value {
1 => Some(Self::Filesystem),
2 => Some(Self::Tcp),
_ => None,
}
}
}
impl BulkFlow {
pub fn from_wire(value: u8) -> Option<Self> {
match value {
1 => Some(Self::HostToGuest),
2 => Some(Self::GuestToHost),
_ => None,
}
}
pub fn mask(self) -> u8 {
match self {
Self::HostToGuest => BULK_FLOW_MASK_HOST_TO_GUEST,
Self::GuestToHost => BULK_FLOW_MASK_GUEST_TO_HOST,
}
}
}
impl BulkOffer {
pub fn filesystem_read() -> Self {
Self::new(DEFAULT_FILESYSTEM_BULK_RECORD_PAYLOAD, DEFAULT_BULK_WINDOW)
}
pub fn filesystem_write() -> Self {
Self::new(DEFAULT_FILESYSTEM_BULK_RECORD_PAYLOAD, 0)
}
pub fn tcp() -> Self {
Self::new(DEFAULT_BULK_RECORD_PAYLOAD, DEFAULT_BULK_WINDOW)
}
pub fn validate(self) -> Result<Self, BulkStateError> {
validate_record_limit(self.max_record_payload)?;
if self.format != BULK_FORMAT_RAW_V1 {
return Err(BulkStateError::UnsupportedFormat(self.format));
}
if self.guest_to_host_credit_limit > MAX_BULK_WINDOW {
return Err(BulkStateError::InvalidCredit(format!(
"initial guest-to-host limit {} exceeds {}",
self.guest_to_host_credit_limit, MAX_BULK_WINDOW
)));
}
Ok(self)
}
fn new(max_record_payload: u32, guest_to_host_credit_limit: u64) -> Self {
Self {
format: BULK_FORMAT_RAW_V1,
max_record_payload,
guest_to_host_credit_limit,
}
}
}
impl BulkAccepted {
pub fn validate_against(
self,
offer: BulkOffer,
kind: BulkKind,
flows: u8,
) -> Result<Self, BulkStateError> {
let offer = offer.validate()?;
validate_record_limit(self.max_record_payload)?;
if self.kind != kind || self.flows != flows || self.flows & !0b11 != 0 {
return Err(BulkStateError::FlowMismatch);
}
if self.format != BULK_FORMAT_RAW_V1 {
return Err(BulkStateError::UnsupportedFormat(self.format));
}
if self.max_record_payload > offer.max_record_payload {
return Err(BulkStateError::InvalidRecordLimit(self.max_record_payload));
}
if self.guest_to_host_credit_limit != offer.guest_to_host_credit_limit {
return Err(BulkStateError::InvalidCredit(
"guest-to-host grant was not echoed exactly".into(),
));
}
if self.host_to_guest_credit_limit > MAX_BULK_WINDOW {
return Err(BulkStateError::InvalidCredit(
"host-to-guest grant exceeds the generation-8 window".into(),
));
}
let host_to_guest = self.flows & BULK_FLOW_MASK_HOST_TO_GUEST != 0;
let guest_to_host = self.flows & BULK_FLOW_MASK_GUEST_TO_HOST != 0;
if host_to_guest != (self.host_to_guest_credit_limit != 0) {
return Err(BulkStateError::InvalidCredit(
"host-to-guest credit must be nonzero exactly when that flow is enabled".into(),
));
}
if guest_to_host != (self.guest_to_host_credit_limit != 0) {
return Err(BulkStateError::InvalidCredit(
"guest-to-host credit must be nonzero exactly when that flow is enabled".into(),
));
}
Ok(self)
}
}
impl BulkSendState {
pub fn new(
kind: BulkKind,
flow: BulkFlow,
max_record_payload: u32,
credit_limit: u64,
) -> Result<Self, BulkStateError> {
validate_record_limit(max_record_payload)?;
if credit_limit > MAX_BULK_WINDOW {
return Err(BulkStateError::InvalidCredit(
"initial credit exceeds the generation-8 window".into(),
));
}
Ok(Self {
kind,
flow,
max_record_payload,
next_offset: 0,
consumed_offset: 0,
credit_limit,
finished: false,
})
}
pub fn next_offset(&self) -> u64 {
self.next_offset
}
pub fn credit_limit(&self) -> u64 {
self.credit_limit
}
pub fn max_record_payload(&self) -> u32 {
self.max_record_payload
}
pub fn available_credit(&self) -> u64 {
self.credit_limit.saturating_sub(self.next_offset)
}
pub fn admit(&mut self, payload_len: usize) -> Result<u64, BulkStateError> {
if self.finished {
return Err(BulkStateError::AlreadyFinished);
}
validate_payload_len(payload_len, self.max_record_payload)?;
let end = self
.next_offset
.checked_add(payload_len as u64)
.ok_or(BulkStateError::OffsetOverflow)?;
if end > self.credit_limit {
return Err(BulkStateError::CreditExceeded {
end,
limit: self.credit_limit,
});
}
let offset = self.next_offset;
self.next_offset = end;
Ok(offset)
}
pub fn apply_credit(&mut self, credit: BulkCredit) -> Result<bool, BulkStateError> {
if credit.kind != self.kind || credit.flow != self.flow {
return Err(BulkStateError::FlowMismatch);
}
if credit.consumed_offset > self.next_offset {
return Err(BulkStateError::InvalidCredit(
"peer consumed bytes the sender has not admitted".into(),
));
}
if credit.credit_limit < credit.consumed_offset
|| credit.credit_limit - credit.consumed_offset > MAX_BULK_WINDOW
{
return Err(BulkStateError::InvalidCredit(
"credit limit is outside the allowed absolute window".into(),
));
}
if credit.consumed_offset <= self.consumed_offset
&& credit.credit_limit <= self.credit_limit
{
return Ok(false);
}
if credit.consumed_offset < self.consumed_offset
|| credit.credit_limit < self.credit_limit
|| credit.credit_limit < self.next_offset
{
return Err(BulkStateError::InvalidCredit(
"credit fields advanced inconsistently".into(),
));
}
self.consumed_offset = credit.consumed_offset;
self.credit_limit = credit.credit_limit;
Ok(true)
}
pub fn finish(&mut self) -> Result<BulkFinish, BulkStateError> {
if self.finished {
return Err(BulkStateError::AlreadyFinished);
}
self.finished = true;
Ok(BulkFinish {
kind: self.kind,
flow: self.flow,
final_offset: self.next_offset,
})
}
}
impl BulkReceiveState {
pub fn new(
kind: BulkKind,
flow: BulkFlow,
max_record_payload: u32,
credit_limit: u64,
window: u64,
) -> Result<Self, BulkStateError> {
validate_record_limit(max_record_payload)?;
if window == 0 || window > MAX_BULK_WINDOW || credit_limit > window {
return Err(BulkStateError::InvalidCredit(
"invalid initial receive window".into(),
));
}
Ok(Self {
kind,
flow,
max_record_payload,
window,
next_expected_offset: 0,
consumed_offset: 0,
credit_limit,
finished: false,
})
}
pub fn next_expected_offset(&self) -> u64 {
self.next_expected_offset
}
pub fn credit_limit(&self) -> u64 {
self.credit_limit
}
pub fn accept_record(&mut self, record: &BulkRecord) -> Result<u64, BulkStateError> {
if self.finished {
return Err(BulkStateError::AlreadyFinished);
}
if record.kind != self.kind || record.flow != self.flow {
return Err(BulkStateError::FlowMismatch);
}
validate_payload_len(record.payload.len(), self.max_record_payload)?;
if record.offset != self.next_expected_offset {
return Err(BulkStateError::OffsetMismatch {
expected: self.next_expected_offset,
actual: record.offset,
});
}
let end = record
.offset
.checked_add(record.payload.len() as u64)
.ok_or(BulkStateError::OffsetOverflow)?;
if end > self.credit_limit {
return Err(BulkStateError::CreditExceeded {
end,
limit: self.credit_limit,
});
}
self.next_expected_offset = end;
Ok(end)
}
pub fn consume(&mut self, consumed_offset: u64) -> Result<Option<BulkCredit>, BulkStateError> {
if consumed_offset < self.consumed_offset || consumed_offset > self.next_expected_offset {
return Err(BulkStateError::InvalidCredit(
"consumed offset is outside admitted bytes".into(),
));
}
self.consumed_offset = consumed_offset;
if self.credit_limit - self.consumed_offset > self.window / 2 {
return Ok(None);
}
let next_limit = self
.consumed_offset
.checked_add(self.window)
.ok_or(BulkStateError::OffsetOverflow)?;
if next_limit <= self.credit_limit {
return Ok(None);
}
self.credit_limit = next_limit;
Ok(Some(BulkCredit {
kind: self.kind,
flow: self.flow,
consumed_offset: self.consumed_offset,
credit_limit: self.credit_limit,
}))
}
pub fn accept_finish(&mut self, finish: BulkFinish) -> Result<(), BulkStateError> {
if self.finished {
return Err(BulkStateError::AlreadyFinished);
}
if finish.kind != self.kind || finish.flow != self.flow {
return Err(BulkStateError::FlowMismatch);
}
if finish.final_offset != self.next_expected_offset {
return Err(BulkStateError::FinishMismatch {
expected: self.next_expected_offset,
actual: finish.final_offset,
});
}
self.finished = true;
Ok(())
}
}
macro_rules! impl_wire_enum {
($type:ty, $parse:path) => {
impl Serialize for $type {
fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
where
S: Serializer,
{
serializer.serialize_u8(*self as u8)
}
}
impl<'de> Deserialize<'de> for $type {
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
where
D: Deserializer<'de>,
{
let value = u8::deserialize(deserializer)?;
$parse(value).ok_or_else(|| serde::de::Error::custom("unknown bulk enum value"))
}
}
};
}
impl_wire_enum!(BulkKind, BulkKind::from_wire);
impl_wire_enum!(BulkFlow, BulkFlow::from_wire);
impl Serialize for BulkCancelReason {
fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
where
S: Serializer,
{
serializer.serialize_u8(*self as u8)
}
}
impl<'de> Deserialize<'de> for BulkCancelReason {
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
where
D: Deserializer<'de>,
{
let value = u8::deserialize(deserializer)?;
match value {
1 => Ok(Self::CallerCancelled),
2 => Ok(Self::DestinationIo),
3 => Ok(Self::ResourceLimit),
4 => Ok(Self::TransportFailure),
5 => Ok(Self::ProtocolState),
_ => Err(serde::de::Error::custom("unknown bulk cancel reason")),
}
}
}
fn validate_record_limit(limit: u32) -> Result<(), BulkStateError> {
if !(MIN_BULK_RECORD_PAYLOAD..=MAX_BULK_RECORD_PAYLOAD).contains(&limit) {
return Err(BulkStateError::InvalidRecordLimit(limit));
}
Ok(())
}
fn validate_payload_len(length: usize, max: u32) -> Result<(), BulkStateError> {
if length == 0 || length > max as usize {
return Err(BulkStateError::InvalidPayloadLength { length, max });
}
Ok(())
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn default_offers_keep_filesystem_throughput_and_tcp_latency_granularity() {
assert_eq!(
BulkOffer::filesystem_read().max_record_payload,
DEFAULT_FILESYSTEM_BULK_RECORD_PAYLOAD
);
assert_eq!(
BulkOffer::filesystem_write().max_record_payload,
DEFAULT_FILESYSTEM_BULK_RECORD_PAYLOAD
);
assert_eq!(
BulkOffer::tcp().max_record_payload,
DEFAULT_BULK_RECORD_PAYLOAD
);
}
#[test]
fn new_filesystem_offer_accepts_an_older_peers_smaller_limit() {
let offer = BulkOffer::filesystem_write();
let accepted = BulkAccepted {
kind: BulkKind::Filesystem,
flows: BULK_FLOW_MASK_HOST_TO_GUEST,
format: BULK_FORMAT_RAW_V1,
max_record_payload: DEFAULT_BULK_RECORD_PAYLOAD,
host_to_guest_credit_limit: DEFAULT_BULK_WINDOW,
guest_to_host_credit_limit: 0,
};
let negotiated = accepted
.validate_against(offer, BulkKind::Filesystem, BULK_FLOW_MASK_HOST_TO_GUEST)
.unwrap();
assert_eq!(negotiated.max_record_payload, DEFAULT_BULK_RECORD_PAYLOAD);
}
#[test]
fn sender_stops_at_exact_credit_and_accepts_absolute_replenishment() {
let mut sender = BulkSendState::new(
BulkKind::Filesystem,
BulkFlow::GuestToHost,
MIN_BULK_RECORD_PAYLOAD,
MIN_BULK_RECORD_PAYLOAD as u64,
)
.unwrap();
assert_eq!(sender.admit(MIN_BULK_RECORD_PAYLOAD as usize).unwrap(), 0);
assert!(matches!(
sender.admit(1),
Err(BulkStateError::CreditExceeded { .. })
));
assert!(
sender
.apply_credit(BulkCredit {
kind: BulkKind::Filesystem,
flow: BulkFlow::GuestToHost,
consumed_offset: MIN_BULK_RECORD_PAYLOAD as u64,
credit_limit: 2 * MIN_BULK_RECORD_PAYLOAD as u64,
})
.unwrap()
);
assert_eq!(
sender.admit(MIN_BULK_RECORD_PAYLOAD as usize).unwrap(),
MIN_BULK_RECORD_PAYLOAD as u64
);
}
#[test]
fn receiver_rejects_gap_and_replenishes_at_half_window() {
let window = 2 * MIN_BULK_RECORD_PAYLOAD as u64;
let mut receiver = BulkReceiveState::new(
BulkKind::Tcp,
BulkFlow::HostToGuest,
MIN_BULK_RECORD_PAYLOAD,
window,
window,
)
.unwrap();
let gap = BulkRecord {
id: 4,
kind: BulkKind::Tcp,
flow: BulkFlow::HostToGuest,
offset: 1,
payload: Bytes::from_static(b"x"),
};
assert!(matches!(
receiver.accept_record(&gap),
Err(BulkStateError::OffsetMismatch { .. })
));
let record = BulkRecord {
offset: 0,
payload: Bytes::from(vec![0; MIN_BULK_RECORD_PAYLOAD as usize]),
..gap
};
let end = receiver.accept_record(&record).unwrap();
let credit = receiver.consume(end).unwrap().unwrap();
assert_eq!(credit.consumed_offset, MIN_BULK_RECORD_PAYLOAD as u64);
assert_eq!(credit.credit_limit, 3 * MIN_BULK_RECORD_PAYLOAD as u64);
}
}