use std::path::{Path, PathBuf};
use std::sync::Arc;
use std::time::Duration;
use serde::{Deserialize, Serialize};
use tokio::io::{AsyncSeekExt, AsyncWriteExt, SeekFrom};
use zenoh::qos::{CongestionControl, Priority, Reliability};
use zenoh_ext::{
AdvancedPublisherBuilderExt, AdvancedSubscriberBuilderExt, CacheConfig, HistoryConfig,
MissDetectionConfig, RecoveryConfig,
};
use crate::cancel::CancelToken;
use crate::chunk::TransferChunks;
use crate::client::Overwrite;
use crate::error::{BlobError, Result};
use crate::hash::Hash;
use crate::id::BlobId;
use crate::manifest::{BlobSpec, Manifest};
use crate::obs::{TransferStats, zdebug};
use crate::prefix::{QueryPrefix, ServePrefix};
use crate::progress::{Progress, ProgressSink};
use crate::verify;
pub fn fanout_key(prefix: &str, id: &str) -> String {
format!("{prefix}/{id}/fanout")
}
#[derive(Serialize, Deserialize)]
struct FanoutMessage {
version: u16,
frame: FanoutFrame,
}
#[derive(Serialize, Deserialize)]
enum FanoutFrame {
Manifest(Manifest),
Slice {
index: u32,
bao: Vec<u8>,
},
}
#[derive(Debug, Clone)]
pub struct FanoutConfig {
pub heartbeat: Duration,
pub stall_timeout: Duration,
pub overwrite: Overwrite,
pub max_early_bytes: usize,
pub max_blob_size: u64,
pub cache_samples: usize,
}
impl Default for FanoutConfig {
fn default() -> Self {
FanoutConfig {
heartbeat: Duration::from_millis(500),
stall_timeout: Duration::from_secs(30),
overwrite: Overwrite::default(),
max_early_bytes: 16 * 1024 * 1024,
max_blob_size: 1 << 40,
cache_samples: 4096,
}
}
}
const EARLY_SLICE_MAX_FRAMES: usize = 512;
#[derive(Debug)]
pub struct FanoutHandle {
stop: Arc<tokio::sync::Notify>,
join: tokio::task::JoinHandle<Result<()>>,
}
impl FanoutHandle {
pub async fn shutdown(self) -> Result<()> {
self.stop.notify_one();
self.join.await.map_err(BlobError::Task)?
}
}
pub async fn fanout_file(
session: &zenoh::Session,
prefix: &ServePrefix,
spec: BlobSpec,
path: impl Into<PathBuf>,
cfg: FanoutConfig,
) -> Result<(Manifest, FanoutHandle)> {
let path = path.into();
let hash_path = path.clone();
let (outboard, total_len) =
tokio::task::spawn_blocking(move || -> std::io::Result<(verify::MemOutboard, u64)> {
let file = std::fs::File::open(&hash_path)?;
let total_len = file.metadata()?.len();
Ok((verify::compute_outboard(file)?, total_len))
})
.await??;
let outboard = Arc::new(outboard);
let chunks = TransferChunks::new(spec.chunk_size, total_len)?;
let count = chunks.count();
let manifest = Manifest {
version: crate::wire::WIRE_VERSION,
id: BlobId::new(spec.id.clone())?,
filename: spec.filename,
total_len,
chunk_size: spec.chunk_size,
root: Hash::from(outboard.root),
created_ms: spec.created_ms,
ext: crate::wire::Ext::new(),
};
manifest.validate(u64::MAX)?;
let stop = Arc::new(tokio::sync::Notify::new());
let stop2 = stop.clone();
let key = fanout_key(prefix.as_str(), &manifest.id);
let task_manifest = manifest.clone();
let session = session.clone();
let join = tokio::spawn(async move {
let publisher = session
.declare_publisher(key)
.congestion_control(CongestionControl::Block)
.reliability(Reliability::Reliable)
.priority(Priority::DataLow)
.cache(
CacheConfig::default()
.max_samples(cfg.cache_samples.max(1).min(count as usize + 1)),
)
.sample_miss_detection(MissDetectionConfig::default().heartbeat(cfg.heartbeat))
.publisher_detection()
.await
.map_err(BlobError::zenoh)?;
publisher
.put(crate::wire::encode(&FanoutMessage {
version: crate::wire::WIRE_VERSION,
frame: FanoutFrame::Manifest(task_manifest.clone()),
})?)
.encoding(&crate::wire::ENC_FANOUT)
.await
.map_err(BlobError::zenoh)?;
let mut reader = tokio::task::spawn_blocking(move || std::fs::File::open(&path)).await??;
for index in 0..count {
let ob = outboard.clone();
let byte_range = chunks.byte_range(index);
let (r, slice) = tokio::task::spawn_blocking(
move || -> (std::fs::File, std::io::Result<Vec<u8>>) {
let slice =
verify::encode_slice(&reader, &*ob, verify::chunk_range(byte_range));
(reader, slice)
},
)
.await?;
reader = r;
publisher
.put(crate::wire::encode(&FanoutMessage {
version: crate::wire::WIRE_VERSION,
frame: FanoutFrame::Slice { index, bao: slice? },
})?)
.encoding(&crate::wire::ENC_FANOUT)
.await
.map_err(BlobError::zenoh)?;
}
zdebug!(id = %task_manifest.id, chunks = count, "fanout published; serving cache");
stop2.notified().await;
Ok(())
});
Ok((manifest, FanoutHandle { stop, join }))
}
#[allow(clippy::too_many_arguments)] pub async fn receive_fanout(
session: &zenoh::Session,
prefix: &QueryPrefix,
id: &str,
expected_root: Option<Hash>,
dest: &Path,
sink: &dyn ProgressSink,
cancel: &CancelToken,
cfg: FanoutConfig,
) -> Result<TransferStats> {
let started_at = tokio::time::Instant::now();
if cfg.overwrite == Overwrite::Refuse && tokio::fs::try_exists(dest).await? {
return Err(BlobError::DestinationExists(dest.to_path_buf()));
}
let subscriber = session
.declare_subscriber(fanout_key(prefix.as_str(), id))
.history(HistoryConfig::default().detect_late_publishers())
.recovery(RecoveryConfig::default().heartbeat())
.query_timeout(cfg.stall_timeout)
.await
.map_err(BlobError::zenoh)?;
let mut early: Vec<(u32, Vec<u8>)> = Vec::new();
let mut early_bytes = 0usize;
let mut deferred_reject: Option<BlobError> = None;
let last_progress = tokio::time::Instant::now();
let (m, chunks, root) = loop {
if last_progress.elapsed() > cfg.stall_timeout {
return Err(deferred_reject.unwrap_or(BlobError::Incomplete {
received: 0,
total: 0,
}));
}
if cancel.is_cancelled() {
sink.emit(Progress::Cancelled {
received: 0,
total: 0,
});
return Err(BlobError::Cancelled {
received: 0,
total: 0,
});
}
let waited = cancel
.until_cancelled(tokio::time::timeout(
cfg.stall_timeout,
subscriber.recv_async(),
))
.await;
let sample = match waited {
Some(Ok(Ok(sample))) => sample,
None => {
sink.emit(Progress::Cancelled {
received: 0,
total: 0,
});
return Err(BlobError::Cancelled {
received: 0,
total: 0,
});
}
_ => {
return Err(deferred_reject.unwrap_or(BlobError::Incomplete {
received: 0,
total: 0,
}));
}
};
if !crate::wire::ENC_FANOUT.matches(sample.encoding()) {
continue;
}
let Ok(msg) = crate::wire::decode::<FanoutMessage>(&sample.payload().to_bytes()) else {
continue;
};
if msg.version != crate::wire::WIRE_VERSION {
continue;
}
match msg.frame {
FanoutFrame::Manifest(m) => {
let verdict = m
.validate(cfg.max_blob_size)
.and_then(|()| {
if m.id != id {
return Err(BlobError::MalformedMessage(format!(
"fanout manifest id {:?} does not match {id:?}",
m.id
)));
}
if let Some(expected) = expected_root
&& m.root != expected
{
return Err(BlobError::RootMismatch {
expected,
actual: m.root,
});
}
if m.total_len == 0 && m.root != Hash::of(b"") {
return Err(BlobError::RootMismatch {
expected: Hash::of(b""),
actual: m.root,
});
}
Ok(())
})
.and_then(|()| m.chunks());
match verdict {
Ok(chunks) => {
let root: blake3::Hash = m.root.into();
break (m, chunks, root);
}
Err(e) => {
deferred_reject.get_or_insert(e);
continue; }
}
}
FanoutFrame::Slice { index, bao } => {
if early.len() < EARLY_SLICE_MAX_FRAMES
&& early_bytes + bao.len() <= cfg.max_early_bytes
{
early_bytes += bao.len();
early.push((index, bao));
}
}
}
};
let count = chunks.count();
sink.emit(Progress::Started {
total_len: m.total_len,
chunk_count: count,
});
if let Some(parent) = dest.parent()
&& !parent.as_os_str().is_empty()
{
tokio::fs::create_dir_all(parent).await?;
}
let part = {
let mut p = dest.as_os_str().to_os_string();
p.push(".part");
PathBuf::from(p)
};
{
let f = tokio::fs::File::create(&part).await?;
f.set_len(m.total_len).await?;
}
let mut file = tokio::fs::OpenOptions::new()
.write(true)
.open(&part)
.await?;
let mut present: Vec<bool> = vec![false; count as usize];
let mut received = 0u32;
let mut stats = TransferStats::default();
let result: Result<()> = async {
for (index, bao) in early.drain(..) {
apply_slice(
&m,
&chunks,
&root,
&mut file,
&mut present,
&mut received,
&mut stats,
sink,
index,
&bao,
)
.await?;
}
let mut last_progress = tokio::time::Instant::now();
let mut progressed = received;
while received < count {
if received > progressed {
progressed = received;
last_progress = tokio::time::Instant::now();
} else if last_progress.elapsed() > cfg.stall_timeout {
return Err(BlobError::Incomplete {
received,
total: count,
});
}
if cancel.is_cancelled() {
sink.emit(Progress::Cancelled {
received,
total: count,
});
return Err(BlobError::Cancelled {
received,
total: count,
});
}
let waited = cancel
.until_cancelled(tokio::time::timeout(
cfg.stall_timeout,
subscriber.recv_async(),
))
.await;
let sample = match waited {
Some(Ok(Ok(sample))) => sample,
None => {
sink.emit(Progress::Cancelled {
received,
total: count,
});
return Err(BlobError::Cancelled {
received,
total: count,
});
}
_ => {
return Err(BlobError::Incomplete {
received,
total: count,
});
}
};
if !crate::wire::ENC_FANOUT.matches(sample.encoding()) {
continue;
}
let Ok(msg) = crate::wire::decode::<FanoutMessage>(&sample.payload().to_bytes()) else {
continue;
};
if msg.version != crate::wire::WIRE_VERSION {
continue;
}
if let FanoutFrame::Slice { index, bao } = msg.frame {
apply_slice(
&m,
&chunks,
&root,
&mut file,
&mut present,
&mut received,
&mut stats,
sink,
index,
&bao,
)
.await?;
}
}
file.sync_data().await?;
Ok(())
}
.await;
drop(file);
if let Err(e) = result {
let _ = tokio::fs::remove_file(&part).await;
return Err(e);
}
if cfg.overwrite == Overwrite::Refuse && tokio::fs::try_exists(dest).await? {
return Err(BlobError::DestinationExists(dest.to_path_buf()));
}
tokio::fs::rename(&part, dest).await?;
sink.emit(Progress::Completed {
path: dest.to_path_buf(),
});
stats.elapsed = started_at.elapsed();
Ok(stats)
}
#[allow(clippy::too_many_arguments)]
async fn apply_slice(
m: &Manifest,
chunks: &TransferChunks,
root: &blake3::Hash,
file: &mut tokio::fs::File,
present: &mut [bool],
received: &mut u32,
stats: &mut TransferStats,
sink: &dyn ProgressSink,
index: u32,
bao: &[u8],
) -> Result<()> {
let count = chunks.count();
if index >= count || present[index as usize] {
return Ok(());
}
let mut leaves: Vec<(u64, Vec<u8>)> = Vec::new();
let decoded = verify::decode_slice(
root,
m.total_len,
verify::chunk_range(chunks.byte_range(index)),
bao,
|off, data| {
leaves.push((off, data.to_vec()));
Ok(())
},
);
if decoded.is_err() {
stats.rejected += 1;
return Ok(());
}
for (off, data) in leaves {
file.seek(SeekFrom::Start(off)).await?;
file.write_all(&data).await?;
}
present[index as usize] = true;
*received += 1;
stats.chunks_fetched += 1;
stats.bytes_fetched += chunks.len_of(index) as u64;
sink.emit(Progress::Chunk {
index,
received: *received,
total: count,
bytes_received: stats.bytes_fetched,
});
Ok(())
}