use std::path::{Path, PathBuf};
use std::sync::Arc;
use std::time::Duration;
use serde::{Deserialize, Serialize};
use tokio::io::{AsyncSeekExt, AsyncWriteExt, SeekFrom};
use zenoh::qos::{CongestionControl, Priority, Reliability};
use zenoh_ext::{
AdvancedPublisherBuilderExt, AdvancedSubscriberBuilderExt, CacheConfig, HistoryConfig,
MissDetectionConfig, RecoveryConfig,
};
use crate::cancel::CancelToken;
use crate::chunk::TransferChunks;
use crate::client::Overwrite;
use crate::error::{BlobError, Result};
use crate::hash::Hash;
use crate::manifest::{BlobSpec, Manifest};
use crate::obs::{TransferStats, zdebug};
use crate::progress::{Progress, ProgressSink};
use crate::verify;
pub fn fanout_key(prefix: &str, id: &str) -> String {
format!("{prefix}/{id}/fanout")
}
#[derive(Serialize, Deserialize)]
enum FanoutFrame {
Manifest(Manifest),
Slice {
index: u32,
bao: Vec<u8>,
},
}
#[derive(Debug, Clone)]
pub struct FanoutConfig {
pub heartbeat: Duration,
pub stall_timeout: Duration,
pub overwrite: Overwrite,
}
impl Default for FanoutConfig {
fn default() -> Self {
FanoutConfig {
heartbeat: Duration::from_millis(500),
stall_timeout: Duration::from_secs(30),
overwrite: Overwrite::default(),
}
}
}
const EARLY_SLICE_MAX_FRAMES: usize = 512;
const EARLY_SLICE_MAX_BYTES: usize = 256 * 1024 * 1024;
pub struct FanoutHandle {
stop: Arc<tokio::sync::Notify>,
join: tokio::task::JoinHandle<Result<()>>,
}
impl FanoutHandle {
pub async fn shutdown(self) -> Result<()> {
self.stop.notify_one();
self.join
.await
.map_err(|e| BlobError::Protocol(format!("fanout task join: {e}")))?
}
}
pub async fn fanout_file(
session: Arc<zenoh::Session>,
prefix: &str,
spec: BlobSpec,
path: impl Into<PathBuf>,
cfg: FanoutConfig,
) -> Result<(Manifest, FanoutHandle)> {
let path = path.into();
let hash_path = path.clone();
let (outboard, total_len) =
tokio::task::spawn_blocking(move || -> std::io::Result<(verify::MemOutboard, u64)> {
let file = std::fs::File::open(&hash_path)?;
let total_len = file.metadata()?.len();
Ok((verify::compute_outboard(file)?, total_len))
})
.await
.map_err(|e| BlobError::Protocol(format!("hash task: {e}")))??;
let outboard = Arc::new(outboard);
let chunks = TransferChunks::new(spec.chunk_size, total_len)?;
let count = chunks.count();
let manifest = Manifest {
version: crate::wire::WIRE_VERSION,
id: spec.id.clone(),
filename: spec.filename,
total_len,
chunk_size: spec.chunk_size,
root: Hash::from(outboard.root),
created_ms: spec.created_ms,
};
manifest.validate(u64::MAX)?;
let stop = Arc::new(tokio::sync::Notify::new());
let stop2 = stop.clone();
let key = fanout_key(prefix, &manifest.id);
let task_manifest = manifest.clone();
let join = tokio::spawn(async move {
let publisher = session
.declare_publisher(key)
.congestion_control(CongestionControl::Block)
.reliability(Reliability::Reliable)
.priority(Priority::DataLow)
.cache(CacheConfig::default().max_samples(count as usize + 1))
.sample_miss_detection(MissDetectionConfig::default().heartbeat(cfg.heartbeat))
.publisher_detection()
.await
.map_err(BlobError::zenoh)?;
publisher
.put(crate::wire::encode(&(
crate::wire::WIRE_VERSION,
FanoutFrame::Manifest(task_manifest.clone()),
))?)
.await
.map_err(BlobError::zenoh)?;
let mut reader = tokio::task::spawn_blocking(move || std::fs::File::open(&path))
.await
.map_err(|e| BlobError::Protocol(format!("open task: {e}")))??;
for index in 0..count {
let ob = outboard.clone();
let byte_range = chunks.byte_range(index);
let (r, slice) = tokio::task::spawn_blocking(
move || -> (std::fs::File, std::io::Result<Vec<u8>>) {
let slice =
verify::encode_slice(&reader, &*ob, verify::chunk_range(byte_range));
(reader, slice)
},
)
.await
.map_err(|e| BlobError::Protocol(format!("encode task: {e}")))?;
reader = r;
publisher
.put(crate::wire::encode(&(
crate::wire::WIRE_VERSION,
FanoutFrame::Slice { index, bao: slice? },
))?)
.await
.map_err(BlobError::zenoh)?;
}
zdebug!(id = %task_manifest.id, chunks = count, "fanout published; serving cache");
stop2.notified().await;
Ok(())
});
Ok((manifest, FanoutHandle { stop, join }))
}
#[allow(clippy::too_many_arguments)] pub async fn receive_fanout(
session: Arc<zenoh::Session>,
prefix: &str,
id: &str,
expected_root: Option<Hash>,
dest: &Path,
sink: &dyn ProgressSink,
cancel: &CancelToken,
cfg: FanoutConfig,
) -> Result<TransferStats> {
let started_at = tokio::time::Instant::now();
let subscriber = session
.declare_subscriber(fanout_key(prefix, id))
.history(HistoryConfig::default().detect_late_publishers())
.recovery(RecoveryConfig::default().heartbeat())
.query_timeout(cfg.stall_timeout)
.await
.map_err(BlobError::zenoh)?;
let mut early: Vec<(u32, Vec<u8>)> = Vec::new();
let mut early_bytes = 0usize;
let mut deferred_reject: Option<BlobError> = None;
let (m, chunks, root) = loop {
if cancel.is_cancelled() {
sink.emit(Progress::Cancelled {
received: 0,
total: 0,
});
return Err(BlobError::Cancelled {
received: 0,
total: 0,
});
}
let sample = match tokio::time::timeout(cfg.stall_timeout, subscriber.recv_async()).await {
Ok(Ok(sample)) => sample,
_ => {
return Err(deferred_reject.unwrap_or(BlobError::Incomplete {
received: 0,
total: 0,
}));
}
};
let Ok((version, frame)) =
crate::wire::decode::<(u16, FanoutFrame)>(&sample.payload().to_bytes())
else {
continue;
};
if version != crate::wire::WIRE_VERSION {
continue;
}
match frame {
FanoutFrame::Manifest(m) => {
let verdict = m
.validate(1 << 40)
.and_then(|()| {
if m.id != id {
return Err(BlobError::Protocol(format!(
"fanout manifest id {:?} does not match {id:?}",
m.id
)));
}
if let Some(expected) = expected_root
&& m.root != expected
{
return Err(BlobError::RootMismatch {
expected,
actual: m.root,
});
}
if m.total_len == 0 && m.root != Hash::of(b"") {
return Err(BlobError::RootMismatch {
expected: Hash::of(b""),
actual: m.root,
});
}
Ok(())
})
.and_then(|()| m.chunks());
match verdict {
Ok(chunks) => {
let root: blake3::Hash = m.root.into();
break (m, chunks, root);
}
Err(e) => {
deferred_reject.get_or_insert(e);
continue; }
}
}
FanoutFrame::Slice { index, bao } => {
if early.len() < EARLY_SLICE_MAX_FRAMES
&& early_bytes + bao.len() <= EARLY_SLICE_MAX_BYTES
{
early_bytes += bao.len();
early.push((index, bao));
}
}
}
};
let count = chunks.count();
sink.emit(Progress::Started {
total_len: m.total_len,
chunk_count: count,
});
if let Some(parent) = dest.parent()
&& !parent.as_os_str().is_empty()
{
tokio::fs::create_dir_all(parent).await?;
}
let part = {
let mut p = dest.as_os_str().to_os_string();
p.push(".part");
PathBuf::from(p)
};
{
let f = tokio::fs::File::create(&part).await?;
f.set_len(m.total_len).await?;
}
let mut file = tokio::fs::OpenOptions::new()
.write(true)
.open(&part)
.await?;
let mut present: Vec<bool> = vec![false; count as usize];
let mut received = 0u32;
let mut stats = TransferStats::default();
let result: Result<()> = async {
for (index, bao) in early.drain(..) {
apply_slice(
&m,
&chunks,
&root,
&mut file,
&mut present,
&mut received,
&mut stats,
sink,
index,
&bao,
)
.await?;
}
while received < count {
if cancel.is_cancelled() {
sink.emit(Progress::Cancelled {
received,
total: count,
});
return Err(BlobError::Cancelled {
received,
total: count,
});
}
let sample =
match tokio::time::timeout(cfg.stall_timeout, subscriber.recv_async()).await {
Ok(Ok(sample)) => sample,
_ => {
return Err(BlobError::Incomplete {
received,
total: count,
});
}
};
let Ok((version, frame)) =
crate::wire::decode::<(u16, FanoutFrame)>(&sample.payload().to_bytes())
else {
continue;
};
if version != crate::wire::WIRE_VERSION {
continue;
}
if let FanoutFrame::Slice { index, bao } = frame {
apply_slice(
&m,
&chunks,
&root,
&mut file,
&mut present,
&mut received,
&mut stats,
sink,
index,
&bao,
)
.await?;
}
}
file.sync_data().await?;
if cfg.overwrite == Overwrite::Refuse && tokio::fs::try_exists(dest).await? {
return Err(BlobError::DestinationExists(dest.to_path_buf()));
}
Ok(())
}
.await;
drop(file);
if let Err(e) = result {
let _ = tokio::fs::remove_file(&part).await;
return Err(e);
}
tokio::fs::rename(&part, dest).await?;
sink.emit(Progress::Completed {
path: dest.to_path_buf(),
});
stats.elapsed = started_at.elapsed();
Ok(stats)
}
#[allow(clippy::too_many_arguments)]
async fn apply_slice(
m: &Manifest,
chunks: &TransferChunks,
root: &blake3::Hash,
file: &mut tokio::fs::File,
present: &mut [bool],
received: &mut u32,
stats: &mut TransferStats,
sink: &dyn ProgressSink,
index: u32,
bao: &[u8],
) -> Result<()> {
let count = chunks.count();
if index >= count || present[index as usize] {
return Ok(());
}
let mut leaves: Vec<(u64, Vec<u8>)> = Vec::new();
let decoded = verify::decode_slice(
root,
m.total_len,
verify::chunk_range(chunks.byte_range(index)),
bao,
|off, data| {
leaves.push((off, data.to_vec()));
Ok(())
},
);
if decoded.is_err() {
stats.rejected += 1;
return Ok(());
}
for (off, data) in leaves {
file.seek(SeekFrom::Start(off)).await?;
file.write_all(&data).await?;
}
present[index as usize] = true;
*received += 1;
stats.chunks_fetched += 1;
stats.bytes_fetched += chunks.len_of(index) as u64;
sink.emit(Progress::Chunk {
index,
received: *received,
total: count,
bytes_received: stats.bytes_fetched,
});
Ok(())
}