zblob 0.3.0

Resumable verified-streaming blob & directory transfer over Zenoh (BLAKE3/bao, range resume, content-addressed dedup)
Documentation
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
//! Serverless Tier-2 — publish a snapshot into a **Zenoh storage**.
//!
//! Instead of running a [`crate::TreeServer`] for the lifetime of a transfer, a
//! producer can PUT its content-addressed chunks and tree index into a
//! router-hosted storage (the `zenoh-plugin-storage-manager`) and then exit. The
//! storage retains the keys, dedups them fleet-wide (a chunk PUT by any producer
//! is reused by all), and answers the GETs that [`crate::TreeClient`] already
//! issues — so the client needs no changes and the producer needn't stay alive.
//!
//! Content addressing makes this safe: a chunk key `<store>/<algo>/<hash>` only
//! ever maps to one byte string, so the storage's last-writer-wins reconciliation
//! is a no-op and re-publishing is idempotent.
//!
//! **Durability caveat**: a resolved `put` means the sample was handed to the
//! transport, *not* that a storage retained it — and index/chunk keys may land
//! on different storages with no ordering guarantee. [`SnapshotPublisher::publish`]
//! therefore finishes with a **read-back settle phase**: it GETs the index and
//! a sample of chunk keys until they answer (or `settle` expires).
//!
//! Be precise about what that buys, because the obvious reading — "publish
//! returned Ok, so a client can fetch this now" — is only true under
//! [`SettleCoverage::All`]. The default is `Sample(8)`, which establishes that
//! the storage received *something* and says nothing about the chunks it did
//! not probe. And **any** responder satisfies a probe, so a [`TreeServer`] on
//! the same prefix makes the phase report success without a storage having
//! retained anything — easy to arrange accidentally while developing. Use
//! `All` when the producer is about to exit.

use std::sync::Arc;
use std::time::Duration;

use zenoh::qos::{CongestionControl, Priority};
use zenoh::query::ConsolidationMode;

use crate::compress::{ChunkCompression, pack};
use crate::error::{BlobError, Result};
use crate::hash::{Hash, HashAlgo};
use crate::keys::{store_key, tree_key};
use crate::prefix::ServePrefix;
use crate::store::ContentStore;
use crate::tree::TreeIndex;
use crate::wire::{ENC_CHUNK, ENC_INDEX, encode};

/// PUT one content-addressed chunk into the storage under `store_prefix`.
///
/// The key is `<store_prefix>/blake3/<hash>`; the value is the chunk wrapped
/// in the self-describing container frame (`0x00` + raw bytes, or a zstd
/// frame when `compression` says so) — the same framing `TreeServer` puts on
/// the wire. Idempotent — re-PUTting an identical chunk is a no-op.
async fn publish_chunk(
    session: &zenoh::Session,
    store_prefix: &ServePrefix,
    hash: &Hash,
    bytes: &[u8],
    compression: ChunkCompression,
) -> Result<()> {
    session
        .put(
            store_key(store_prefix.as_str(), HashAlgo::Blake3, hash),
            pack(bytes, compression)?,
        )
        .encoding(&ENC_CHUNK)
        // Publications default to `Drop` (unlike queries, which default to
        // `Block` and lend it to their replies — see the crate docs, fact 1).
        // A dropped chunk here is invisible: the settle phase samples, so the
        // publish returns Ok and the producer exits, and the loss surfaces
        // much later on a consumer as an unresolvable NotFound. Bulk data is
        // exactly the case where blocking is right.
        .congestion_control(CongestionControl::Block)
        .priority(Priority::DataLow)
        .await
        .map_err(BlobError::zenoh)
}

/// PUT the chunks `index` references into the storage. Returns how many were
/// published. Stops at the first PUT error.
///
/// Scoped to the snapshot, not to the store: a store commonly holds chunks
/// from other snapshots — and, on a shared machine, other tenants' — while the
/// storage being published into is typically fleet-wide.
async fn publish_snapshot_chunks(
    session: &zenoh::Session,
    store_prefix: &ServePrefix,
    index: &TreeIndex,
    store: &Arc<dyn ContentStore>,
    compression: ChunkCompression,
) -> Result<u32> {
    publish_hashes(
        session,
        store_prefix,
        &index.needed_chunks(),
        store,
        compression,
    )
    .await
}

/// PUT **every** chunk in `store` into the storage. Returns how many were
/// published. Stops at the first PUT error.
///
/// This publishes whatever the store happens to hold, which is rarely what a
/// snapshot publisher wants — prefer [`SnapshotPublisher::chunks_for`]. It stays for
/// the case where a store *is* the unit being replicated (mirroring a whole
/// content store to a router).
async fn publish_store(
    session: &zenoh::Session,
    store_prefix: &ServePrefix,
    store: &Arc<dyn ContentStore>,
    compression: ChunkCompression,
) -> Result<u32> {
    // Enumerating a DirStore is a recursive `read_dir`; it does not belong on
    // an async thread any more than the reads that follow it do.
    let enumerate = store.clone();
    let hashes = tokio::task::spawn_blocking(move || enumerate.hashes())
        .await
        .map_err(BlobError::from)??;
    publish_hashes(session, store_prefix, &hashes, store, compression).await
}

/// How many chunks are read out of the store per blocking dispatch.
///
/// The store is synchronous, so reading it from an `async fn` blocks the
/// reactor — which is what this used to do, once per chunk, for a whole
/// snapshot. Reading in batches keeps that off the async threads while
/// bounding how much sits in memory at once.
const PUBLISH_READ_BATCH: usize = 32;

async fn publish_hashes(
    session: &zenoh::Session,
    store_prefix: &ServePrefix,
    hashes: &[Hash],
    store: &Arc<dyn ContentStore>,
    compression: ChunkCompression,
) -> Result<u32> {
    let mut published = 0u32;
    for group in hashes.chunks(PUBLISH_READ_BATCH) {
        let store = store.clone();
        let wanted = group.to_vec();
        // One dispatch per batch, and `get_many` so a transactional store
        // sees one visit rather than `PUBLISH_READ_BATCH` of them.
        let bytes = tokio::task::spawn_blocking(move || store.get_many(&wanted))
            .await
            .map_err(BlobError::from)??;
        for (hash, maybe) in group.iter().zip(bytes) {
            let bytes = maybe.ok_or_else(|| BlobError::NotFound(hash.to_string()))?;
            publish_chunk(session, store_prefix, hash, &bytes, compression).await?;
            published += 1;
        }
    }
    Ok(published)
}

/// PUT a tree index into the storage at `<tree_prefix>/<id>`. A
/// [`crate::TreeClient`] with the matching `tree_prefix` then GETs it like any
/// other index.
async fn publish_index(
    session: &zenoh::Session,
    tree_prefix: &ServePrefix,
    index: &TreeIndex,
) -> Result<()> {
    let payload = encode(index)?;
    session
        .put(tree_key(tree_prefix.as_str(), &index.id), payload)
        .encoding(&ENC_INDEX)
        // See publish_chunk: publications default to Drop, and losing the
        // index loses the snapshot.
        .congestion_control(CongestionControl::Block)
        .priority(Priority::DataLow)
        .await
        .map_err(BlobError::zenoh)
}

/// Which of `n` chunk indices the read-back phase probes.
///
/// Extracted from `publish_snapshot` because it was unreachable from a test
/// there: the whole `Sample` arm — the default — was dead code as far as the
/// suite was concerned, and it is arithmetic with an off-by-one in every
/// direction (empty input, `k` larger than `n`, `k` of zero).
///
/// The result is ascending, duplicate-free, always contains the first and last
/// index, and never exceeds `k` entries.
fn sample_indices(n: usize, coverage: SettleCoverage) -> Vec<usize> {
    debug_assert!(n > 0);
    match coverage {
        SettleCoverage::All => (0..n).collect(),
        SettleCoverage::Sample(k) => {
            let k = k.max(1);
            let step = (n / k).max(1);
            let mut picked: Vec<usize> = (0..n).step_by(step).take(k).collect();
            // The last chunk matters most — it is the one a truncated publish
            // loses — so it is always probed. Swapping it in for the final
            // stride pick rather than appending keeps the promised bound of
            // `k` keys, which appending quietly broke.
            if picked.last() != Some(&(n - 1)) {
                if picked.len() >= k {
                    picked.pop();
                }
                picked.push(n - 1);
            }
            picked
        }
    }
}

/// How much of a published snapshot the read-back phase actually checks.
///
/// The distinction matters because the phase's whole job is to let a producer
/// exit safely, and "the storage answered for 7 of 10,000 keys" is a very
/// different claim from "the snapshot is retrievable". Sampling is a smoke
/// test for *did the storage receive anything at all*; it cannot detect
/// individual losses in between.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
#[non_exhaustive]
pub enum SettleCoverage {
    /// Probe the index plus at most `n` chunk keys, spread deterministically
    /// across the snapshot (first, last, and an even stride between).
    Sample(usize),
    /// Probe the index and **every** chunk. Proportional to the snapshot and
    /// the only setting that actually establishes retrievability.
    All,
}

impl Default for SettleCoverage {
    fn default() -> Self {
        SettleCoverage::Sample(8)
    }
}

/// Publish a whole snapshot — chunks then index — into the storage, then
/// **verify by reading back** (see the module docs) within `settle`. After
/// this resolves the producer may exit: a client can fetch the snapshot.
///
/// Only the chunks `index` references are published, not the whole `store`.
///
/// Note what the read-back can and cannot tell you. Any responder counts, so
/// a `TreeServer` running on the same prefix answers the probes and the phase
/// reports success without a storage having retained anything — which is easy
/// to arrange accidentally in development. And under
/// [`SettleCoverage::Sample`] the unprobed chunks are simply unknown. Use
/// [`SettleCoverage::All`] when the producer is about to exit and the
/// snapshot has to be there.
#[allow(clippy::too_many_arguments)]
async fn publish_snapshot(
    session: &zenoh::Session,
    store_prefix: &ServePrefix,
    tree_prefix: &ServePrefix,
    index: &TreeIndex,
    store: &Arc<dyn ContentStore>,
    compression: ChunkCompression,
    coverage: SettleCoverage,
    settle: Duration,
) -> Result<()> {
    publish_snapshot_chunks(session, store_prefix, index, store, compression).await?;
    publish_index(session, tree_prefix, index).await?;

    // Read-back: the index, plus chunk keys per `coverage` (deterministic,
    // no RNG).
    let needed = index.needed_chunks();
    let mut probes: Vec<String> = vec![tree_key(tree_prefix.as_str(), &index.id)];
    let n = needed.len();
    if n > 0 {
        probes.extend(
            sample_indices(n, coverage)
                .into_iter()
                .map(|i| store_key(store_prefix.as_str(), HashAlgo::Blake3, &needed[i])),
        );
    }

    let deadline = tokio::time::Instant::now() + settle;
    for key in probes {
        loop {
            if probe_key(session, &key).await? {
                break;
            }
            if tokio::time::Instant::now() >= deadline {
                return Err(BlobError::NotSettled(format!(
                    "storage did not settle within {settle:?}: {key} still unanswered"
                )));
            }
            tokio::time::sleep(Duration::from_millis(100)).await;
        }
    }
    Ok(())
}

/// One GET probe: does anything answer for `key`?
async fn probe_key(session: &zenoh::Session, key: &str) -> Result<bool> {
    let replies = session
        .get(key)
        .consolidation(ConsolidationMode::None)
        .priority(Priority::DataLow)
        .timeout(Duration::from_secs(2))
        .await
        .map_err(BlobError::zenoh)?;
    while let Ok(reply) = replies.recv_async().await {
        if reply.result().is_ok() {
            return Ok(true);
        }
    }
    Ok(false)
}

/// Publishes content-addressed chunks into a storage under one prefix.
///
/// The five `publish_*` free functions this replaces shared four parameters
/// and differed by one, and the widest of them took **eight** — session,
/// two prefixes, index, store, compression, coverage and settle — in an
/// order nothing but the compiler could keep straight. A publisher is
/// configured once and then asked to publish things.
///
/// A `Publisher` alone can only publish chunks. Publishing a *snapshot* also
/// needs a tree prefix, so [`snapshots`](Self::snapshots) produces a
/// [`SnapshotPublisher`] that has one — rather than an optional field that
/// makes `publish` fail at runtime on a publisher that could never have
/// worked.
#[derive(Debug, Clone)]
pub struct Publisher {
    session: zenoh::Session,
    store_prefix: ServePrefix,
    compression: ChunkCompression,
}

impl Publisher {
    /// A publisher writing chunks under `store_prefix`, uncompressed.
    #[must_use]
    pub fn new(session: &zenoh::Session, store_prefix: ServePrefix) -> Self {
        Publisher {
            session: session.clone(),
            store_prefix,
            compression: ChunkCompression::default(),
        }
    }

    /// Compress chunks at rest with `compression` (default: none).
    ///
    /// Chunks are self-describing containers, so a reader handles either.
    #[must_use]
    pub fn compression(mut self, compression: ChunkCompression) -> Self {
        self.compression = compression;
        self
    }

    /// Also publish tree indices, under `tree_prefix`.
    #[must_use]
    pub fn snapshots(self, tree_prefix: ServePrefix) -> SnapshotPublisher {
        SnapshotPublisher {
            chunks: self,
            tree_prefix,
            coverage: SettleCoverage::default(),
            settle: Duration::from_secs(10),
        }
    }

    /// The prefix chunks are published under.
    #[must_use]
    pub fn store_prefix(&self) -> &ServePrefix {
        &self.store_prefix
    }

    /// PUT one chunk. Idempotent — re-PUTting identical bytes is a no-op.
    pub async fn chunk(&self, hash: &Hash, bytes: &[u8]) -> Result<()> {
        publish_chunk(
            &self.session,
            &self.store_prefix,
            hash,
            bytes,
            self.compression,
        )
        .await
    }

    /// PUT the named chunks out of `store`. Returns how many were published;
    /// a hash the store does not hold is an error
    /// ([`BlobError::NotFound`]) — the caller named it, so silently
    /// publishing less than asked would make the count a lie.
    pub async fn chunks(&self, hashes: &[Hash], store: &Arc<dyn ContentStore>) -> Result<u32> {
        publish_hashes(
            &self.session,
            &self.store_prefix,
            hashes,
            store,
            self.compression,
        )
        .await
    }

    /// PUT **every** chunk in `store`. Returns how many were published.
    ///
    /// This publishes whatever the store happens to hold, which is rarely
    /// what a snapshot publisher wants — prefer
    /// [`SnapshotPublisher::chunks_for`]. It stays for the case where a store
    /// *is* the unit being replicated (mirroring a whole content store to a
    /// router).
    pub async fn store(&self, store: &Arc<dyn ContentStore>) -> Result<u32> {
        publish_store(&self.session, &self.store_prefix, store, self.compression).await
    }
}

/// A [`Publisher`] that also publishes tree indices — see
/// [`Publisher::snapshots`].
///
/// Derefs to `Publisher`, so the chunk methods are all available here too.
#[derive(Debug, Clone)]
pub struct SnapshotPublisher {
    chunks: Publisher,
    tree_prefix: ServePrefix,
    coverage: SettleCoverage,
    settle: Duration,
}

impl std::ops::Deref for SnapshotPublisher {
    type Target = Publisher;
    fn deref(&self) -> &Publisher {
        &self.chunks
    }
}

impl SnapshotPublisher {
    /// How much of a published snapshot [`publish`](Self::publish) reads back
    /// (default: [`SettleCoverage::Sample(8)`](SettleCoverage::Sample)).
    #[must_use]
    pub fn coverage(mut self, coverage: SettleCoverage) -> Self {
        self.coverage = coverage;
        self
    }

    /// How long [`publish`](Self::publish) waits for the read-back to settle
    /// (default: 10 s).
    #[must_use]
    pub fn settle(mut self, settle: Duration) -> Self {
        self.settle = settle;
        self
    }

    /// The prefix indices are published under.
    #[must_use]
    pub fn tree_prefix(&self) -> &ServePrefix {
        &self.tree_prefix
    }

    /// PUT a tree index at `<tree_prefix>/<id>`, without its chunks. A
    /// [`crate::TreeClient`] with the matching prefix then GETs it like any
    /// other index.
    pub async fn index(&self, index: &TreeIndex) -> Result<()> {
        publish_index(&self.chunks.session, &self.tree_prefix, index).await
    }

    /// PUT exactly the chunks `index` references (not the whole store).
    /// Returns how many were published.
    pub async fn chunks_for(
        &self,
        index: &TreeIndex,
        store: &Arc<dyn ContentStore>,
    ) -> Result<u32> {
        publish_snapshot_chunks(
            &self.chunks.session,
            &self.chunks.store_prefix,
            index,
            store,
            self.chunks.compression,
        )
        .await
    }

    /// Publish a whole snapshot — chunks then index — then **verify by
    /// reading back** (see the module docs) within the settle budget. After
    /// this resolves the producer may exit.
    ///
    /// Note what the read-back can and cannot tell you. Any responder counts,
    /// so a `TreeServer` running on the same prefix answers the probes and the
    /// phase reports success without a storage having retained anything —
    /// which is easy to arrange accidentally in development. And under
    /// [`SettleCoverage::Sample`] the unprobed chunks are simply unknown. Use
    /// [`SettleCoverage::All`] when the producer is about to exit and the
    /// snapshot has to be there.
    pub async fn publish(&self, index: &TreeIndex, store: &Arc<dyn ContentStore>) -> Result<()> {
        publish_snapshot(
            &self.chunks.session,
            &self.chunks.store_prefix,
            &self.tree_prefix,
            index,
            store,
            self.chunks.compression,
            self.coverage,
            self.settle,
        )
        .await
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    /// The read-back's sample is what decides whether a producer may exit, so
    /// its arithmetic is load-bearing — and it was reachable only through a
    /// live Zenoh session, which is why none of it was covered.
    #[test]
    fn sampling_always_covers_the_ends_and_respects_its_bound() {
        for n in 1..=200usize {
            // `All` is the whole snapshot, in order.
            let all = sample_indices(n, SettleCoverage::All);
            assert_eq!(all, (0..n).collect::<Vec<_>>(), "All at n={n}");

            for k in [1usize, 2, 3, 8, 64, 1000] {
                let picked = sample_indices(n, SettleCoverage::Sample(k));
                let label = format!("n={n} k={k}");

                assert!(!picked.is_empty(), "{label}: sampled nothing");
                assert!(picked.len() <= k, "{label}: {} entries", picked.len());
                assert!(picked.len() <= n, "{label}: more picks than chunks");
                assert!(
                    picked.windows(2).all(|w| w[0] < w[1]),
                    "{label}: not strictly ascending: {picked:?}"
                );
                assert!(picked.iter().all(|i| *i < n), "{label}: index out of range");
                assert_eq!(
                    *picked.last().unwrap(),
                    n - 1,
                    "{label}: the last chunk — the one a truncated publish \
                     loses — must always be probed"
                );
                if k > 1 || n == 1 {
                    assert_eq!(picked[0], 0, "{label}: the first chunk must be probed");
                }
            }
        }
    }

    /// `Sample(0)` would otherwise mean "probe nothing", i.e. a settle phase
    /// that establishes nothing while reporting success.
    #[test]
    fn a_zero_sample_still_probes() {
        let picked = sample_indices(10, SettleCoverage::Sample(0));
        assert_eq!(picked, vec![9]);
    }

    /// The default is a sample, not the whole snapshot — the distinction the
    /// type exists to make, and one a caller must be able to rely on when
    /// reasoning about publish cost.
    #[test]
    fn the_default_coverage_samples() {
        assert_eq!(SettleCoverage::default(), SettleCoverage::Sample(8));
        assert_eq!(sample_indices(1000, SettleCoverage::default()).len(), 8);
        assert_eq!(sample_indices(1000, SettleCoverage::All).len(), 1000);
    }
}