zenkey_fleet/bus/
blob.rs

1//! The `@blob` plane, as an explorer sees it (RFC 07 §2; issues #58, #68).
2//!
3//! RFC v1.8 modelled `[[blob]]` in the registry, in its own words, "so that an
4//! explorer can see which origins serve blobs, and of which tier". This module
5//! is the half that makes that true: the registry projection ([`blob_list`]),
6//! the addressing type ([`BlobTarget`]), and — behind the `blob` feature — the
7//! two bus operations §2.5 sanctions, in the order it sanctions them.
8//!
9//! **The shape of the plane, and why it is not just another fan-out.** Every
10//! other plane an explorer reads answers in kilobytes. `@blob` answers in
11//! files. So RFC 07 §2.5 splits the interaction in two: *probe* across origins
12//! with a tiny reply (`have`, `manifest`), then *fetch* from the one origin you
13//! chose, at its concrete key. A wildcard-origin bulk fetch is not a slow path
14//! to be discouraged — Zenoh cannot cancel replies already in flight, so N
15//! holders cost N× the bytes with no way to stop them — and this crate makes it
16//! unspellable rather than unfashionable: the wide form is a
17//! [`BlobProbePrefix`], which is not a [`Key`] and does not convert into one,
18//! and `blob_fetch` takes a concrete origin that goes through
19//! [`zenkey::RemoteOrigin::parse`]. (The two functions are named without
20//! links because they exist only under the `blob` feature, and a link that
21//! resolves in one build configuration and not the other is a docs-lane
22//! failure waiting for whoever turns the feature off.)
23//!
24//! **What this module does not implement.** Verified streaming. RFC 07 §2
25//! names `zblob` the reference client and §2.1 makes per-reply verification
26//! *before disk* normative; a second implementation of an integrity anchor is a
27//! second thing that can be wrong about the same bytes. So the fetch path is
28//! zblob's, and this module's job is to spell the keys through zenkey's typed
29//! builders, attribute replies the way RFC 05 §2.1 requires, and report the
30//! result in a shape both frontends can render.
31
32use std::collections::{BTreeMap, BTreeSet};
33
34use crate::{Error, Result};
35use zenkey::grammar::{self, BlobTier, ContentHash, Origin};
36use zenkey::{BlobProbePrefix, Key, RegistrySlice};
37
38use crate::report::{BlobList, BlobListSource, BlobTierRow};
39
40/// What a blob command addresses: RFC 07 §2's three shapes, each validated.
41///
42/// There is deliberately no `String` constructor for the content-addressed
43/// tiers — a `tree` or `store` address is a [`ContentHash`] or it does not
44/// exist. RFC 07 §2.3 revoked the caller-chosen tree name (`tree/nightly`) in
45/// v1.7, and the generated builders have refused to spell one ever since; this
46/// type refuses for the same reason, at the point where an operator's typing
47/// enters the system.
48#[derive(Debug, Clone, PartialEq, Eq)]
49pub enum BlobTarget {
50    /// Tier-1: a named artifact, whose endpoints live under it (RFC 07 §2.2).
51    Artifact { id: String },
52    /// Tier-2: a directory index, keyed by its own root (RFC 07 §2.3).
53    Tree { root: ContentHash },
54    /// Tier-2: one content-addressed chunk (RFC 07 §2.4).
55    Store { algo: String, hash: ContentHash },
56}
57
58impl BlobTarget {
59    /// Parse the one spelling both frontends accept:
60    ///
61    /// ```text
62    /// <id>  |  artifact/<id>  |  tree/<hex>  |  store/<algo>/<hex>
63    /// ```
64    ///
65    /// A bare id is Tier-1, because that is the tier an operator has an id
66    /// *for*: Tier-2 addresses are hashes, and nobody types one from memory.
67    ///
68    /// A **ULID-shaped** id is lowercased at build time (RFC 03 §2: ULIDs
69    /// are key-encoded in lowercase, and Crockford base32 decodes
70    /// case-insensitively — the canonical uppercase display form and the
71    /// lowercase key spelling are the *same* id, so the probe still asks
72    /// for what the caller was given). Any other id that is not a valid
73    /// RFC 03 §2 plain chunk is refused with the citation: outside the
74    /// case-insensitive ULID domain, rewriting the caller's id would mean
75    /// probing for something they did not ask for and reporting holders of
76    /// it, which is worse than refusing (the v1.4 exemption for
77    /// case-sensitive domains).
78    pub fn parse(spec: &str) -> Result<BlobTarget> {
79        let spec = spec.trim().trim_matches('/');
80        if spec.is_empty() {
81            return Err(Error::unaskable(
82                "blob target",
83                "is empty: expected <id>, artifact/<id>, tree/<hex>, or \
84                 store/<algo>/<hex>",
85            ));
86        }
87        let parts: Vec<&str> = spec.split('/').collect();
88        match parts.as_slice() {
89            ["artifact", id] => Self::artifact(id),
90            ["tree"] => Err(Error::unaskable(
91                "tree/",
92                "needs the tree's root hash: `tree/<hex>` (RFC 07 §2.3 — a tree \
93                 is keyed by its own root, and a caller-chosen name has no \
94                 spelling)",
95            )),
96            ["tree", root] => Ok(BlobTarget::Tree {
97                root: content_hash(root, "tree")?,
98            }),
99            ["store"] | ["store", _] => Err(Error::unaskable(
100                "store/",
101                "needs both chunks: `store/<algo>/<hex>` (RFC 07 §2.4)",
102            )),
103            ["store", algo, hash] => {
104                if !grammar::is_valid_plain_chunk(algo) {
105                    return Err(Error::unaskable(
106                        algo.to_string(),
107                        "is not a valid algorithm chunk: RFC 03 §2 requires \
108                         [a-z0-9]([a-z0-9._-]*[a-z0-9])?",
109                    ));
110                }
111                Ok(BlobTarget::Store {
112                    algo: (*algo).to_string(),
113                    hash: content_hash(hash, "store")?,
114                })
115            }
116            [id] => Self::artifact(id),
117            _ => Err(Error::unaskable(
118                spec.to_string(),
119                "is not a blob target: expected <id>, artifact/<id>, tree/<hex>, \
120                 or store/<algo>/<hex>",
121            )),
122        }
123    }
124
125    fn artifact(id: &str) -> Result<BlobTarget> {
126        // RFC 03 §2: a ULID is key-encoded in lowercase, at build time —
127        // this is that build time. Both cases decode to the same id, so
128        // this probes for exactly what the caller was given.
129        if let Some(lower) = zenkey::slug::ulid_slug(id) {
130            return Ok(BlobTarget::Artifact { id: lower });
131        }
132        if !grammar::is_valid_plain_chunk(id) {
133            let hint = if id.chars().any(|c| c.is_ascii_uppercase()) {
134                " — key chunks have no uppercase spelling (RFC 03 §2, RFC 07 §2.2), and only a ULID-shaped id is safely lowercased for you; lowercase this one at the source, so the id you probe for is the id you were given"
135            } else {
136                ""
137            };
138            return Err(Error::unaskable(
139                id.to_string(),
140                format!(
141                    "is not a valid artifact id: RFC 03 §2 requires one plain \
142                     chunk matching [a-z0-9]([a-z0-9._-]*[a-z0-9])?{hint}"
143                ),
144            ));
145        }
146        Ok(BlobTarget::Artifact { id: id.to_string() })
147    }
148
149    pub fn tier(&self) -> BlobTier {
150        match self {
151            BlobTarget::Artifact { .. } => BlobTier::Artifact,
152            BlobTarget::Tree { .. } => BlobTier::Tree,
153            BlobTarget::Store { .. } => BlobTier::Store,
154        }
155    }
156
157    /// The `*`-origin probe prefix (RFC 07 §2.5) — the only wildcard form that
158    /// exists for this plane, and not a [`Key`].
159    pub fn probe_prefix(&self) -> BlobProbePrefix {
160        BlobProbePrefix::new(self.tier())
161    }
162
163    /// This target's concrete key under one origin — the only fetchable form.
164    ///
165    /// For Tier-1 that is the artifact's base key, which the endpoint tails of
166    /// RFC 07 §2.2 hang off; for Tier-2 the key *is* the object.
167    pub fn key_at(&self, origin: &Origin) -> Result<Key> {
168        let key = match self {
169            BlobTarget::Artifact { id } => grammar::blob_key(origin, BlobTier::Artifact, &[id])?,
170            BlobTarget::Tree { root } => grammar::blob_tree_key(origin, root)?,
171            BlobTarget::Store { algo, hash } => grammar::blob_store_key(origin, algo, hash)?,
172        };
173        Ok(key)
174    }
175
176    /// The tier prefix under one origin: `v1/<origin>/@blob/<tier>`. This is
177    /// what the reference client's endpoint helpers append to.
178    pub fn prefix_at(&self, origin: &Origin) -> Key {
179        grammar::blob_tier_prefix(origin, self.tier())
180    }
181
182    /// The canonical spelling, which round-trips through [`parse`](Self::parse).
183    pub fn spelling(&self) -> String {
184        match self {
185            BlobTarget::Artifact { id } => format!("artifact/{id}"),
186            BlobTarget::Tree { root } => format!("tree/{root}"),
187            BlobTarget::Store { algo, hash } => format!("store/{algo}/{hash}"),
188        }
189    }
190
191    /// The tier-1 id, for the reference client's per-id endpoint helpers.
192    ///
193    /// Feature-gated with its only callers: without the transport there is no
194    /// per-id endpoint to build, and an always-compiled private helper nobody
195    /// calls is a dead-code warning in every build that turns `blob` off.
196    #[cfg(feature = "blob")]
197    pub(crate) fn artifact_id(&self) -> Option<&str> {
198        match self {
199            BlobTarget::Artifact { id } => Some(id),
200            _ => None,
201        }
202    }
203}
204
205fn content_hash(text: &str, tier: &str) -> Result<ContentHash> {
206    ContentHash::parse(text).map_err(|e| {
207        Error::unaskable(
208            text.to_string(),
209            format!(
210                "is not a content hash for `{tier}`: {e} (RFC 07 §2.3/§2.4 — \
211                 the key is the digest, so it is lowercase hex of even length)"
212            ),
213        )
214    })
215}
216
217/// Which producers declare which `@blob` tiers, from registry slices.
218///
219/// **No bus traffic.** This is what a slice *says*, which is the only thing
220/// RFC 08 §2's `[[blob]]` table can tell anyone: a producer declaring a tier
221/// claims it serves that tier's endpoints, never that it holds any particular
222/// blob. Possession is a probe's answer, and only a probe's.
223///
224/// `roster` is the liveliness map as [`crate::bus::roster::roster()`] returns it (origin →
225/// producers), inverted here to fill `origins`. Pass `None` when it was not
226/// asked — an offline `--registry` read has learned nothing about who is up,
227/// and `origins: None` is how that stays distinguishable from "declared, but
228/// nobody is serving it" (RFC 09 §5.1 O4).
229pub fn blob_list(
230    slices: &[RegistrySlice],
231    roster: Option<&BTreeMap<String, Vec<String>>>,
232    source: BlobListSource,
233) -> BlobList {
234    // roster is origin → producers; the row wants producer → origins.
235    let by_producer: Option<BTreeMap<&str, Vec<String>>> = roster.map(|r| {
236        let mut out: BTreeMap<&str, Vec<String>> = BTreeMap::new();
237        for (origin, producers) in r {
238            for producer in producers {
239                out.entry(producer.as_str())
240                    .or_default()
241                    .push(origin.clone());
242            }
243        }
244        out
245    });
246
247    let mut tiers = Vec::new();
248    let mut slices_without_blob = 0usize;
249    for slice in slices {
250        if slice.blob.is_empty() {
251            slices_without_blob += 1;
252            continue;
253        }
254        for decl in &slice.blob {
255            tiers.push(BlobTierRow {
256                producer: slice.name.clone(),
257                registry_version: slice.version.clone(),
258                known_tier: decl.tier.known().is_some(),
259                tier: decl.tier.token().to_string(),
260                endpoints: decl.endpoints.clone(),
261                algo: decl.algo.clone(),
262                reference: decl.reference.clone(),
263                encoding: decl
264                    .encoding
265                    .as_ref()
266                    .map(|e| e.as_encoding_str().to_string()),
267                since: decl.since.clone(),
268                description: decl.description.clone(),
269                origins: by_producer
270                    .as_ref()
271                    .map(|m| m.get(slice.name.as_str()).cloned().unwrap_or_default())
272                    .into(),
273            });
274        }
275    }
276    tiers.sort_by(|a, b| (&a.producer, &a.tier).cmp(&(&b.producer, &b.tier)));
277
278    BlobList {
279        tiers,
280        source,
281        slices_considered: slices.len(),
282        slices_without_blob,
283    }
284}
285
286/// Producers whose slice declares `tier` — the capability claim behind a
287/// probe, so silence stays legible (RFC 05 §3.1).
288pub fn declared_by(slices: &[RegistrySlice], tier: BlobTier) -> Vec<String> {
289    let mut names: BTreeSet<String> = BTreeSet::new();
290    for slice in slices {
291        if slice.serves_blob_tier(tier) {
292            names.insert(slice.name.clone());
293        }
294    }
295    names.into_iter().collect()
296}
297
298#[cfg(feature = "blob")]
299mod transfer;
300#[cfg(feature = "blob")]
301pub use transfer::{BlobFetchSpec, FETCH_PRIORITY, blob_fetch, blob_probe, blob_tree_index};
302
303#[cfg(test)]
304mod tests {
305    use super::*;
306
307    const HASH: &str = "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef";
308
309    fn origin() -> Origin {
310        Origin::Host(zenkey::HostId::parse("h-3fa9c2d41b7e").unwrap())
311    }
312
313    #[test]
314    fn a_bare_id_is_tier_one() {
315        assert_eq!(
316            BlobTarget::parse("01jqz3demo0001").unwrap(),
317            BlobTarget::Artifact {
318                id: "01jqz3demo0001".into()
319            }
320        );
321        assert_eq!(
322            BlobTarget::parse("artifact/01jqz3demo0001").unwrap(),
323            BlobTarget::parse("01jqz3demo0001").unwrap()
324        );
325    }
326
327    #[test]
328    fn every_target_round_trips_through_its_spelling() {
329        for spec in [
330            "artifact/01jqz3demo0001",
331            &format!("tree/{HASH}"),
332            &format!("store/blake3/{HASH}"),
333        ] {
334            let target = BlobTarget::parse(spec).unwrap();
335            assert_eq!(target.spelling(), spec);
336            assert_eq!(BlobTarget::parse(&target.spelling()).unwrap(), target);
337        }
338    }
339
340    #[test]
341    fn an_uppercase_ulid_is_lowercased_at_build_time() {
342        // The canonical display form of a ULID. RFC 03 §2 keys ULIDs in
343        // lowercase, *lowercased at build time*: Crockford base32 decodes
344        // case-insensitively, so the lowercase key spelling names exactly
345        // the id the caller gave us — refusing it (or escaping it into
346        // `_xNN_` chunks, as the generated builders once did) would be
347        // manufacturing a second spelling for one id (G-07a).
348        let target = BlobTarget::parse("01JGXQZ4YQK8V6TXW3M9F2A7CD").unwrap();
349        assert_eq!(
350            target,
351            BlobTarget::Artifact {
352                id: "01jgxqz4yqk8v6txw3m9f2a7cd".into()
353            }
354        );
355        assert_eq!(
356            target,
357            BlobTarget::parse("01jgxqz4yqk8v6txw3m9f2a7cd").unwrap(),
358            "both cases of one ULID are one target"
359        );
360    }
361
362    #[test]
363    fn an_uppercase_non_ulid_is_refused_with_the_citation() {
364        // Uppercase but not ULID-shaped (16 chars): outside the
365        // case-insensitive ULID domain nothing is safely lowercased —
366        // refused with the pointer, not guessed at (RFC 03 §2, v1.4).
367        let err = BlobTarget::parse("01HQXK8F9C2N4PZQ")
368            .unwrap_err()
369            .to_string();
370        assert!(err.contains("RFC 03 §2"), "{err}");
371        assert!(err.contains("lowercase"), "{err}");
372        assert!(err.contains("ULID-shaped"), "{err}");
373    }
374
375    #[test]
376    fn a_wildcard_is_not_a_target() {
377        for spec in ["*", "**", "artifact/*", "v1/*/@blob/artifact", "a/b/c/d"] {
378            assert!(
379                BlobTarget::parse(spec).is_err(),
380                "`{spec}` must not parse as a blob target"
381            );
382        }
383    }
384
385    #[test]
386    fn tier_two_needs_a_hash_not_a_name() {
387        // RFC 07 §2.3's revoked spelling, and the shapes around it.
388        for spec in ["tree/nightly", "tree", "store", "store/blake3", "tree/abc"] {
389            assert!(
390                BlobTarget::parse(spec).is_err(),
391                "`{spec}` must not parse as a blob target"
392            );
393        }
394        assert!(BlobTarget::parse(&format!("tree/{HASH}")).is_ok());
395    }
396
397    #[test]
398    fn keys_come_out_of_the_typed_builders() {
399        let o = origin();
400        assert_eq!(
401            BlobTarget::parse("01jqz3demo0001")
402                .unwrap()
403                .key_at(&o)
404                .unwrap()
405                .as_str(),
406            "v1/h-3fa9c2d41b7e/@blob/artifact/01jqz3demo0001"
407        );
408        assert_eq!(
409            BlobTarget::parse(&format!("store/blake3/{HASH}"))
410                .unwrap()
411                .key_at(&o)
412                .unwrap()
413                .as_str(),
414            format!("v1/h-3fa9c2d41b7e/@blob/store/blake3/{HASH}")
415        );
416        assert_eq!(
417            BlobTarget::parse("01jqz3demo0001")
418                .unwrap()
419                .prefix_at(&o)
420                .as_str(),
421            "v1/h-3fa9c2d41b7e/@blob/artifact"
422        );
423        assert_eq!(
424            BlobTarget::parse("01jqz3demo0001")
425                .unwrap()
426                .probe_prefix()
427                .as_str(),
428            "v1/*/@blob/artifact"
429        );
430    }
431
432    fn slice_with_blob(name: &str, body: &str) -> RegistrySlice {
433        let toml = format!(
434            "[registry]\nversion = \"7\"\napp = \"demo\"\nconvention = 1\n\n\
435             [producer]\nname = \"{name}\"\n\n{body}"
436        );
437        zenkey::parse_slice(&toml).unwrap()
438    }
439
440    #[test]
441    fn a_declaration_without_a_roster_says_so() {
442        let slices = vec![
443            slice_with_blob(
444                "netring",
445                "[[blob]]\ntier = \"artifact\"\nendpoints = [\"manifest\", \"have\"]\n",
446            ),
447            slice_with_blob("quiet", ""),
448        ];
449        let list = blob_list(&slices, None, BlobListSource::RegistryDirs);
450        assert_eq!(list.tiers.len(), 1);
451        assert_eq!(list.slices_considered, 2);
452        assert_eq!(list.slices_without_blob, 1);
453        // O4: nobody asked who is up, so this is not "no origin serves it".
454        assert!(list.tiers[0].origins.is_not_asked());
455
456        let roster = BTreeMap::from([("h-3fa9c2d41b7e".to_string(), vec!["netring".to_string()])]);
457        let joined = blob_list(&slices, Some(&roster), BlobListSource::Bus);
458        assert_eq!(
459            joined.tiers[0].origins.as_deref(),
460            Some(["h-3fa9c2d41b7e".to_string()].as_slice())
461        );
462    }
463
464    #[test]
465    fn an_unreserved_tier_survives_flagged_rather_than_dropped() {
466        // O1: a declaration this build does not understand is a fact about the
467        // fleet, and dropping it would report a registry we did not read.
468        let slices = vec![slice_with_blob("future", "[[blob]]\ntier = \"hologram\"\n")];
469        let list = blob_list(&slices, None, BlobListSource::Bus);
470        assert_eq!(list.tiers.len(), 1);
471        assert_eq!(list.tiers[0].tier, "hologram");
472        assert!(!list.tiers[0].known_tier);
473    }
474
475    #[test]
476    fn declared_by_names_the_claimants() {
477        let slices = vec![
478            slice_with_blob("netring", "[[blob]]\ntier = \"artifact\"\n"),
479            slice_with_blob("logs", "[[blob]]\ntier = \"store\"\nalgo = \"blake3\"\n"),
480        ];
481        assert_eq!(declared_by(&slices, BlobTier::Artifact), vec!["netring"]);
482        assert_eq!(declared_by(&slices, BlobTier::Store), vec!["logs"]);
483        assert!(declared_by(&slices, BlobTier::Tree).is_empty());
484    }
485}
zenkey_fleet/bus/blob.rs

zenkey_fleet/bus/
blob.rs
