mbx_cache_protocol/
lib.rs

1//! Wire types and constants shared by mbx remote cache clients and servers.
2//!
3//! These records are protocol-owned: changing their serialized shape or a
4//! framing constant is a wire-format change. Transport, authentication, local
5//! storage, and adapter behavior deliberately live outside this crate.
6#![deny(missing_docs)]
7
8use serde::{Deserialize, Serialize};
9use sha2::Digest as _;
10use std::collections::BTreeMap;
11use std::fmt;
12use std::fs::File;
13use std::io::Read;
14use std::path::Path;
15
16/// Major version of the HTTP cache protocol.
17pub const PROTOCOL_VERSION: u8 = 1;
18/// Header carrying the negotiated cache protocol version.
19pub const PROTOCOL_HEADER: &str = "mbx-cache-protocol";
20/// Header carrying the caller's isolated cache namespace.
21pub const NAMESPACE_HEADER: &str = "mbx-cache-namespace";
22/// Media type for canonical [`ActionResult`] JSON records.
23pub const ACTION_RESULT_MEDIA_TYPE: &str = "application/vnd.mbx.cache-action-result.v1+json";
24/// Media type for canonical [`Directory`] JSON records.
25pub const DIRECTORY_MEDIA_TYPE: &str = "application/vnd.mbx.cache-directory.v1+json";
26/// Media type for adapter-specific action metadata blobs.
27pub const CLIENT_METADATA_MEDIA_TYPE: &str = "application/vnd.mbx.cache-client-metadata.v1+json";
28/// Media type for task-to-action prediction manifests.
29pub const TASK_ACTION_MANIFEST_MEDIA_TYPE: &str =
30    "application/vnd.mbx.cache-task-action-manifest.v1+json";
31/// Media type for opaque content-addressed blobs.
32pub const BLOB_MEDIA_TYPE: &str = "application/octet-stream";
33/// Media type for framed batches of content-addressed blobs.
34pub const BLOB_PACK_MEDIA_TYPE: &str = "application/vnd.mbx.cache-blob-pack.v1";
35/// Media type for a JSON list of digests.
36pub const DIGEST_LIST_MEDIA_TYPE: &str = "application/vnd.mbx.cache-digests.v1+json";
37/// Media type for a JSON batch of [`ActionResult`] records.
38///
39/// Each record names the action it belongs to, so the batch is unordered and
40/// carries only the results a service actually holds.
41pub const ACTION_RESULT_BATCH_MEDIA_TYPE: &str =
42    "application/vnd.mbx.cache-action-result-batch.v1+json";
43/// Media type for the receipt describing an accepted blob-pack upload.
44pub const BLOB_PACK_RECEIPT_MEDIA_TYPE: &str =
45    "application/vnd.mbx.cache-blob-pack-receipt.v1+json";
46/// Header declaring the number of blobs in a blob pack.
47pub const BLOB_PACK_BLOBS_HEADER: &str = "mbx-cache-pack-blobs";
48/// Header declaring the total payload bytes in a blob pack.
49pub const BLOB_PACK_BYTES_HEADER: &str = "mbx-cache-pack-bytes";
50/// Magic prefix identifying a version-one blob pack.
51pub const BLOB_PACK_MAGIC: &[u8; 8] = b"MBXPACK1";
52/// Bytes before each blob pack payload: algorithm, hash, and length.
53pub const BLOB_PACK_HEADER_BYTES: u64 = 1 + 32 + 8;
54/// Maximum number of digests in a batch request supported by the protocol.
55pub const MAX_BATCH_ITEMS: usize = 10_000;
56/// Maximum predictions carried by one task action manifest.
57pub const MAX_TASK_ACTION_PREDICTIONS: usize = 16 * 1024;
58/// Maximum serialized adapter payload in one action prediction.
59pub const MAX_ACTION_PREDICTION_PAYLOAD: usize = 256 * 1024;
60
61/// Serialize a protocol record using the JSON Canonicalization Scheme.
62pub fn canonical_json(value: &impl Serialize) -> serde_json::Result<Vec<u8>> {
63    serde_json_canonicalizer::to_vec(value)
64}
65
66/// Algorithm-tagged digest and exact byte length of a cache object.
67#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
68#[serde(deny_unknown_fields)]
69pub struct Digest {
70    /// Hash algorithm name (`blake3` or `sha256`).
71    pub algorithm: String,
72    /// Lowercase hexadecimal hash value.
73    pub hash: String,
74    /// Exact uncompressed object length in bytes.
75    pub size: u64,
76}
77
78impl Digest {
79    /// Compute a BLAKE3 digest for in-memory bytes.
80    pub fn blake3(bytes: &[u8]) -> Self {
81        Self {
82            algorithm: DigestAlgorithm::Blake3.into(),
83            hash: blake3::hash(bytes).to_hex().to_string(),
84            size: bytes.len() as u64,
85        }
86    }
87
88    /// Hash a file with BLAKE3 while counting bytes in the same pass.
89    pub fn blake3_file(path: &Path) -> eyre::Result<Self> {
90        let (hash, size) = hash_file(path, DigestAlgorithm::Blake3)?;
91        Ok(Self {
92            algorithm: DigestAlgorithm::Blake3.into(),
93            hash,
94            size,
95        })
96    }
97
98    /// Validate the algorithm and lowercase hexadecimal representation.
99    pub fn validate(&self) -> eyre::Result<()> {
100        self.algorithm_kind()?;
101        if self.hash.len() != 64
102            || !self
103                .hash
104                .bytes()
105                .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte))
106        {
107            eyre::bail!("invalid remote cache digest");
108        }
109        Ok(())
110    }
111
112    /// Return whether `bytes` have this digest and declared length.
113    pub fn matches_bytes(&self, bytes: &[u8]) -> eyre::Result<bool> {
114        self.validate()?;
115        if self.size != bytes.len() as u64 {
116            return Ok(false);
117        }
118        let hash = match self.algorithm_kind()? {
119            DigestAlgorithm::Blake3 => blake3::hash(bytes).to_hex().to_string(),
120            DigestAlgorithm::Sha256 => hex::encode(sha2::Sha256::digest(bytes)),
121        };
122        Ok(self.hash == hash)
123    }
124
125    /// Stream a file and return whether it has this digest and length.
126    pub fn matches_file(&self, path: &Path) -> eyre::Result<bool> {
127        self.validate()?;
128        let (hash, size) = hash_file(path, self.algorithm_kind()?)?;
129        Ok(self.size == size && self.hash == hash)
130    }
131
132    /// Stable storage key containing the algorithm, hash, and byte length.
133    pub fn key(&self) -> String {
134        format!("{}/{}/{}", self.algorithm, self.hash, self.size)
135    }
136
137    /// Parse the algorithm tag into its closed version-one enum.
138    pub fn algorithm_kind(&self) -> eyre::Result<DigestAlgorithm> {
139        Ok(self.algorithm.parse()?)
140    }
141}
142
143/// Hash algorithms supported by protocol version one.
144#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
145#[serde(rename_all = "lowercase")]
146pub enum DigestAlgorithm {
147    /// BLAKE3.
148    Blake3,
149    /// SHA-256.
150    Sha256,
151}
152
153impl DigestAlgorithm {
154    /// Lowercase name serialized on the wire and used in endpoint paths.
155    pub const fn as_str(self) -> &'static str {
156        match self {
157            Self::Blake3 => "blake3",
158            Self::Sha256 => "sha256",
159        }
160    }
161}
162
163impl fmt::Display for DigestAlgorithm {
164    fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
165        formatter.write_str(self.as_str())
166    }
167}
168
169impl std::str::FromStr for DigestAlgorithm {
170    type Err = ParseDigestAlgorithmError;
171
172    fn from_str(value: &str) -> Result<Self, Self::Err> {
173        match value {
174            "blake3" => Ok(Self::Blake3),
175            "sha256" => Ok(Self::Sha256),
176            _ => Err(ParseDigestAlgorithmError),
177        }
178    }
179}
180
181impl From<DigestAlgorithm> for String {
182    fn from(algorithm: DigestAlgorithm) -> Self {
183        algorithm.as_str().into()
184    }
185}
186
187/// A digest algorithm name was outside the version-one contract.
188#[derive(Debug, Clone, Copy, PartialEq, Eq)]
189pub struct ParseDigestAlgorithmError;
190
191impl fmt::Display for ParseDigestAlgorithmError {
192    fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
193        formatter.write_str("unsupported remote cache digest algorithm")
194    }
195}
196
197impl std::error::Error for ParseDigestAlgorithmError {}
198
199fn hash_file(path: &Path, algorithm: DigestAlgorithm) -> eyre::Result<(String, u64)> {
200    let mut file = File::open(path)?;
201    let mut buffer = [0; 64 * 1024];
202    let mut size = 0;
203    let mut blake3 = blake3::Hasher::new();
204    let mut sha256 = sha2::Sha256::new();
205    loop {
206        let count = file.read(&mut buffer)?;
207        if count == 0 {
208            break;
209        }
210        match algorithm {
211            DigestAlgorithm::Blake3 => {
212                blake3.update(&buffer[..count]);
213            }
214            DigestAlgorithm::Sha256 => {
215                sha256.update(&buffer[..count]);
216            }
217        }
218        size += count as u64;
219    }
220    let hash = match algorithm {
221        DigestAlgorithm::Blake3 => blake3.finalize().to_hex().to_string(),
222        DigestAlgorithm::Sha256 => hex::encode(sha256.finalize()),
223    };
224    Ok((hash, size))
225}
226
227/// A canonical action-result record referencing objects in the CAS.
228#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
229#[serde(deny_unknown_fields)]
230pub struct ActionResult {
231    /// Digest of the canonical action descriptor this record satisfies.
232    pub action: Digest,
233    /// Optional adapter metadata blob.
234    #[serde(default)]
235    pub metadata: Option<Digest>,
236    /// Optional digest of the root [`Directory`] containing outputs.
237    #[serde(default)]
238    pub output_root: Option<Digest>,
239    /// Action-result schema version.
240    pub version: u8,
241}
242
243/// A canonical directory object stored in the CAS.
244#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
245#[serde(deny_unknown_fields)]
246pub struct Directory {
247    /// Child directory entries, sorted canonically by name.
248    pub directories: Vec<DirectoryNode>,
249    /// Child file entries, sorted canonically by name.
250    pub files: Vec<FileNode>,
251    /// Child symbolic-link entries, sorted canonically by name.
252    pub symlinks: Vec<SymlinkNode>,
253    /// Directory-object schema version.
254    pub version: u8,
255}
256
257/// A child directory entry in a canonical cache directory.
258#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
259#[serde(deny_unknown_fields)]
260pub struct DirectoryNode {
261    /// Digest of the child [`Directory`].
262    pub digest: Digest,
263    /// Platform mode bits recorded for the directory.
264    pub mode: u32,
265    /// Single path-component name within the parent directory.
266    pub name: String,
267}
268
269/// A file entry in a canonical cache directory.
270#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
271#[serde(deny_unknown_fields)]
272pub struct FileNode {
273    /// Digest of the file contents.
274    pub digest: Digest,
275    /// Whether the file should be restored as executable.
276    pub executable: bool,
277    /// Platform mode bits recorded for the file.
278    pub mode: u32,
279    /// Single path-component name within the parent directory.
280    pub name: String,
281}
282
283/// A symbolic-link entry in a canonical cache directory.
284#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
285#[serde(deny_unknown_fields)]
286pub struct SymlinkNode {
287    /// Platform mode bits recorded for the symbolic link.
288    pub mode: u32,
289    /// Single path-component name within the parent directory.
290    pub name: String,
291    /// Link target text exactly as recorded by the producer.
292    pub target: String,
293}
294
295/// Rust-specific action metadata stored alongside compiled outputs.
296#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
297#[serde(deny_unknown_fields)]
298pub struct RustcMetadata {
299    /// Metadata schema version.
300    pub version: u8,
301    /// Adapter-defined output kind.
302    pub kind: String,
303    /// Digest of captured compiler standard output.
304    pub stdout: Digest,
305    /// Digest of captured compiler standard error.
306    pub stderr: Digest,
307}
308
309impl RustcMetadata {
310    /// Whether the metadata satisfies the version-one rustc schema invariants.
311    pub fn validate(&self) -> bool {
312        self.version == 1
313            && self.kind == "rustc"
314            && self.stdout.validate().is_ok()
315            && self.stderr.validate().is_ok()
316    }
317}
318
319/// C and C++ action metadata stored alongside compiled objects.
320#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
321#[serde(deny_unknown_fields)]
322pub struct CcMetadata {
323    /// Metadata schema version.
324    pub version: u8,
325    /// Adapter-defined output kind.
326    pub kind: String,
327    /// Digest of captured compiler standard output.
328    pub stdout: Digest,
329    /// Digest of captured compiler standard error.
330    pub stderr: Digest,
331}
332
333impl CcMetadata {
334    /// Whether the metadata satisfies the version-one cc schema invariants.
335    pub fn validate(&self) -> bool {
336        self.version == 1
337            && self.kind == "cc"
338            && self.stdout.validate().is_ok()
339            && self.stderr.validate().is_ok()
340    }
341}
342
343/// Adapter-owned data needed to reconstruct an action from a prior task run.
344#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
345#[serde(deny_unknown_fields)]
346pub struct ActionPrediction {
347    /// Invocation digest used to locate this prediction.
348    pub invocation: Digest,
349    /// Full action digest produced when the prediction was recorded.
350    pub action: Digest,
351    /// Adapter name that owns and understands `payload`.
352    pub adapter: String,
353    /// Adapter-defined serialized input prediction.
354    pub payload: String,
355}
356
357/// Predictions associated with one stable task identity.
358#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
359#[serde(deny_unknown_fields)]
360pub struct TaskActionManifest {
361    /// Manifest schema version.
362    pub version: u8,
363    /// Stable task identity.
364    pub task: String,
365    /// Predicted actions, uniquely keyed by invocation digest.
366    pub predictions: Vec<ActionPrediction>,
367}
368
369#[derive(Serialize)]
370struct TaskActionManifestSelector<'a> {
371    kind: &'static str,
372    task: &'a str,
373    version: u8,
374}
375
376impl TaskActionManifest {
377    /// Whether the manifest satisfies the version-one wire invariants.
378    pub fn validate(&self) -> bool {
379        let mut invocations = std::collections::BTreeSet::new();
380        self.version == 1
381            && valid_task_identity(&self.task)
382            && self.predictions.len() <= MAX_TASK_ACTION_PREDICTIONS
383            && self.predictions.iter().all(|prediction| {
384                prediction.validate() && invocations.insert(&prediction.invocation)
385            })
386    }
387
388    /// Digest selecting this task identity's action manifest.
389    pub fn selector_digest(&self) -> Digest {
390        Self::selector(&self.task)
391            .expect("manifest task identity must be valid")
392            .1
393    }
394
395    /// Canonical selector bytes and digest for a task identity.
396    pub fn selector(task: &str) -> eyre::Result<(Vec<u8>, Digest)> {
397        if !valid_task_identity(task) {
398            eyre::bail!("invalid task action manifest identity");
399        }
400        let selector = canonical_json(&TaskActionManifestSelector {
401            kind: "task_action_manifest",
402            task,
403            version: 1,
404        })?;
405        let digest = Digest::blake3(&selector);
406        Ok((selector, digest))
407    }
408}
409
410impl ActionPrediction {
411    /// Whether the prediction satisfies the version-one wire invariants.
412    pub fn validate(&self) -> bool {
413        self.action.algorithm == DigestAlgorithm::Blake3.as_str()
414            && self.action.validate().is_ok()
415            && self.invocation.algorithm == DigestAlgorithm::Blake3.as_str()
416            && self.invocation.validate().is_ok()
417            && !self.adapter.is_empty()
418            && self
419                .adapter
420                .bytes()
421                .all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'_'))
422            && self.payload.len() <= MAX_ACTION_PREDICTION_PAYLOAD
423            && serde_json::from_str::<serde_json::Value>(&self.payload).is_ok()
424    }
425}
426
427fn valid_task_identity(value: &str) -> bool {
428    value.len() == 64
429        && value
430            .bytes()
431            .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte))
432}
433
434/// Version advertised by a cache service.
435///
436/// Capability records are the protocol's additive surface: a server may
437/// advertise fields a client does not know, so every type below stays open to
438/// extension rather than requiring a major release per advertised field.
439#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
440#[non_exhaustive]
441pub struct CapabilityProtocol {
442    /// Protocol major version.
443    pub major: u8,
444    /// Backward-compatible protocol revision.
445    #[serde(default)]
446    pub minor: u8,
447}
448
449impl CapabilityProtocol {
450    /// A protocol version advertisement.
451    pub fn new(major: u8, minor: u8) -> Self {
452        Self { major, minor }
453    }
454}
455
456/// Schemas supported for one action adapter.
457#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
458#[non_exhaustive]
459pub struct ActionKindCapability {
460    /// Action descriptor schema version.
461    pub action_schema: u8,
462    /// Adapter metadata schema version.
463    pub metadata_schema: u8,
464}
465
466impl ActionKindCapability {
467    /// The schema pair one adapter accepts.
468    pub fn new(action_schema: u8, metadata_schema: u8) -> Self {
469        Self {
470            action_schema,
471            metadata_schema,
472        }
473    }
474}
475
476/// Optional server protocol features.
477#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
478#[non_exhaustive]
479pub struct CapabilityFeatures {
480    /// Conditional action-manifest endpoints are available.
481    #[serde(default)]
482    pub action_manifests: bool,
483    /// Missing-blob batch queries are available.
484    #[serde(default)]
485    pub batch: bool,
486    /// Batched action-result lookups are available.
487    ///
488    /// Distinct from [`Self::batch`], which covers missing-blob queries only. A
489    /// service that answered those before this feature existed advertises
490    /// `batch` without implementing the action-result endpoint.
491    #[serde(default)]
492    pub action_batch: bool,
493    /// Framed blob-pack downloads are available.
494    #[serde(default)]
495    pub blob_packs: bool,
496    /// Framed blob-pack uploads are available.
497    #[serde(default)]
498    pub blob_pack_uploads: bool,
499    /// Resumable uploads are available.
500    #[serde(default)]
501    pub resumable_uploads: bool,
502    /// Delegated transfers are available.
503    #[serde(default)]
504    pub delegated_transfers: bool,
505}
506
507/// Server-advertised request and object limits.
508#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
509#[non_exhaustive]
510pub struct CapabilityLimits {
511    /// Maximum digests accepted by one batch request.
512    #[serde(default)]
513    pub max_batch_items: u64,
514    /// Maximum blob size eligible for inline transfer.
515    #[serde(default)]
516    pub max_inline_blob_bytes: u64,
517    /// Maximum size of an individual blob.
518    #[serde(default)]
519    pub max_blob_bytes: u64,
520    /// Maximum declared payload bytes in one blob pack.
521    #[serde(default)]
522    pub max_pack_bytes: u64,
523}
524
525/// Cache service capabilities negotiated before optional protocol features.
526#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
527#[non_exhaustive]
528pub struct Capabilities {
529    /// Protocol version implemented by the server.
530    pub protocol: CapabilityProtocol,
531    /// Digest algorithms accepted by the service.
532    #[serde(default)]
533    pub digest_algorithms: Vec<String>,
534    /// Content codings accepted and produced by the service.
535    #[serde(default)]
536    pub compressors: Vec<String>,
537    /// Adapter schemas accepted by the service.
538    #[serde(default)]
539    pub action_kinds: BTreeMap<String, ActionKindCapability>,
540    /// Optional endpoint features.
541    #[serde(default)]
542    pub features: CapabilityFeatures,
543    /// Server-enforced request limits.
544    #[serde(default)]
545    pub limits: CapabilityLimits,
546}
547
548impl Capabilities {
549    /// A baseline advertisement for `protocol`, claiming no optional features.
550    ///
551    /// The remaining fields are public and assignable, so a service adds only
552    /// what it actually supports.
553    pub fn new(protocol: CapabilityProtocol) -> Self {
554        Self {
555            protocol,
556            digest_algorithms: Vec::new(),
557            compressors: Vec::new(),
558            action_kinds: BTreeMap::new(),
559            features: CapabilityFeatures::default(),
560            limits: CapabilityLimits::default(),
561        }
562    }
563}
564
565#[cfg(test)]
566mod tests {
567    use super::*;
568
569    #[test]
570    fn digest_validation_is_exact() {
571        let valid = Digest {
572            algorithm: DigestAlgorithm::Blake3.into(),
573            hash: "a".repeat(64),
574            size: 42,
575        };
576        assert!(valid.validate().is_ok());
577        assert!(
578            Digest {
579                hash: "A".repeat(64),
580                ..valid.clone()
581            }
582            .validate()
583            .is_err()
584        );
585        assert!(
586            Digest {
587                algorithm: "md5".into(),
588                ..valid
589            }
590            .validate()
591            .is_err()
592        );
593    }
594
595    #[test]
596    fn canonical_json_is_independent_of_map_insertion_order() {
597        #[derive(Serialize)]
598        struct ZThenA {
599            z: u8,
600            a: bool,
601        }
602
603        #[derive(Serialize)]
604        struct AThenZ {
605            a: bool,
606            z: u8,
607        }
608
609        assert_eq!(
610            canonical_json(&ZThenA { z: 1, a: true }).unwrap(),
611            canonical_json(&AThenZ { a: true, z: 1 }).unwrap()
612        );
613    }
614}
mbx_cache_protocol/lib.rs

mbx_cache_protocol/
lib.rs
