decdn_cache/
lib.rs

1//! The deCDN node's cache engine: an `iroh-blobs` store with origin
2//! pull-through, range admission, and pluggable admission and eviction
3//! policies.
4//!
5//! [`CacheEngine`] owns the store. On a cache miss it pulls from a configured
6//! [`Origin`] and verifies the bytes against their BLAKE3 hash before it
7//! stores or returns them. Pinned blobs, probe holds, and the deny-sets are
8//! enforced by the engine itself, whatever policy is plugged in.
9//!
10//! # Entry points
11//!
12//! | You want to | Use |
13//! |---|---|
14//! | Open a store | [`CacheEngine::open`] and its variants |
15//! | Read a whole blob, pulling it from the origin on a miss | [`CacheEngine::get`], [`CacheEngine::populate`] |
16//! | Stream a verified sub-range from the origin, one window per draw | [`CacheEngine::origin_range_wire`] |
17//! | Export a stored range as the bao wire encoding | [`CacheEngine::export_bao_range_stream`] |
18//! | Admit a verified bao stream into the store | [`CacheEngine::admit_bao_stream`] |
19//! | Share one in-progress fill between several readers | [`FillRegistry`], [`FillSession`] |
20//! | Pick an origin backend | [`FilesystemOrigin`], [`HttpOrigin`], [`S3Origin`], or your own [`Origin`] |
21//! | Choose what to admit and what to evict | [`AdmissionPolicy`], [`EvictionPolicy`] ([ADR 040]) |
22//! | Bound a failing origin | [`OriginBreaker`], [`RetryPolicy`] |
23//!
24//! # Seams
25//!
26//! This crate has no payment logic, so it runs on its own for local
27//! development and tests. A paid node-to-node pull is one more [`Origin`]: the
28//! node supplies an origin that pays a peer over `cdn/client/v1`. The crate has
29//! no mode branching and no feature flags. The node's wiring layer picks the
30//! backends ([PoC/production seams]).
31//!
32//! [ADR 040]: https://github.com/decdn/decdn/blob/main/adr/040-cache-policy.md
33//! [PoC/production seams]: https://github.com/decdn/decdn/blob/main/adr/appendix-poc-production-seams.md
34
35pub mod circuit_breaker;
36pub mod engine;
37pub mod error;
38pub mod fill_session;
39pub mod metrics;
40pub mod origin;
41pub mod origin_probe;
42pub mod origin_range;
43mod outboard_cache;
44pub mod policy;
45pub mod probe_hold;
46pub mod range_pull;
47pub mod ranged_store;
48pub mod retry;
49pub mod serve_store;
50
51pub use circuit_breaker::{
52    Admission, BreakerState, Clock, ManualClock, OriginBreaker, OriginOutcome, SystemClock,
53};
54/// Bao chunk-group granularity (16 KiB, ADR 038), re-exported because it is a
55/// *billing-visible* property of [`CacheEngine::export_bao_range_stream`]: the
56/// export snaps a requested range out to the enclosing group boundaries and the
57/// serve path does not trim back, so the payer is charged for the aligned
58/// superset. Any caller pricing a range before serving it must align first or it
59/// under-reserves by up to two groups.
60pub use decdn_bao_range::CHUNK_GROUP_BYTES;
61pub use engine::{
62    CacheEngine, EvictionCandidates, EvictionPreview, OriginHeldReport, OriginPresence,
63    PresentRanges, ServeAudit,
64};
65pub use error::{
66    CacheError, CacheResult, ErrorChain, OriginError, OriginPullError, SupportedEncoding,
67};
68pub use fill_session::{
69    Demand, DemandSlot, DownstreamWatch, FillClaim, FillError, FillRegistry, FillSession, Frontier,
70    HashOutboard, ObserverLease, SessionOutboardReader, ended_at_held_bytes_cap, is_unlanded,
71    unlanded,
72};
73/// The blob-store hash: `decdn_cache::Hash` is `iroh_blobs::Hash`, the BLAKE3
74/// digest the iroh-blobs store keys on. The *config-vocabulary* hash that crosses
75/// operator-facing config/admin surfaces is the distinct
76/// [`decdn_config_types::Hash`]; [`to_store_hash`]/[`from_store_hash`]
77/// convert between the two (both are the same 32 bytes).
78pub use iroh_blobs::Hash;
79pub use metrics::CacheMetrics;
80pub use origin::{
81    FilesystemOrigin, HttpOrigin, Origin, OriginFetch, OriginRangeFetch, OriginRangeRequest,
82    OutboardFetch, S3Credentials, S3Origin, S3OriginConfig,
83};
84pub use origin_range::{MAX_CONCURRENT_RANGE_PULLS, OriginRangeWire, RANGE_PULL_WINDOW_BYTES};
85pub use policy::{
86    AdmissionContext, AdmissionDecision, AdmissionPolicy, AlwaysAdmit, EvictionContext,
87    EvictionPlan, EvictionPolicy, FrequencyEstimator, LruEviction, Segment,
88};
89pub use probe_hold::{
90    PROBE_HOLD_DURATION, PROBE_HOLD_MARGIN, PROBE_SLASH_WINDOW, ProbeHoldOutcome,
91};
92pub use ranged_store::NodeRangedStore;
93pub use serve_store::{EncodeStream, PresentRangeWatch, ServeStore};
94
95// Config-vocabulary types live in the `decdn-config-types` leaf crate
96// (no iroh-blobs / no AWS SDK) so the publisher CLI doesn't link the
97// storage stack (see `adr/appendix-binaries.md`). They are re-exported here so the
98// node names its cache vocabulary from one crate. `decdn_config_types::Hash`
99// is NOT re-exported as `decdn_cache::Hash` — that name stays the store
100// hash above.
101/// The config-vocabulary hash type (`decdn_config_types::Hash`), re-exported so
102/// callers can build a [`DeniedHashes`] / [`PinnedHashes`] without taking a
103/// direct dependency on the leaf crate. Distinct from [`struct@Hash`], which is
104/// the iroh-blobs store hash — see the `hash_bridge` conversions.
105pub use decdn_config_types::Hash as LeafHash;
106pub use decdn_config_types::{
107    Bytes, CircuitBreakerPolicy, DEFAULT_MAX_PROBE_HOLDS, DEFAULT_USER_AGENT, DecompressMode,
108    DeniedHashes, HashParseError, OriginKind, OriginUrl, PinDiff, PinnedHashes, RetryPolicy,
109    parse_origin_url, redact_for_log,
110};
111
112/// Convert a config-vocabulary [`decdn_config_types::Hash`] into the
113/// blob-store [`struct@Hash`]. Both are 32-byte BLAKE3 digests; this is a pure
114/// byte-for-byte reinterpretation. Used at the node's
115/// admin boundary where `parse_hash_arg` yields the config-vocabulary
116/// hash that must then index the store.
117#[must_use]
118pub const fn to_store_hash(h: decdn_config_types::Hash) -> Hash {
119    Hash::from_bytes(h.to_bytes())
120}
121
122/// Convert a blob-store [`struct@Hash`] into a config-vocabulary
123/// [`decdn_config_types::Hash`].
124#[must_use]
125#[allow(clippy::missing_const_for_fn)] // iroh_blobs::Hash::as_bytes is not const
126pub fn from_store_hash(h: Hash) -> decdn_config_types::Hash {
127    decdn_config_types::Hash::from_bytes(*h.as_bytes())
128}
129
130#[cfg(test)]
131#[allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] // tests
132mod hash_bridge_tests;
decdn_cache/lib.rs

decdn_cache/
lib.rs
