snapdir_stores/
lib.rs

1//! snapdir stores library.
2//!
3//! Storage backends for snapdir snapshots plus the store-routing and
4//! external-store shim that implement snapdir's store dispatch:
5//!
6//! - [`FileStore`] — the in-process `file://` backend.
7//! - [`S3Store`] — the native AWS-SDK `s3://` backend (ring rustls).
8//! - [`B2Store`] — the native AWS-SDK `b2://` backend, pointed at Backblaze
9//!   B2's S3-compatible endpoint (wraps [`S3Store`] with a custom endpoint).
10//! - [`GcsStore`] — the native `google-cloud-storage` `gs://` backend
11//!   (ring rustls; ADC credential chain).
12//! - [`router`] — scheme → adapter resolution, including the hardcoded
13//!   `gs://`→`gcs` special case for the Google Cloud Storage adapter.
14//! - [`shim`] ([`ExternalStore`]) — the emit-command shim that dispatches
15//!   third-party `snapdir-<name>-store` binaries via the documented
16//!   `get-manifest-command` / `get-fetch-files-command` / `get-push-command`
17//!   contract.
18//! - [`limits`] ([`BackendLimits`], [`for_scheme`]) — the published per-backend
19//!   request-rate / bandwidth caps (rate caps only; no retry policy) keyed by
20//!   storage scheme, used to pace transfers under each provider's documented
21//!   limits.
22//! - [`transfer`] ([`TransferConfig`], [`RateLimiter`], [`run_concurrent`]) —
23//!   the concurrency + bandwidth-limiting foundation each store carries via a
24//!   [`TransferConfig`] for the (later) concurrent transfer loops.
25//! - [`retry`] ([`RetryPolicy`], [`retry_async`], [`retry_blocking`]) — a pure,
26//!   injectable full-jitter exponential-backoff engine (no real clock/sleep of
27//!   its own; the [`Jitter`] source and [`AsyncSleeper`]/[`BlockingSleeper`] are
28//!   injected). SDK-agnostic over an [`Attempt`] outcome; wiring into the
29//!   S3/GCS/B2 call sites is a later gate.
30//! - [`adaptive`] ([`AdaptiveGate`], [`AdaptiveController`]) — pure, injectable
31//!   adaptive control: a resizable concurrency permit pool (async + blocking)
32//!   plus a deterministic slow-start/AIMD controller that turns injected op
33//!   samples + system metrics into a concurrency limit and target byte-rate
34//!   (wiring into the live transfer loops is a later gate).
35//! - [`stream`] ([`StreamStore`]) — object/manifest-level, content-addressed,
36//!   verified read/write primitives (the foundation for store-to-store sync),
37//!   implemented for [`FileStore`], [`S3Store`], [`GcsStore`], and [`B2Store`].
38//! - [`sync`] ([`sync_snapshot`], [`SyncReport`]) — streaming store-to-store
39//!   snapshot copy: walks a source manifest and copies its raw objects
40//!   source → dest through memory only (no local filesystem staging),
41//!   parallelized across a rayon pool and throttled by a
42//!   [`BlockingRateLimiter`](transfer::BlockingRateLimiter); writes the manifest
43//!   last (all-or-nothing).
44
45pub mod adaptive;
46pub mod b2_store;
47pub(crate) mod fetch;
48pub mod file_store;
49pub mod gcs_store;
50pub mod limits;
51pub(crate) mod push;
52pub mod retry;
53pub mod router;
54pub mod s3_store;
55pub mod shim;
56pub mod stream;
57pub mod sync;
58pub mod transfer;
59pub(crate) mod util;
60
61pub use adaptive::{
62    p95_object_size, AdaptiveController, AdaptiveGate, AdaptivePolicy, ControllerDriver, Decision,
63    OpResult, OpSample,
64};
65pub use b2_store::B2Store;
66pub use file_store::FileStore;
67pub use gcs_store::{GcsLocation, GcsStore};
68pub use limits::{for_scheme, BackendLimits};
69pub use retry::{
70    parse_retry_after, retry_async, retry_blocking, retry_network, AsyncSleeper, Attempt,
71    BlockingSleeper, DefaultJitter, FixedJitter, Jitter, RetryPolicy, ThreadSleeper, TokioSleeper,
72};
73pub use router::{resolve_adapter, Adapter, RouteError};
74pub use s3_store::{S3Location, S3Store};
75pub use shim::ExternalStore;
76pub use stream::StreamStore;
77pub use sync::{sync_snapshot, SyncReport};
78pub use transfer::{
79    classify_error, run_adaptive, run_concurrent, AdaptivePolicy as TransferAdaptivePolicy,
80    BlockingRateLimiter, RateLimiter, TransferConfig,
81};
snapdir_stores/lib.rs

snapdir_stores/
lib.rs
