snapdir_stores/lib.rs
1//! snapdir stores library.
2//!
3//! Storage backends for snapdir snapshots plus the store-routing and
4//! external-store shim that implement snapdir's store dispatch:
5//!
6//! - [`FileStore`] — the in-process `file://` backend.
7//! - [`S3Store`] — the native AWS-SDK `s3://` backend (ring rustls).
8//! - [`B2Store`] — the native AWS-SDK `b2://` backend, pointed at Backblaze
9//! B2's S3-compatible endpoint (wraps [`S3Store`] with a custom endpoint).
10//! - [`GcsStore`] — the native `google-cloud-storage` `gs://` backend
11//! (ring rustls; ADC credential chain).
12//! - [`router`] — scheme → adapter resolution, including the hardcoded
13//! `gs://`→`gcs` special case for the Google Cloud Storage adapter.
14//! - [`shim`] ([`ExternalStore`]) — the emit-command shim that dispatches
15//! third-party `snapdir-<name>-store` binaries via the documented
16//! `get-manifest-command` / `get-fetch-files-command` / `get-push-command`
17//! contract.
18//! - [`transfer`] ([`TransferConfig`], [`RateLimiter`], [`run_concurrent`]) —
19//! the concurrency + bandwidth-limiting foundation each store carries via a
20//! [`TransferConfig`] for the (later) concurrent transfer loops.
21//! - [`adaptive`] ([`AdaptiveGate`], [`AdaptiveController`]) — pure, injectable
22//! adaptive control: a resizable concurrency permit pool (async + blocking)
23//! plus a deterministic slow-start/AIMD controller that turns injected op
24//! samples + system metrics into a concurrency limit and target byte-rate
25//! (wiring into the live transfer loops is a later gate).
26//! - [`stream`] ([`StreamStore`]) — object/manifest-level, content-addressed,
27//! verified read/write primitives (the foundation for store-to-store sync),
28//! implemented for [`FileStore`], [`S3Store`], [`GcsStore`], and [`B2Store`].
29//! - [`sync`] ([`sync_snapshot`], [`SyncReport`]) — streaming store-to-store
30//! snapshot copy: walks a source manifest and copies its raw objects
31//! source → dest through memory only (no local filesystem staging),
32//! parallelized across a rayon pool and throttled by a
33//! [`BlockingRateLimiter`](transfer::BlockingRateLimiter); writes the manifest
34//! last (all-or-nothing).
35
36pub mod adaptive;
37pub mod b2_store;
38pub(crate) mod fetch;
39pub mod file_store;
40pub mod gcs_store;
41pub(crate) mod push;
42pub mod router;
43pub mod s3_store;
44pub mod shim;
45pub mod stream;
46pub mod sync;
47pub mod transfer;
48pub(crate) mod util;
49
50pub use adaptive::{
51 p95_object_size, AdaptiveController, AdaptiveGate, AdaptivePolicy, ControllerDriver, Decision,
52 OpResult, OpSample,
53};
54pub use b2_store::B2Store;
55pub use file_store::FileStore;
56pub use gcs_store::{GcsLocation, GcsStore};
57pub use router::{resolve_adapter, Adapter, RouteError};
58pub use s3_store::{S3Location, S3Store};
59pub use shim::ExternalStore;
60pub use stream::StreamStore;
61pub use sync::{sync_snapshot, SyncReport};
62pub use transfer::{
63 classify_error, run_adaptive, run_concurrent, AdaptivePolicy as TransferAdaptivePolicy,
64 BlockingRateLimiter, RateLimiter, TransferConfig,
65};