snapdir_stores/limits.rs
1//! Per-backend rate-limit table (RATE CAPS ONLY).
2//!
3//! [`BackendLimits`] carries the published per-backend request-rate and
4//! bandwidth ceilings for each storage scheme so the transfer layer can pace
5//! requests/bytes to stay under the backend's documented limits and avoid
6//! provider-side throttling (HTTP 429 / `SlowDown` / `rateLimitExceeded`).
7//!
8//! This module is deliberately **self-contained**: it holds caps only and has
9//! NO notion of retry/backoff (that policy is global and lives in a separate
10//! module added by a later gate). It depends on nothing beyond `std` and the
11//! scheme strings the [`router`](crate::router) already uses.
12//!
13//! The caps are matched on the canonical scheme/adapter names produced by
14//! [`crate::router::Adapter::name`] — `"file"`, `"s3"`, `"b2"`, `"gcs"` (the
15//! `gs://` URL scheme resolves to the `"gcs"` adapter) — plus any unknown /
16//! external scheme, which is treated as unlimited.
17
18/// Published rate caps for a single storage backend.
19///
20/// Every field is a hard ceiling expressed per second; `None` means "no
21/// documented limit" (the transfer layer leaves that dimension unthrottled).
22/// These are **rate caps only** — there is intentionally no retry/backoff
23/// field here (retry policy is global and lives elsewhere).
24#[derive(Debug, Clone, Copy, PartialEq, Eq)]
25pub struct BackendLimits {
26 /// Max GET/HEAD requests per second (`None` = unlimited).
27 pub read_rps: Option<u64>,
28 /// Max PUT requests per second (`None` = unlimited).
29 pub write_rps: Option<u64>,
30 /// Max download bytes per second (`None` = unlimited).
31 pub read_bps: Option<u64>,
32 /// Max upload bytes per second (`None` = unlimited).
33 pub write_bps: Option<u64>,
34}
35
36impl BackendLimits {
37 /// The fully-unlimited limit set (every dimension `None`). Used for the
38 /// `file` backend and any unknown / external scheme.
39 pub const UNLIMITED: BackendLimits = BackendLimits {
40 read_rps: None,
41 write_rps: None,
42 read_bps: None,
43 write_bps: None,
44 };
45}
46
47/// Returns the published [`BackendLimits`] for a storage `scheme`.
48///
49/// `scheme` is the canonical adapter name from
50/// [`crate::router::Adapter::name`] (`"file"`, `"s3"`, `"b2"`, `"gcs"`); the
51/// `gs://` URL scheme is also accepted as an alias for `"gcs"`. Any other
52/// (unknown / third-party / external) scheme — including `"file"` — is treated
53/// as unlimited.
54#[must_use]
55pub fn for_scheme(scheme: &str) -> BackendLimits {
56 match scheme {
57 // AWS S3: >=5,500 read and >=3,500 write requests/sec per prefix; no
58 // documented per-prefix bandwidth cap.
59 // https://docs.aws.amazon.com/AmazonS3/latest/userguide/optimizing-performance.html
60 "s3" => BackendLimits {
61 read_rps: Some(5500),
62 write_rps: Some(3500),
63 read_bps: None,
64 write_bps: None,
65 },
66 // Google Cloud Storage: initial ~5,000 read and ~1,000 write
67 // requests/sec per bucket (autoscales upward); no documented bandwidth
68 // cap. 429 rateLimitExceeded is retryable.
69 // https://docs.cloud.google.com/storage/docs/request-rate
70 "gcs" | "gs" => BackendLimits {
71 read_rps: Some(5000),
72 write_rps: Some(1000),
73 read_bps: None,
74 write_bps: None,
75 },
76 // Backblaze B2 (<=10TB accounts), per account: download 1,200 req/min
77 // = 20 req/s & 200Mbit/s = 25MB/s; upload 3,000 req/min = 50 req/s &
78 // 800Mbit/s = 100MB/s.
79 // https://www.backblaze.com/docs/cloud-storage-rate-limits
80 "b2" => BackendLimits {
81 read_rps: Some(20),
82 write_rps: Some(50),
83 read_bps: Some(25 * 1024 * 1024),
84 write_bps: Some(100 * 1024 * 1024),
85 },
86 // Local filesystem + any unknown / external backend: no rate caps.
87 _ => BackendLimits::UNLIMITED,
88 }
89}
90
91#[cfg(test)]
92mod tests {
93 use super::*;
94 use crate::transfer::RateLimiter;
95 use std::time::Duration;
96
97 /// Builds a current-thread tokio runtime with time enabled (mirrors the
98 /// `RateLimiter` unit-test harness in `transfer.rs`).
99 fn runtime() -> tokio::runtime::Runtime {
100 tokio::runtime::Builder::new_current_thread()
101 .enable_time()
102 .build()
103 .expect("build tokio runtime")
104 }
105
106 #[test]
107 fn limits_for_scheme_file_is_unlimited() {
108 assert_eq!(for_scheme("file"), BackendLimits::UNLIMITED);
109 let l = for_scheme("file");
110 assert_eq!(l.read_rps, None);
111 assert_eq!(l.write_rps, None);
112 assert_eq!(l.read_bps, None);
113 assert_eq!(l.write_bps, None);
114 }
115
116 #[test]
117 fn limits_for_scheme_unknown_is_unlimited() {
118 // External / third-party schemes carry no documented caps.
119 assert_eq!(for_scheme("mock"), BackendLimits::UNLIMITED);
120 assert_eq!(for_scheme("azure"), BackendLimits::UNLIMITED);
121 assert_eq!(for_scheme(""), BackendLimits::UNLIMITED);
122 }
123
124 #[test]
125 fn limits_for_scheme_s3() {
126 let l = for_scheme("s3");
127 assert_eq!(l.read_rps, Some(5500));
128 assert_eq!(l.write_rps, Some(3500));
129 assert_eq!(l.read_bps, None);
130 assert_eq!(l.write_bps, None);
131 }
132
133 #[test]
134 fn limits_for_scheme_gcs() {
135 let expected = BackendLimits {
136 read_rps: Some(5000),
137 write_rps: Some(1000),
138 read_bps: None,
139 write_bps: None,
140 };
141 assert_eq!(for_scheme("gcs"), expected);
142 // The `gs://` URL scheme is an accepted alias for the `gcs` adapter.
143 assert_eq!(for_scheme("gs"), expected);
144 }
145
146 #[test]
147 fn limits_for_scheme_b2() {
148 let l = for_scheme("b2");
149 assert_eq!(l.read_rps, Some(20));
150 assert_eq!(l.write_rps, Some(50));
151 assert_eq!(l.read_bps, Some(25 * 1024 * 1024));
152 assert_eq!(l.write_bps, Some(100 * 1024 * 1024));
153 }
154
155 /// An unlimited request limiter (`None`) is a no-op: acquiring many request
156 /// tokens returns essentially instantly.
157 #[test]
158 fn limits_request_limiter_unlimited_is_noop() {
159 let rt = runtime();
160 rt.block_on(async {
161 let limiter = RateLimiter::new(None);
162 let start = tokio::time::Instant::now();
163 for _ in 0..1000 {
164 limiter.acquire(1).await; // one token == one request
165 }
166 assert!(
167 start.elapsed() < Duration::from_millis(200),
168 "unlimited request limiter must not pace requests"
169 );
170 });
171 }
172
173 /// A request limiter built from a req/s rate paces requests: with the
174 /// "tokens are requests" mapping, `acquire(1)` per request, issuing 2x one
175 /// second's burst must wait ~1s for the bucket to refill. Mirrors the
176 /// `transfer::tests::transfer_config_rate_limiter` pacing pattern, but the
177 /// budget unit is requests, not bytes.
178 #[test]
179 fn limits_request_limiter_paces_requests() {
180 let rt = runtime();
181 rt.block_on(async {
182 // 5 requests/sec. The bucket starts full (5), so the first 5
183 // `acquire(1)` calls are free; the next 5 must wait ~1s to refill.
184 let rps = 5;
185 let limiter = RateLimiter::new(Some(rps));
186 let start = tokio::time::Instant::now();
187 for _ in 0..(rps * 2) {
188 limiter.acquire(1).await; // one token == one request
189 }
190 let elapsed = start.elapsed();
191 assert!(
192 elapsed >= Duration::from_millis(900),
193 "a request limiter fed {rps} req/s should pace 2x burst to ~1s, took {elapsed:?}"
194 );
195 });
196 }
197}