aws_s3_transfer/runtime/buffer_pool/
config.rs1use std::fmt;
9
10use super::admission::MAX_PACKED_CARRIERS;
11use super::geometry::PoolGeometry;
12use super::virtual_memory::page_size;
13use super::CarrierCount;
14use crate::config::MemoryBudgetConfig;
15
16const DEFAULT_CARRIER_BYTES: usize = 16 * 1024;
18
19const DEFAULT_BLOCK_BYTES: usize = 128 * 1024 * 1024;
24
25const DEFAULT_OPTIMISTIC_SCAN_BYTES: usize = 32 * 1024 * 1024;
29
30const AUTO_MEMORY_SHARE_DENOMINATOR: usize = 4;
32
33const AUTO_CAPACITY_MAX_BYTES: u64 = 32 * 1024 * 1024 * 1024;
35
36const AUTO_CAPACITY_FALLBACK_BYTES: u64 = 2 * 1024 * 1024 * 1024;
38
39const F64_FRACTION_BITS: u32 = 52;
41
42const F64_EXPONENT_BIAS: u32 = 1023;
44
45#[non_exhaustive]
47#[derive(Clone, Copy, Debug, Eq, PartialEq)]
48pub enum BufferPoolBuildError {
49 InvalidCapacity,
51 MemoryDetectionUnavailable,
53 CapacityOverflow,
55 UnsupportedPageGeometry,
57}
58
59impl fmt::Display for BufferPoolBuildError {
60 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
61 match self {
62 Self::InvalidCapacity => f.write_str("buffer-pool capacity is invalid"),
63 Self::MemoryDetectionUnavailable => {
64 f.write_str("effective memory could not be detected")
65 }
66 Self::CapacityOverflow => {
67 f.write_str("buffer-pool capacity exceeds its representation")
68 }
69 Self::UnsupportedPageGeometry => {
70 f.write_str("runtime page geometry cannot support the buffer pool")
71 }
72 }
73 }
74}
75
76impl std::error::Error for BufferPoolBuildError {}
77
78#[derive(Clone, Debug, Default)]
84pub struct BufferPoolBuilder {
85 memory_budget: MemoryBudgetConfig,
86}
87
88impl BufferPoolBuilder {
89 pub fn memory_budget(mut self, memory_budget: MemoryBudgetConfig) -> Self {
91 self.memory_budget = memory_budget;
92 self
93 }
94
95 pub fn build(self) -> Result<super::BufferPool, BufferPoolBuildError> {
103 let needs_memory_detection = match self.memory_budget {
104 MemoryBudgetConfig::Auto => true,
105 MemoryBudgetConfig::Fraction(fraction)
106 if fraction.is_finite() && 0.0 < fraction && fraction <= 1.0 =>
107 {
108 true
109 }
110 MemoryBudgetConfig::Fraction(_) => {
111 return Err(BufferPoolBuildError::InvalidCapacity);
112 }
113 MemoryBudgetConfig::Limit(_) => false,
114 };
115 let detected_memory = needs_memory_detection
116 .then(crate::runtime::platform::available_ram)
117 .flatten();
118 let diagnostics = crate::config::DiagnosticsConfig::from_env().memory();
119 super::BufferPool::from_capacity(self.memory_budget, detected_memory, diagnostics)
120 }
121}
122
123#[derive(Clone, Copy, Debug, Eq, PartialEq)]
125pub(super) struct PoolConfig {
126 pub(super) geometry: PoolGeometry,
128 pub(super) configured_capacity: CarrierCount,
130 pub(super) optimistic_scan_words: usize,
132}
133
134impl PoolConfig {
135 pub(super) fn resolve(
137 capacity: MemoryBudgetConfig,
138 detected_memory: Option<usize>,
139 ) -> Result<Self, BufferPoolBuildError> {
140 let page_size = page_size().map_err(|_| BufferPoolBuildError::UnsupportedPageGeometry)?;
141 Self::resolve_for_page(capacity, detected_memory, page_size.get())
142 }
143
144 pub(super) fn resolve_for_page(
146 capacity: MemoryBudgetConfig,
147 detected_memory: Option<usize>,
148 page_size: usize,
149 ) -> Result<Self, BufferPoolBuildError> {
150 let carrier_size = DEFAULT_CARRIER_BYTES
151 .div_ceil(page_size)
152 .checked_mul(page_size)
153 .ok_or(BufferPoolBuildError::CapacityOverflow)?;
154
155 let bytes = resolve_capacity_bytes(capacity, detected_memory)?;
156 let carriers = bytes / carrier_size;
157 if carriers == 0 {
158 return Err(BufferPoolBuildError::InvalidCapacity);
159 }
160 let configured_capacity = CarrierCount::new(carriers);
161 if configured_capacity > MAX_PACKED_CARRIERS {
162 return Err(BufferPoolBuildError::CapacityOverflow);
163 }
164 let target_block_carriers = (DEFAULT_BLOCK_BYTES / carrier_size).max(1);
165 let block_carriers = target_block_carriers.min(configured_capacity.get());
166 let block_size = carrier_size
167 .checked_mul(block_carriers)
168 .ok_or(BufferPoolBuildError::CapacityOverflow)?;
169 let geometry = PoolGeometry::new(page_size, block_size, carrier_size)
170 .map_err(|_| BufferPoolBuildError::UnsupportedPageGeometry)?;
171 let optimistic_scan_words = DEFAULT_OPTIMISTIC_SCAN_BYTES
172 .div_ceil(carrier_size)
173 .div_ceil(u64::BITS as usize);
174
175 Ok(Self {
176 geometry,
177 configured_capacity,
178 optimistic_scan_words,
179 })
180 }
181}
182
183pub(crate) fn auto_capacity_bytes(detected_memory: Option<usize>) -> usize {
190 match detected_memory {
191 Some(bytes) => {
192 let maximum = usize::try_from(AUTO_CAPACITY_MAX_BYTES).unwrap_or(usize::MAX);
193 (bytes / AUTO_MEMORY_SHARE_DENOMINATOR).min(maximum)
194 }
195 None => usize::try_from(AUTO_CAPACITY_FALLBACK_BYTES).unwrap_or(usize::MAX),
196 }
197}
198
199fn resolve_capacity_bytes(
201 capacity: MemoryBudgetConfig,
202 detected_memory: Option<usize>,
203) -> Result<usize, BufferPoolBuildError> {
204 match capacity {
205 MemoryBudgetConfig::Auto => Ok(auto_capacity_bytes(detected_memory)),
206 MemoryBudgetConfig::Fraction(fraction) => {
207 if !(0.0 < fraction && fraction <= 1.0) {
208 return Err(BufferPoolBuildError::InvalidCapacity);
209 }
210 let detected =
211 detected_memory.ok_or(BufferPoolBuildError::MemoryDetectionUnavailable)?;
212 Ok(floor_fraction_of(detected, fraction))
213 }
214 MemoryBudgetConfig::Limit(bytes) => {
215 usize::try_from(bytes.as_u64()).map_err(|_| BufferPoolBuildError::CapacityOverflow)
216 }
217 }
218}
219
220fn floor_fraction_of(total: usize, fraction: f64) -> usize {
226 debug_assert!(0.0 < fraction && fraction <= 1.0);
227
228 let bits = fraction.to_bits();
229 let exponent = ((bits >> F64_FRACTION_BITS) & 0x7ff) as u32;
230 let stored_fraction = bits & ((1_u64 << F64_FRACTION_BITS) - 1);
231 let (significand, shift) = if exponent == 0 {
232 (
233 u128::from(stored_fraction),
234 F64_EXPONENT_BIAS + F64_FRACTION_BITS - 1,
235 )
236 } else {
237 (
238 u128::from(stored_fraction | (1_u64 << F64_FRACTION_BITS)),
239 F64_EXPONENT_BIAS + F64_FRACTION_BITS - exponent,
240 )
241 };
242 let product = (total as u128) * significand;
243 if shift >= u128::BITS {
244 return 0;
245 }
246 (product >> shift) as usize
247}
248
249#[cfg(test)]
250mod tests {
251 use super::*;
252 use crate::runtime::buffer_pool::BufferPool;
253 use crate::types::ByteSize;
254
255 const GIB: usize = 1024 * 1024 * 1024;
256
257 #[test]
258 fn test_auto_uses_quarter_of_detected_memory() {
259 let resolved =
260 PoolConfig::resolve_for_page(MemoryBudgetConfig::Auto, Some(2 * GIB), 4096).unwrap();
261 assert_eq!(
262 resolved.configured_capacity.get() * DEFAULT_CARRIER_BYTES,
263 GIB / 2
264 );
265 }
266
267 #[test]
268 #[cfg(target_pointer_width = "64")]
269 fn test_auto_caps_large_machines() {
270 let resolved =
271 PoolConfig::resolve_for_page(MemoryBudgetConfig::Auto, Some(768 * GIB), 4096).unwrap();
272 assert_eq!(
273 resolved.configured_capacity.get() * DEFAULT_CARRIER_BYTES,
274 32 * GIB
275 );
276 }
277
278 #[test]
279 fn test_auto_uses_fallback_without_detection() {
280 let resolved = PoolConfig::resolve_for_page(MemoryBudgetConfig::Auto, None, 4096).unwrap();
281 assert_eq!(
282 resolved.configured_capacity.get() * DEFAULT_CARRIER_BYTES,
283 2 * GIB
284 );
285 }
286
287 #[test]
288 fn test_fraction_requires_detection() {
289 assert_eq!(
290 PoolConfig::resolve_for_page(MemoryBudgetConfig::Fraction(0.5), None, 4096),
291 Err(BufferPoolBuildError::MemoryDetectionUnavailable)
292 );
293 }
294
295 #[test]
296 fn test_fraction_rejects_invalid_values() {
297 for fraction in [f64::NAN, f64::INFINITY, -0.5, 0.0, 1.5] {
298 assert_eq!(
299 PoolConfig::resolve_for_page(
300 MemoryBudgetConfig::Fraction(fraction),
301 Some(GIB),
302 4096,
303 ),
304 Err(BufferPoolBuildError::InvalidCapacity)
305 );
306 }
307 }
308
309 #[test]
310 fn test_fraction_rejects_zero_before_capacity_resolution() {
311 assert_eq!(
312 resolve_capacity_bytes(MemoryBudgetConfig::Fraction(0.0), Some(GIB)),
313 Err(BufferPoolBuildError::InvalidCapacity)
314 );
315 }
316
317 #[test]
318 fn test_fraction_accepts_the_inclusive_upper_bound() {
319 assert_eq!(
320 resolve_capacity_bytes(MemoryBudgetConfig::Fraction(1.0), Some(GIB)),
321 Ok(GIB)
322 );
323 }
324
325 #[test]
326 fn test_fraction_rounds_down_to_complete_carriers() {
327 let resolved =
328 PoolConfig::resolve_for_page(MemoryBudgetConfig::Fraction(0.5), Some(GIB + 1), 4096)
329 .unwrap();
330 assert_eq!(
331 resolved.configured_capacity.get() * DEFAULT_CARRIER_BYTES,
332 GIB / 2
333 );
334 }
335
336 #[test]
337 fn test_fraction_does_not_round_above_the_exact_binary_fraction() {
338 let resolved = PoolConfig::resolve_for_page(
339 MemoryBudgetConfig::Fraction(0.3333333333333333),
340 Some(1_074_266_112),
341 4096,
342 )
343 .unwrap();
344 assert_eq!(resolved.configured_capacity, CarrierCount::new(21855));
345 }
346
347 #[test]
348 fn test_tiny_fraction_resolves_below_one_carrier_without_panicking() {
349 let capacity = MemoryBudgetConfig::Fraction(1e-30);
350 assert_eq!(resolve_capacity_bytes(capacity.clone(), Some(GIB)), Ok(0));
351 assert_eq!(
352 PoolConfig::resolve_for_page(capacity, Some(GIB), 4096),
353 Err(BufferPoolBuildError::InvalidCapacity)
354 );
355 }
356
357 #[test]
358 fn test_non_aligned_limit_cannot_fund_same_sized_reservation() {
359 let limit = DEFAULT_CARRIER_BYTES + 123;
362 let resolved = PoolConfig::resolve_for_page(
363 MemoryBudgetConfig::Limit(ByteSize::new(limit as u64)),
364 None,
365 4096,
366 )
367 .unwrap();
368 let envelope = resolved.geometry.carriers_for_bytes(limit).unwrap();
369
370 assert_eq!(resolved.configured_capacity, CarrierCount::new(1));
371 assert_eq!(envelope, CarrierCount::new(2));
372 assert!(envelope > resolved.configured_capacity);
373 }
374
375 #[test]
376 fn test_capacity_below_one_carrier_is_rejected() {
377 assert_eq!(
378 PoolConfig::resolve_for_page(
379 MemoryBudgetConfig::Limit(ByteSize::new((DEFAULT_CARRIER_BYTES - 1) as u64)),
380 None,
381 4096,
382 ),
383 Err(BufferPoolBuildError::InvalidCapacity)
384 );
385 }
386
387 #[test]
388 fn test_carrier_rounds_up_to_large_runtime_page() {
389 let resolved = PoolConfig::resolve_for_page(
390 MemoryBudgetConfig::Limit(ByteSize::new(256 * 1024)),
391 None,
392 128 * 1024,
393 )
394 .unwrap();
395 assert_eq!(resolved.geometry.carrier_size(), 128 * 1024);
396 assert_eq!(resolved.geometry.block_size(), 256 * 1024);
397 assert_eq!(resolved.configured_capacity, CarrierCount::new(2));
398 }
399
400 #[test]
401 fn test_block_size_is_independent_of_carrier_bitmap_width() {
402 let resolved = PoolConfig::resolve_for_page(
403 MemoryBudgetConfig::Limit(ByteSize::new(GIB as u64)),
404 None,
405 4096,
406 )
407 .unwrap();
408
409 assert_eq!(resolved.geometry.carrier_size(), 16 * 1024);
410 assert_eq!(resolved.geometry.block_size(), 128 * 1024 * 1024);
411 assert_eq!(
412 resolved.geometry.carriers_per_block(),
413 CarrierCount::new(8192)
414 );
415 assert_eq!(resolved.geometry.bitmap_words(), 128);
416 }
417
418 #[test]
419 fn test_optimistic_scan_preserves_its_byte_reach_across_page_geometry() {
420 for page_size in [4096, 128 * 1024] {
421 let resolved = PoolConfig::resolve_for_page(
422 MemoryBudgetConfig::Limit(ByteSize::new(GIB as u64)),
423 None,
424 page_size,
425 )
426 .unwrap();
427 let carrier_size = resolved.geometry.carrier_size();
428 let scan_bytes = resolved.optimistic_scan_words * u64::BITS as usize * carrier_size;
429
430 assert!(scan_bytes >= DEFAULT_OPTIMISTIC_SCAN_BYTES);
431 assert!(scan_bytes - DEFAULT_OPTIMISTIC_SCAN_BYTES < u64::BITS as usize * carrier_size);
432 }
433 }
434
435 #[test]
436 fn test_eight_mibibyte_claim_uses_optimistic_path_after_preparation() {
437 const PART_BYTES: usize = 8 * 1024 * 1024;
438
439 let pool = BufferPool::from_capacity(
440 MemoryBudgetConfig::Limit(ByteSize::new(GIB as u64)),
441 None,
442 crate::config::MemoryDiagnosticsConfig::default(),
443 )
444 .unwrap();
445 let prepared = pool
446 .acquire_unreserved(pool.inner.geometry.carrier_size())
447 .unwrap();
448 drop(prepared);
449 let fallbacks_before = pool.diagnostics().serialized_fallbacks;
450
451 let part = pool.acquire_unreserved(PART_BYTES).unwrap();
452
453 assert_eq!(pool.diagnostics().serialized_fallbacks, fallbacks_before);
454 drop(part);
455 }
456
457 #[test]
458 fn test_block_size_is_capped_by_small_configured_capacity() {
459 let resolved = PoolConfig::resolve_for_page(
460 MemoryBudgetConfig::Limit(ByteSize::new(8 * 1024 * 1024)),
461 None,
462 4096,
463 )
464 .unwrap();
465
466 assert_eq!(resolved.geometry.block_size(), 8 * 1024 * 1024);
467 assert_eq!(
468 resolved.geometry.carriers_per_block(),
469 resolved.configured_capacity
470 );
471 }
472
473 #[cfg(target_pointer_width = "64")]
474 #[test]
475 fn test_capacity_accepts_the_exact_packed_carrier_limit() {
476 let bytes = u32::MAX as usize * DEFAULT_CARRIER_BYTES;
477 let resolved = PoolConfig::resolve_for_page(
478 MemoryBudgetConfig::Limit(ByteSize::new(bytes as u64)),
479 None,
480 4096,
481 )
482 .unwrap();
483 assert_eq!(resolved.configured_capacity, MAX_PACKED_CARRIERS);
484 }
485
486 #[cfg(target_pointer_width = "64")]
487 #[test]
488 fn test_capacity_rejects_packed_carrier_overflow() {
489 let bytes = (u32::MAX as usize + 1) * DEFAULT_CARRIER_BYTES;
490 assert_eq!(
491 PoolConfig::resolve_for_page(
492 MemoryBudgetConfig::Limit(ByteSize::new(bytes as u64)),
493 None,
494 4096
495 ),
496 Err(BufferPoolBuildError::CapacityOverflow)
497 );
498 }
499
500 #[cfg(target_pointer_width = "64")]
503 #[test]
504 fn test_pool_construction_prepares_no_capacity() {
505 let pool = BufferPool::from_capacity(
506 MemoryBudgetConfig::Limit(ByteSize::new(GIB as u64)),
507 Some(8 * GIB),
508 crate::config::MemoryDiagnosticsConfig::default(),
509 )
510 .unwrap();
511 assert_eq!(pool.inner.geometry.carrier_size(), DEFAULT_CARRIER_BYTES);
512 assert_eq!(pool.inner.geometry.block_size(), DEFAULT_BLOCK_BYTES);
513 assert_eq!(pool.metrics().capacity_bytes(), GIB as u64);
514 assert_eq!(pool.metrics().prepared_capacity_bytes(), 0);
515 }
516
517 #[test]
518 fn test_public_builder_constructs_a_lazy_explicit_pool() {
519 let pool = BufferPool::builder()
520 .memory_budget(MemoryBudgetConfig::Limit(ByteSize::new(GIB as u64)))
521 .build()
522 .unwrap();
523
524 assert_eq!(pool.carrier_size(), DEFAULT_CARRIER_BYTES);
525 assert_eq!(pool.metrics().capacity_bytes(), GIB as u64);
526 assert_eq!(pool.metrics().prepared_capacity_bytes(), 0);
527 }
528
529 #[test]
530 fn test_public_builder_rejects_invalid_capacity() {
531 for fraction in [f64::NAN, f64::NEG_INFINITY, -0.1, 0.0, 1.1, f64::INFINITY] {
532 assert_eq!(
533 BufferPool::builder()
534 .memory_budget(MemoryBudgetConfig::Fraction(fraction))
535 .build()
536 .unwrap_err(),
537 BufferPoolBuildError::InvalidCapacity
538 );
539 }
540 assert_eq!(
541 BufferPool::builder()
542 .memory_budget(MemoryBudgetConfig::Limit(ByteSize::new(1)))
543 .build()
544 .unwrap_err(),
545 BufferPoolBuildError::InvalidCapacity
546 );
547 }
548}