Skip to main content

aws_s3_transfer/runtime/buffer_pool/
config.rs

1/*
2 * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
3 * SPDX-License-Identifier: Apache-2.0
4 */
5
6//! Checked capacity policy and storage defaults for one buffer pool.
7
8use std::fmt;
9
10use super::admission::MAX_PACKED_CARRIERS;
11use super::geometry::PoolGeometry;
12use super::virtual_memory::page_size;
13use super::CarrierCount;
14use crate::config::MemoryBudgetConfig;
15
16/// Default carrier size before runtime page-size alignment.
17const DEFAULT_CARRIER_BYTES: usize = 16 * 1024;
18
19/// Target bytes prepared and reclaimed as one block.
20///
21/// Pools configured below this value use their complete carrier-rounded
22/// capacity as one block.
23const DEFAULT_BLOCK_BYTES: usize = 128 * 1024 * 1024;
24
25/// Target byte range represented by one optimistic bitmap scan.
26///
27/// The actual reach rounds up to a complete bitmap word.
28const DEFAULT_OPTIMISTIC_SCAN_BYTES: usize = 32 * 1024 * 1024;
29
30/// Denominator of the detected-memory share selected by automatic capacity.
31const AUTO_MEMORY_SHARE_DENOMINATOR: usize = 4;
32
33/// Maximum automatic capacity in bytes.
34const AUTO_CAPACITY_MAX_BYTES: u64 = 32 * 1024 * 1024 * 1024;
35
36/// Automatic capacity when effective memory cannot be detected.
37const AUTO_CAPACITY_FALLBACK_BYTES: u64 = 2 * 1024 * 1024 * 1024;
38
39/// Explicit fraction bits stored by an IEEE 754 binary64 value.
40const F64_FRACTION_BITS: u32 = 52;
41
42/// Exponent bias used by an IEEE 754 binary64 value.
43const F64_EXPONENT_BIAS: u32 = 1023;
44
45/// Failure to resolve capacity and geometry before pool publication.
46#[non_exhaustive]
47#[derive(Clone, Copy, Debug, Eq, PartialEq)]
48pub enum BufferPoolBuildError {
49    /// Capacity is invalid or resolves below one allocation unit.
50    InvalidCapacity,
51    /// The selected capacity policy requires unavailable memory detection.
52    MemoryDetectionUnavailable,
53    /// Capacity or geometry exceeds its internal representation.
54    CapacityOverflow,
55    /// Runtime page geometry cannot produce a supported pool layout.
56    UnsupportedPageGeometry,
57}
58
59impl fmt::Display for BufferPoolBuildError {
60    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
61        match self {
62            Self::InvalidCapacity => f.write_str("buffer-pool capacity is invalid"),
63            Self::MemoryDetectionUnavailable => {
64                f.write_str("effective memory could not be detected")
65            }
66            Self::CapacityOverflow => {
67                f.write_str("buffer-pool capacity exceeds its representation")
68            }
69            Self::UnsupportedPageGeometry => {
70                f.write_str("runtime page geometry cannot support the buffer pool")
71            }
72        }
73    }
74}
75
76impl std::error::Error for BufferPoolBuildError {}
77
78/// Builder for one caller-owned shared payload-memory pool.
79///
80/// A constructed pool can be cloned and shared by several transfer-manager
81/// clients or other components. All clones use one admission ceiling,
82/// prepared-storage cache, reservation queue, and metrics domain.
83#[derive(Clone, Debug, Default)]
84pub struct BufferPoolBuilder {
85    memory_budget: MemoryBudgetConfig,
86}
87
88impl BufferPoolBuilder {
89    /// Sets the configured admission ceiling for the pool.
90    pub fn memory_budget(mut self, memory_budget: MemoryBudgetConfig) -> Self {
91        self.memory_budget = memory_budget;
92        self
93    }
94
95    /// Constructs an empty pool, detecting machine memory when the selected
96    /// capacity policy requires it.
97    ///
98    /// Construction fixes capacity and geometry but does not reserve virtual
99    /// ranges or prepare physical memory. It reads `AWS_S3_TM_DIAGNOSTICS`, and
100    /// starts the pool's maintenance thread only when that setting enables
101    /// periodic memory snapshots (`memory.snapshot`).
102    pub fn build(self) -> Result<super::BufferPool, BufferPoolBuildError> {
103        let needs_memory_detection = match self.memory_budget {
104            MemoryBudgetConfig::Auto => true,
105            MemoryBudgetConfig::Fraction(fraction)
106                if fraction.is_finite() && 0.0 < fraction && fraction <= 1.0 =>
107            {
108                true
109            }
110            MemoryBudgetConfig::Fraction(_) => {
111                return Err(BufferPoolBuildError::InvalidCapacity);
112            }
113            MemoryBudgetConfig::Limit(_) => false,
114        };
115        let detected_memory = needs_memory_detection
116            .then(crate::runtime::platform::available_ram)
117            .flatten();
118        let diagnostics = crate::config::DiagnosticsConfig::from_env().memory();
119        super::BufferPool::from_capacity(self.memory_budget, detected_memory, diagnostics)
120    }
121}
122
123/// Checked geometry and accounting inputs for constructing an empty pool.
124#[derive(Clone, Copy, Debug, Eq, PartialEq)]
125pub(super) struct PoolConfig {
126    /// Page, carrier, block, and bitmap dimensions.
127    pub(super) geometry: PoolGeometry,
128    /// Normal admission ceiling in complete carriers.
129    pub(super) configured_capacity: CarrierCount,
130    /// Optimistic bitmap words inspected before serialized fallback.
131    pub(super) optimistic_scan_words: usize,
132}
133
134impl PoolConfig {
135    /// Resolves capacity against detected memory and runtime page geometry.
136    pub(super) fn resolve(
137        capacity: MemoryBudgetConfig,
138        detected_memory: Option<usize>,
139    ) -> Result<Self, BufferPoolBuildError> {
140        let page_size = page_size().map_err(|_| BufferPoolBuildError::UnsupportedPageGeometry)?;
141        Self::resolve_for_page(capacity, detected_memory, page_size.get())
142    }
143
144    /// Resolves capacity against explicit page geometry.
145    pub(super) fn resolve_for_page(
146        capacity: MemoryBudgetConfig,
147        detected_memory: Option<usize>,
148        page_size: usize,
149    ) -> Result<Self, BufferPoolBuildError> {
150        let carrier_size = DEFAULT_CARRIER_BYTES
151            .div_ceil(page_size)
152            .checked_mul(page_size)
153            .ok_or(BufferPoolBuildError::CapacityOverflow)?;
154
155        let bytes = resolve_capacity_bytes(capacity, detected_memory)?;
156        let carriers = bytes / carrier_size;
157        if carriers == 0 {
158            return Err(BufferPoolBuildError::InvalidCapacity);
159        }
160        let configured_capacity = CarrierCount::new(carriers);
161        if configured_capacity > MAX_PACKED_CARRIERS {
162            return Err(BufferPoolBuildError::CapacityOverflow);
163        }
164        let target_block_carriers = (DEFAULT_BLOCK_BYTES / carrier_size).max(1);
165        let block_carriers = target_block_carriers.min(configured_capacity.get());
166        let block_size = carrier_size
167            .checked_mul(block_carriers)
168            .ok_or(BufferPoolBuildError::CapacityOverflow)?;
169        let geometry = PoolGeometry::new(page_size, block_size, carrier_size)
170            .map_err(|_| BufferPoolBuildError::UnsupportedPageGeometry)?;
171        let optimistic_scan_words = DEFAULT_OPTIMISTIC_SCAN_BYTES
172            .div_ceil(carrier_size)
173            .div_ceil(u64::BITS as usize);
174
175        Ok(Self {
176            geometry,
177            configured_capacity,
178            optimistic_scan_words,
179        })
180    }
181}
182
183/// Capacity in bytes that [`MemoryBudgetConfig::Auto`] selects, before carrier
184/// rounding.
185///
186/// A share of `detected_memory`, capped at [`AUTO_CAPACITY_MAX_BYTES`], or
187/// [`AUTO_CAPACITY_FALLBACK_BYTES`] when memory is undetected. Saturates where
188/// a bound does not fit `usize`.
189pub(crate) fn auto_capacity_bytes(detected_memory: Option<usize>) -> usize {
190    match detected_memory {
191        Some(bytes) => {
192            let maximum = usize::try_from(AUTO_CAPACITY_MAX_BYTES).unwrap_or(usize::MAX);
193            (bytes / AUTO_MEMORY_SHARE_DENOMINATOR).min(maximum)
194        }
195        None => usize::try_from(AUTO_CAPACITY_FALLBACK_BYTES).unwrap_or(usize::MAX),
196    }
197}
198
199/// Resolves one capacity policy to bytes before carrier rounding.
200fn resolve_capacity_bytes(
201    capacity: MemoryBudgetConfig,
202    detected_memory: Option<usize>,
203) -> Result<usize, BufferPoolBuildError> {
204    match capacity {
205        MemoryBudgetConfig::Auto => Ok(auto_capacity_bytes(detected_memory)),
206        MemoryBudgetConfig::Fraction(fraction) => {
207            if !(0.0 < fraction && fraction <= 1.0) {
208                return Err(BufferPoolBuildError::InvalidCapacity);
209            }
210            let detected =
211                detected_memory.ok_or(BufferPoolBuildError::MemoryDetectionUnavailable)?;
212            Ok(floor_fraction_of(detected, fraction))
213        }
214        MemoryBudgetConfig::Limit(bytes) => {
215            usize::try_from(bytes.as_u64()).map_err(|_| BufferPoolBuildError::CapacityOverflow)
216        }
217    }
218}
219
220/// Multiplies an integer by a checked fraction without rounding above it.
221///
222/// A normal binary64 value is `significand * 2^(exponent - 1023 - 52)`.
223/// Multiplying the integer significand in `u128` and shifting right therefore
224/// computes the exact floor instead of rounding the intermediate `f64`.
225fn floor_fraction_of(total: usize, fraction: f64) -> usize {
226    debug_assert!(0.0 < fraction && fraction <= 1.0);
227
228    let bits = fraction.to_bits();
229    let exponent = ((bits >> F64_FRACTION_BITS) & 0x7ff) as u32;
230    let stored_fraction = bits & ((1_u64 << F64_FRACTION_BITS) - 1);
231    let (significand, shift) = if exponent == 0 {
232        (
233            u128::from(stored_fraction),
234            F64_EXPONENT_BIAS + F64_FRACTION_BITS - 1,
235        )
236    } else {
237        (
238            u128::from(stored_fraction | (1_u64 << F64_FRACTION_BITS)),
239            F64_EXPONENT_BIAS + F64_FRACTION_BITS - exponent,
240        )
241    };
242    let product = (total as u128) * significand;
243    if shift >= u128::BITS {
244        return 0;
245    }
246    (product >> shift) as usize
247}
248
249#[cfg(test)]
250mod tests {
251    use super::*;
252    use crate::runtime::buffer_pool::BufferPool;
253    use crate::types::ByteSize;
254
255    const GIB: usize = 1024 * 1024 * 1024;
256
257    #[test]
258    fn test_auto_uses_quarter_of_detected_memory() {
259        let resolved =
260            PoolConfig::resolve_for_page(MemoryBudgetConfig::Auto, Some(2 * GIB), 4096).unwrap();
261        assert_eq!(
262            resolved.configured_capacity.get() * DEFAULT_CARRIER_BYTES,
263            GIB / 2
264        );
265    }
266
267    #[test]
268    #[cfg(target_pointer_width = "64")]
269    fn test_auto_caps_large_machines() {
270        let resolved =
271            PoolConfig::resolve_for_page(MemoryBudgetConfig::Auto, Some(768 * GIB), 4096).unwrap();
272        assert_eq!(
273            resolved.configured_capacity.get() * DEFAULT_CARRIER_BYTES,
274            32 * GIB
275        );
276    }
277
278    #[test]
279    fn test_auto_uses_fallback_without_detection() {
280        let resolved = PoolConfig::resolve_for_page(MemoryBudgetConfig::Auto, None, 4096).unwrap();
281        assert_eq!(
282            resolved.configured_capacity.get() * DEFAULT_CARRIER_BYTES,
283            2 * GIB
284        );
285    }
286
287    #[test]
288    fn test_fraction_requires_detection() {
289        assert_eq!(
290            PoolConfig::resolve_for_page(MemoryBudgetConfig::Fraction(0.5), None, 4096),
291            Err(BufferPoolBuildError::MemoryDetectionUnavailable)
292        );
293    }
294
295    #[test]
296    fn test_fraction_rejects_invalid_values() {
297        for fraction in [f64::NAN, f64::INFINITY, -0.5, 0.0, 1.5] {
298            assert_eq!(
299                PoolConfig::resolve_for_page(
300                    MemoryBudgetConfig::Fraction(fraction),
301                    Some(GIB),
302                    4096,
303                ),
304                Err(BufferPoolBuildError::InvalidCapacity)
305            );
306        }
307    }
308
309    #[test]
310    fn test_fraction_rejects_zero_before_capacity_resolution() {
311        assert_eq!(
312            resolve_capacity_bytes(MemoryBudgetConfig::Fraction(0.0), Some(GIB)),
313            Err(BufferPoolBuildError::InvalidCapacity)
314        );
315    }
316
317    #[test]
318    fn test_fraction_accepts_the_inclusive_upper_bound() {
319        assert_eq!(
320            resolve_capacity_bytes(MemoryBudgetConfig::Fraction(1.0), Some(GIB)),
321            Ok(GIB)
322        );
323    }
324
325    #[test]
326    fn test_fraction_rounds_down_to_complete_carriers() {
327        let resolved =
328            PoolConfig::resolve_for_page(MemoryBudgetConfig::Fraction(0.5), Some(GIB + 1), 4096)
329                .unwrap();
330        assert_eq!(
331            resolved.configured_capacity.get() * DEFAULT_CARRIER_BYTES,
332            GIB / 2
333        );
334    }
335
336    #[test]
337    fn test_fraction_does_not_round_above_the_exact_binary_fraction() {
338        let resolved = PoolConfig::resolve_for_page(
339            MemoryBudgetConfig::Fraction(0.3333333333333333),
340            Some(1_074_266_112),
341            4096,
342        )
343        .unwrap();
344        assert_eq!(resolved.configured_capacity, CarrierCount::new(21855));
345    }
346
347    #[test]
348    fn test_tiny_fraction_resolves_below_one_carrier_without_panicking() {
349        let capacity = MemoryBudgetConfig::Fraction(1e-30);
350        assert_eq!(resolve_capacity_bytes(capacity.clone(), Some(GIB)), Ok(0));
351        assert_eq!(
352            PoolConfig::resolve_for_page(capacity, Some(GIB), 4096),
353            Err(BufferPoolBuildError::InvalidCapacity)
354        );
355    }
356
357    #[test]
358    fn test_non_aligned_limit_cannot_fund_same_sized_reservation() {
359        // Capacity rounds down so it never exceeds the byte limit, while a
360        // reservation must cover its complete allocation unit.
361        let limit = DEFAULT_CARRIER_BYTES + 123;
362        let resolved = PoolConfig::resolve_for_page(
363            MemoryBudgetConfig::Limit(ByteSize::new(limit as u64)),
364            None,
365            4096,
366        )
367        .unwrap();
368        let envelope = resolved.geometry.carriers_for_bytes(limit).unwrap();
369
370        assert_eq!(resolved.configured_capacity, CarrierCount::new(1));
371        assert_eq!(envelope, CarrierCount::new(2));
372        assert!(envelope > resolved.configured_capacity);
373    }
374
375    #[test]
376    fn test_capacity_below_one_carrier_is_rejected() {
377        assert_eq!(
378            PoolConfig::resolve_for_page(
379                MemoryBudgetConfig::Limit(ByteSize::new((DEFAULT_CARRIER_BYTES - 1) as u64)),
380                None,
381                4096,
382            ),
383            Err(BufferPoolBuildError::InvalidCapacity)
384        );
385    }
386
387    #[test]
388    fn test_carrier_rounds_up_to_large_runtime_page() {
389        let resolved = PoolConfig::resolve_for_page(
390            MemoryBudgetConfig::Limit(ByteSize::new(256 * 1024)),
391            None,
392            128 * 1024,
393        )
394        .unwrap();
395        assert_eq!(resolved.geometry.carrier_size(), 128 * 1024);
396        assert_eq!(resolved.geometry.block_size(), 256 * 1024);
397        assert_eq!(resolved.configured_capacity, CarrierCount::new(2));
398    }
399
400    #[test]
401    fn test_block_size_is_independent_of_carrier_bitmap_width() {
402        let resolved = PoolConfig::resolve_for_page(
403            MemoryBudgetConfig::Limit(ByteSize::new(GIB as u64)),
404            None,
405            4096,
406        )
407        .unwrap();
408
409        assert_eq!(resolved.geometry.carrier_size(), 16 * 1024);
410        assert_eq!(resolved.geometry.block_size(), 128 * 1024 * 1024);
411        assert_eq!(
412            resolved.geometry.carriers_per_block(),
413            CarrierCount::new(8192)
414        );
415        assert_eq!(resolved.geometry.bitmap_words(), 128);
416    }
417
418    #[test]
419    fn test_optimistic_scan_preserves_its_byte_reach_across_page_geometry() {
420        for page_size in [4096, 128 * 1024] {
421            let resolved = PoolConfig::resolve_for_page(
422                MemoryBudgetConfig::Limit(ByteSize::new(GIB as u64)),
423                None,
424                page_size,
425            )
426            .unwrap();
427            let carrier_size = resolved.geometry.carrier_size();
428            let scan_bytes = resolved.optimistic_scan_words * u64::BITS as usize * carrier_size;
429
430            assert!(scan_bytes >= DEFAULT_OPTIMISTIC_SCAN_BYTES);
431            assert!(scan_bytes - DEFAULT_OPTIMISTIC_SCAN_BYTES < u64::BITS as usize * carrier_size);
432        }
433    }
434
435    #[test]
436    fn test_eight_mibibyte_claim_uses_optimistic_path_after_preparation() {
437        const PART_BYTES: usize = 8 * 1024 * 1024;
438
439        let pool = BufferPool::from_capacity(
440            MemoryBudgetConfig::Limit(ByteSize::new(GIB as u64)),
441            None,
442            crate::config::MemoryDiagnosticsConfig::default(),
443        )
444        .unwrap();
445        let prepared = pool
446            .acquire_unreserved(pool.inner.geometry.carrier_size())
447            .unwrap();
448        drop(prepared);
449        let fallbacks_before = pool.diagnostics().serialized_fallbacks;
450
451        let part = pool.acquire_unreserved(PART_BYTES).unwrap();
452
453        assert_eq!(pool.diagnostics().serialized_fallbacks, fallbacks_before);
454        drop(part);
455    }
456
457    #[test]
458    fn test_block_size_is_capped_by_small_configured_capacity() {
459        let resolved = PoolConfig::resolve_for_page(
460            MemoryBudgetConfig::Limit(ByteSize::new(8 * 1024 * 1024)),
461            None,
462            4096,
463        )
464        .unwrap();
465
466        assert_eq!(resolved.geometry.block_size(), 8 * 1024 * 1024);
467        assert_eq!(
468            resolved.geometry.carriers_per_block(),
469            resolved.configured_capacity
470        );
471    }
472
473    #[cfg(target_pointer_width = "64")]
474    #[test]
475    fn test_capacity_accepts_the_exact_packed_carrier_limit() {
476        let bytes = u32::MAX as usize * DEFAULT_CARRIER_BYTES;
477        let resolved = PoolConfig::resolve_for_page(
478            MemoryBudgetConfig::Limit(ByteSize::new(bytes as u64)),
479            None,
480            4096,
481        )
482        .unwrap();
483        assert_eq!(resolved.configured_capacity, MAX_PACKED_CARRIERS);
484    }
485
486    #[cfg(target_pointer_width = "64")]
487    #[test]
488    fn test_capacity_rejects_packed_carrier_overflow() {
489        let bytes = (u32::MAX as usize + 1) * DEFAULT_CARRIER_BYTES;
490        assert_eq!(
491            PoolConfig::resolve_for_page(
492                MemoryBudgetConfig::Limit(ByteSize::new(bytes as u64)),
493                None,
494                4096
495            ),
496            Err(BufferPoolBuildError::CapacityOverflow)
497        );
498    }
499
500    // `8 * GIB` exceeds a 32-bit `usize`; the reserved-vs-prepared split only
501    // needs an above-committed value, which only exists on 64-bit.
502    #[cfg(target_pointer_width = "64")]
503    #[test]
504    fn test_pool_construction_prepares_no_capacity() {
505        let pool = BufferPool::from_capacity(
506            MemoryBudgetConfig::Limit(ByteSize::new(GIB as u64)),
507            Some(8 * GIB),
508            crate::config::MemoryDiagnosticsConfig::default(),
509        )
510        .unwrap();
511        assert_eq!(pool.inner.geometry.carrier_size(), DEFAULT_CARRIER_BYTES);
512        assert_eq!(pool.inner.geometry.block_size(), DEFAULT_BLOCK_BYTES);
513        assert_eq!(pool.metrics().capacity_bytes(), GIB as u64);
514        assert_eq!(pool.metrics().prepared_capacity_bytes(), 0);
515    }
516
517    #[test]
518    fn test_public_builder_constructs_a_lazy_explicit_pool() {
519        let pool = BufferPool::builder()
520            .memory_budget(MemoryBudgetConfig::Limit(ByteSize::new(GIB as u64)))
521            .build()
522            .unwrap();
523
524        assert_eq!(pool.carrier_size(), DEFAULT_CARRIER_BYTES);
525        assert_eq!(pool.metrics().capacity_bytes(), GIB as u64);
526        assert_eq!(pool.metrics().prepared_capacity_bytes(), 0);
527    }
528
529    #[test]
530    fn test_public_builder_rejects_invalid_capacity() {
531        for fraction in [f64::NAN, f64::NEG_INFINITY, -0.1, 0.0, 1.1, f64::INFINITY] {
532            assert_eq!(
533                BufferPool::builder()
534                    .memory_budget(MemoryBudgetConfig::Fraction(fraction))
535                    .build()
536                    .unwrap_err(),
537                BufferPoolBuildError::InvalidCapacity
538            );
539        }
540        assert_eq!(
541            BufferPool::builder()
542                .memory_budget(MemoryBudgetConfig::Limit(ByteSize::new(1)))
543                .build()
544                .unwrap_err(),
545            BufferPoolBuildError::InvalidCapacity
546        );
547    }
548}