xet_runtime/config/groups/
reconstruction.rs

1use std::time::Duration;
2
3use crate::utils::ByteSize;
4
5crate::config_group!({
6
7    /// The minimum size of a single fetch request during reconstruction.
8    /// Individual fetches will request reconstruction terms representing at least this amount of data.
9    ///
10    /// The default value is 256MB.
11    ///
12    /// Use the environment variable `HF_XET_RECONSTRUCTION_MIN_RECONSTRUCTION_FETCH_SIZE` to set this value.
13    ref min_reconstruction_fetch_size: ByteSize = ByteSize::from("256mb");
14
15    /// The maximum size of a single fetch request during reconstruction.
16    /// Individual fetches will not request reconstruction terms representing more than this amount of data.
17    ///
18    /// The default value is 8GB.
19    ///
20    /// Use the environment variable `HF_XET_RECONSTRUCTION_MAX_RECONSTRUCTION_FETCH_SIZE` to set this value.
21    ref max_reconstruction_fetch_size: ByteSize = ByteSize::from("8gb");
22
23    /// The amount of download buffer always available for file reconstruction.
24    /// The full buffer size will be this plus the number of simultaneous active
25    /// file downloads times the per file size up to the global limit of
26    /// download_buffer_limit.
27    ///
28    /// The default is derived from usable memory (the minimum of host RAM and the
29    /// effective cgroup limit): usable/16, clamped to [64MB, 16GB]. If memory cannot
30    /// be determined, or derivation is disabled via
31    /// `HF_XET_DISABLE_MEMORY_DERIVED_DOWNLOAD_BUFFERS=1`, the default is 2GB.
32    ///
33    /// Use the environment variable `HF_XET_RECONSTRUCTION_DOWNLOAD_BUFFER_SIZE` to set this value.
34    ref download_buffer_size: ByteSize = crate::utils::system_memory::default_download_buffer_sizes().size;
35
36    /// The additional download buffer allocated per active file download.
37    /// Each active file download increases the total buffer by this amount.
38    ///
39    /// The default is derived from usable memory: usable/64, clamped to [16MB, 2GB].
40    /// If memory cannot be determined, or derivation is disabled via
41    /// `HF_XET_DISABLE_MEMORY_DERIVED_DOWNLOAD_BUFFERS=1`, the default is 512MB.
42    ///
43    /// Use the environment variable `HF_XET_RECONSTRUCTION_DOWNLOAD_BUFFER_PERFILE_SIZE` to set this value.
44    ref download_buffer_perfile_size: ByteSize = crate::utils::system_memory::default_download_buffer_sizes().perfile;
45
46    /// The maximum total download buffer allowed during file reconstruction.
47    /// The buffer will not grow beyond this limit regardless of the number of concurrent downloads.
48    ///
49    /// The default is derived from usable memory: usable/4, clamped to [264MB, 64GB].
50    /// If memory cannot be determined, or derivation is disabled via
51    /// `HF_XET_DISABLE_MEMORY_DERIVED_DOWNLOAD_BUFFERS=1`, the default is 8GB.
52    ///
53    /// Use the environment variable `HF_XET_RECONSTRUCTION_DOWNLOAD_BUFFER_LIMIT` to set this value.
54    ref download_buffer_limit: ByteSize = crate::utils::system_memory::default_download_buffer_sizes().limit;
55
56    /// The half-life in count of observations for the exponentially weighted moving average used to estimate
57    /// completion rate during reconstruction prefetching.
58    ///
59    /// The default value is 4 observations..
60    ///
61    /// Use the environment variable `HF_XET_RECONSTRUCTION_COMPLETION_RATE_ESTIMATOR_HALF_LIFE` to set this value.
62    ref completion_rate_estimator_half_life: f64 = 4.;
63
64    /// The target time for completing a prefetch block during reconstruction.
65    /// This is used to determine how much data to prefetch ahead.
66    ///
67    /// The default value is 15 minutes.
68    ///
69    /// Use the environment variable `HF_XET_RECONSTRUCTION_TARGET_BLOCK_COMPLETION_TIME` to set this value.
70    ref target_block_completion_time: Duration = Duration::from_secs(15 * 60);
71
72    /// The minimum size of the prefetch buffer during reconstruction.
73    /// The prefetch system will maintain terms representing at least this much always prefetched,
74    /// no matter the estimated completion time.
75    ///
76    /// The default value is 1gb.
77    ///
78    /// Use the environment variable `HF_XET_RECONSTRUCTION_MIN_PREFETCH_BUFFER` to set this value.
79    ref min_prefetch_buffer: ByteSize = ByteSize::from("1gb");
80
81    /// Whether to use vectorized writes (write_vectored) during file reconstruction.
82    /// When true, multiple pending writes are batched and written using write_vectored.
83    /// When false, standard sequential writes are used.
84    ///
85    /// The default value is true.
86    ///
87    /// Use the environment variable `HF_XET_RECONSTRUCTION_USE_VECTORED_WRITE` to set this value.
88    ref use_vectored_write: bool = true;
89
90});
91
92impl ConfigValueGroup {
93    /// Ensure the download buffer values are mutually coherent: `download_buffer_limit`
94    /// must be at least `download_buffer_size`, since the buffer semaphore uses the two
95    /// as its (floor, ceiling) bounds. A user raising only the size (e.g. via the env
96    /// var) gets the limit raised to match rather than a panic at context construction.
97    pub fn normalize(&mut self) {
98        if self.download_buffer_limit < self.download_buffer_size {
99            tracing::warn!(
100                "download_buffer_limit ({}) is below download_buffer_size ({}); raising the limit to match",
101                self.download_buffer_limit,
102                self.download_buffer_size
103            );
104            self.download_buffer_limit = self.download_buffer_size;
105        }
106    }
107}
xet_runtime/config/groups/reconstruction.rs

xet_runtime/config/groups/
reconstruction.rs
