xet_runtime/config/groups/
reconstruction.rs

1use std::time::Duration;
2
3use crate::utils::ByteSize;
4
5crate::config_group!({
6
7    /// The minimum size of a single fetch request during reconstruction.
8    /// Individual fetches will request reconstruction terms representing at least this amount of data.
9    ///
10    /// The default value is 256MB.
11    ///
12    /// Use the environment variable `HF_XET_RECONSTRUCTION_MIN_RECONSTRUCTION_FETCH_SIZE` to set this value.
13    ref min_reconstruction_fetch_size: ByteSize = ByteSize::from("256mb");
14
15    /// The maximum size of a single fetch request during reconstruction.
16    /// Individual fetches will not request reconstruction terms representing more than this amount of data.
17    ///
18    /// The default value is 8GB.
19    ///
20    /// Use the environment variable `HF_XET_RECONSTRUCTION_MAX_RECONSTRUCTION_FETCH_SIZE` to set this value.
21    ref max_reconstruction_fetch_size: ByteSize = ByteSize::from("8gb");
22
23    /// The amount of download buffer always available for file reconstruction.
24    /// The full buffer size will be this plus the number of simultaneous active
25    /// file downloads times the per file size up to the global limit of
26    /// download_buffer_limit.
27    ///
28    /// The default value is 2GB.
29    ///
30    /// Use the environment variable `HF_XET_RECONSTRUCTION_DOWNLOAD_BUFFER_SIZE` to set this value.
31    ref download_buffer_size: ByteSize = ByteSize::from("2gb");
32
33    /// The additional download buffer allocated per active file download.
34    /// Each active file download increases the total buffer by this amount.
35    ///
36    /// The default value is 512MB.
37    ///
38    /// Use the environment variable `HF_XET_RECONSTRUCTION_DOWNLOAD_BUFFER_PERFILE_SIZE` to set this value.
39    ref download_buffer_perfile_size: ByteSize = ByteSize::from("512mb");
40
41    /// The maximum total download buffer allowed during file reconstruction.
42    /// The buffer will not grow beyond this limit regardless of the number of concurrent downloads.
43    ///
44    /// The default value is 8GB.
45    ///
46    /// Use the environment variable `HF_XET_RECONSTRUCTION_DOWNLOAD_BUFFER_LIMIT` to set this value.
47    ref download_buffer_limit: ByteSize = ByteSize::from("8gb");
48
49    /// The half-life in count of observations for the exponentially weighted moving average used to estimate
50    /// completion rate during reconstruction prefetching.
51    ///
52    /// The default value is 4 observations..
53    ///
54    /// Use the environment variable `HF_XET_RECONSTRUCTION_COMPLETION_RATE_ESTIMATOR_HALF_LIFE` to set this value.
55    ref completion_rate_estimator_half_life: f64 = 4.;
56
57    /// The target time for completing a prefetch block during reconstruction.
58    /// This is used to determine how much data to prefetch ahead.
59    ///
60    /// The default value is 15 minutes.
61    ///
62    /// Use the environment variable `HF_XET_RECONSTRUCTION_TARGET_BLOCK_COMPLETION_TIME` to set this value.
63    ref target_block_completion_time: Duration = Duration::from_secs(15 * 60);
64
65    /// The minimum size of the prefetch buffer during reconstruction.
66    /// The prefetch system will maintain terms representing at least this much always prefetched,
67    /// no matter the estimated completion time.
68    ///
69    /// The default value is 1gb.
70    ///
71    /// Use the environment variable `HF_XET_RECONSTRUCTION_MIN_PREFETCH_BUFFER` to set this value.
72    ref min_prefetch_buffer: ByteSize = ByteSize::from("1gb");
73
74    /// Whether to use vectorized writes (write_vectored) during file reconstruction.
75    /// When true, multiple pending writes are batched and written using write_vectored.
76    /// When false, standard sequential writes are used.
77    ///
78    /// The default value is true.
79    ///
80    /// Use the environment variable `HF_XET_RECONSTRUCTION_USE_VECTORED_WRITE` to set this value.
81    ref use_vectored_write: bool = true;
82
83});
xet_runtime/config/groups/reconstruction.rs

xet_runtime/config/groups/
reconstruction.rs
