xet_runtime/config/groups/
shard.rs

1use crate::utils::ByteSize;
2
3crate::config_group!({
4
5    /// The (soft) maximum size in bytes of the shard cache.  Default is 16 GB.
6    ///
7    /// As a rough calculation, a cache of size X will allow for dedup against data
8    /// of size 1000 * X.  The default would allow a 16 TB repo to be deduped effectively.
9    ///
10    /// Note the cache is pruned to below this value at the beginning of a session,
11    /// but during a single session new shards may be added such that this limit is exceeded.
12    ///
13    /// The default value is 16gb.
14    ///
15    /// Use the environment variable `HF_XET_SHARD_CACHE_SIZE_LIMIT` to set this value.
16    ref cache_size_limit : ByteSize = ByteSize::from("16gb");
17
18    /// The maximum size of the chunk index table that's stored in memory.  After this,
19    /// no new chunks are loaded for deduplication.
20    ///
21    /// The default value is 67108864.
22    ///
23    /// Use the environment variable `HF_XET_SHARD_CHUNK_INDEX_TABLE_MAX_SIZE` to set this value.
24    ref chunk_index_table_max_size: usize = 64 * 1024 * 1024;
25
26    /// Subdirectory name for shard cache within the endpoint cache directory.
27    ///
28    /// The default value is "shard-cache".
29    ///
30    /// Use the environment variable `HF_XET_SHARD_CACHE_SUBDIR` to set this value.
31    ref cache_subdir: String = "shard-cache".to_string();
32
33});
xet_runtime/config/groups/shard.rs

xet_runtime/config/groups/
shard.rs
