znippy_common/
common_config.rs

1use crate::int_ring::MINI_SIZE;
2use crate::{ChunkQueue, RingBuffer, int_ring};
3use once_cell::sync::Lazy;
4use std::cmp::{max, min};
5use sysinfo::{MemoryRefreshKind, RefreshKind, System};
6
7impl StrategicConfig {
8    pub fn file_split_block_size_usize(&self) -> usize {
9        self.file_split_block_size.try_into().unwrap()
10    }
11}
12#[derive(Debug, Clone)] //
13pub struct StrategicConfig {
14    pub max_core_allowed: usize,
15    pub max_core_in_flight: usize,
16    pub max_core_in_compress: usize,
17    pub max_mem_allowed: u64,
18    pub min_free_memory_ratio: f32,
19    pub file_split_block_size: u64,
20    pub max_chunks: u32,
21    pub compression_level: i32,
22    pub zstd_output_buffer_size: usize,
23}
24
25pub static CONFIG: Lazy<StrategicConfig> = Lazy::new(strategic_confi_large);
26
27pub fn strategic_config(resource: f32) -> StrategicConfig {
28    let refresh = RefreshKind::everything().with_memory(MemoryRefreshKind::everything());
29    let mut sys = System::new_with_specifics(refresh);
30    sys.refresh_memory();
31
32    let total_memory = sys.total_memory();
33    let max_core_allowed =
34        sysinfo::System::physical_core_count().unwrap_or_else(|| sys.cpus().len());
35
36    let max_core_in_flight = ((max_core_allowed as f32) * 0.90).ceil() as usize;
37    let max_core_in_compress = max_core_allowed.saturating_sub(max_core_in_flight);
38    let min_free_memory_ratio = 1.0 - resource;
39    let compression_level = 19;
40    let max_mem_allowed = ((total_memory as f32) * (1.0 - min_free_memory_ratio)) as u64;
41    let file_split_block_size = 10 * 1024 * 1024;
42    let zstd_output_buffer_size = 1 * 1024 * 1024;
43
44    let mut max_chunks: u32 = (max_mem_allowed / file_split_block_size) as u32;
45
46    log::info!(
47        "[strategic_config] Detekterade {} kärnor och {} MiB minne",
48        max_core_allowed,
49        total_memory / 1024
50    );
51
52    let sc = StrategicConfig {
53        max_core_allowed,
54        max_core_in_flight,
55        max_core_in_compress,
56        max_mem_allowed,
57        min_free_memory_ratio,
58        file_split_block_size,
59        compression_level,
60        max_chunks,
61        zstd_output_buffer_size,
62    };
63    log_strategic_conf(&sc);
64    sc
65}
66
67fn strategic_config_mini() -> StrategicConfig {
68    let mut sc = strategic_config(0.5);
69
70    // Downscale to mini
71    sc.max_chunks = min(sc.max_chunks as u64, int_ring::MINI_SIZE as u64) as u32;
72    sc.max_core_in_flight = min(sc.max_chunks as usize, sc.max_core_in_flight);
73    // TODO could add extra cores to zstd .. i guess.
74
75    log::info!(
76        "[strategic_config mini]  max_core_in_flight={}  max_core_in_compress for zstd {} max_chunks {} ",
77        sc.max_core_in_flight,
78        sc.max_core_in_compress,
79        sc.max_chunks
80    );
81    log_strategic_conf(&sc);
82    sc
83}
84
85fn strategic_confi_large() -> StrategicConfig {
86    let mut sc = strategic_config(1.0);
87
88    // scale to large
89    sc.max_chunks = min(sc.max_chunks as u64, int_ring::LARGE_SIZE as u64) as u32;
90    log::info!(
91        "[strategic_config large]  max_core_in_flight={}  max_core_in_compress for zstd {} max_chunks {} ",
92        sc.max_core_in_flight,
93        sc.max_core_in_compress,
94        sc.max_chunks
95    );
96    log_strategic_conf(&sc);
97    sc
98}
99
100fn log_strategic_conf(sc: &StrategicConfig) {
101    log::info!(
102        "[strategic_config] max_core_in_flight: {} (10%)",
103        sc.max_core_in_flight
104    );
105
106    log::info!(
107        "[strategic_config] max_core_in_compress: {} (90%)",
108        sc.max_core_in_compress
109    );
110    log::info!(
111        "[strategic_config] min_free_memory_ratio: {:.0}%",
112        sc.min_free_memory_ratio * 100.0
113    );
114    log::info!(
115        "[strategic_config] compression_level: {}",
116        sc.compression_level
117    );
118
119    log::info!("[strategic_config] max_chunks: {}", sc.max_chunks);
120
121    log::info!(
122        "[strategic_config] zstd_output_buffer_size: {}",
123        sc.zstd_output_buffer_size
124    );
125}
znippy_common/common_config.rs

znippy_common/
common_config.rs
