znippy_common/
common_config.rs

1use crate::hostinfo;
2use once_cell::sync::Lazy;
3use std::cmp::min;
4
5#[derive(Debug, Clone)] //
6pub struct StrategicConfig {
7    pub max_core_allowed: usize,
8    pub max_core_in_flight: usize,
9    pub max_core_in_compress: usize,
10    pub max_mem_allowed: u64,
11    pub min_free_memory_ratio: f32,
12    pub file_split_block_size: u64,
13    pub max_chunks: u32,
14    pub compression_level: i32,
15    pub zstd_output_buffer_size: usize,
16}
17
18pub static CONFIG: Lazy<StrategicConfig> = Lazy::new(strategic_confi_large);
19
20/// Fraction of the memory actually available to this process that the compress
21/// slot pool may reserve. The pool is one of several allocations a compress run
22/// makes (per-worker zstd/OpenZL scratch, the index batches, the OS page cache),
23/// so it must not claim everything that is free.
24const SLOT_POOL_MEM_FRACTION: f64 = 0.5;
25
26/// Bytes the compress slot pool is allowed to reserve on this host **right now**.
27///
28/// The pool used to be a hardcoded `8 × 200 MiB = 1.6 GiB` reserved before a
29/// single file was opened, independent of both the input size and the memory the
30/// process actually has. On a backup pod with a modest limit that is an
31/// OOM-kill. This is the memory half of the bound (`PoolPlan::plan` applies the
32/// input half).
33///
34/// Order of precedence:
35///  1. `ZNIPPY_SLOT_POOL_BYTES` — an explicit ops override, accepting a plain
36///     byte count or a `KiB`/`MiB`/`GiB` (also `KB`/`MB`/`GB`) suffix.
37///  2. The **cgroup** limit minus what the cgroup is already using — this is the
38///     number that actually kills a container, and it is not the host's free RAM.
39///  3. The host's available memory.
40///
41/// A fraction ([`SLOT_POOL_MEM_FRACTION`]) of whichever bound applies is
42/// returned. Never 0: `PoolPlan` floors the pool at one slice regardless, so a
43/// pathological reading cannot deadlock the pipeline — it just makes it slow.
44///
45/// # Tier 2 was dead until 2026-08-04
46///
47/// This used `sysinfo::System::cgroup_limits()`, which is `limits_for_system()` —
48/// it reads the cgroup-v2 **root** (`/sys/fs/cgroup/memory.max`,
49/// `/sys/fs/cgroup/memory.current`). The root cgroup has no memory-controller
50/// files of its own, so that call returns `None` on a standard cgroup-v2 host
51/// *even when this process sits under a hard limit*. Tier 2 was documented,
52/// green, and unreachable; every run fell through to tier 3, the host's free RAM
53/// — precisely the number the doc comment says is the wrong one in a container.
54/// [`hostinfo::cgroup_memory`] follows `/proc/self/cgroup` instead, so the tier
55/// now fires. Measured on oden 2026-08-04: sysinfo `None`, hostinfo
56/// `total_memory = 472446402560`.
57pub fn slot_pool_budget_bytes() -> u64 {
58    if let Some(explicit) = std::env::var("ZNIPPY_SLOT_POOL_BYTES")
59        .ok()
60        .and_then(|raw| parse_byte_size(&raw))
61    {
62        log::info!("[slot_pool] budget {explicit} bytes (ZNIPPY_SLOT_POOL_BYTES)");
63        return explicit;
64    }
65
66    let host_available = hostinfo::mem_available_bytes().unwrap_or(0);
67    let headroom = match hostinfo::cgroup_memory() {
68        // In a container the cgroup ceiling — not the host's free RAM — is what
69        // the OOM killer measures against.
70        Some(cg) => cg.total_memory.saturating_sub(cg.rss).min(host_available.max(1)),
71        None => host_available,
72    };
73
74    let budget = (headroom as f64 * SLOT_POOL_MEM_FRACTION) as u64;
75    log::info!(
76        "[slot_pool] budget {} bytes ({:.0}% of {} bytes headroom)",
77        budget,
78        SLOT_POOL_MEM_FRACTION * 100.0,
79        headroom
80    );
81    budget
82}
83
84/// Parse `"268435456"`, `"256MiB"`, `"1 GiB"`, `"512MB"` → bytes.
85/// Returns `None` for anything it cannot read, so a typo falls back to detection
86/// rather than silently reserving something absurd.
87pub fn parse_byte_size(raw: &str) -> Option<u64> {
88    let s = raw.trim();
89    let (digits, unit) = s.split_at(s.find(|c: char| !c.is_ascii_digit()).unwrap_or(s.len()));
90    let n: u64 = digits.parse().ok()?;
91    let mult = match unit.trim().to_ascii_lowercase().as_str() {
92        "" | "b" => 1u64,
93        "k" | "kb" | "kib" => 1024,
94        "m" | "mb" | "mib" => 1024 * 1024,
95        "g" | "gb" | "gib" => 1024 * 1024 * 1024,
96        _ => return None,
97    };
98    n.checked_mul(mult)
99}
100
101pub fn strategic_config(resource: f32) -> StrategicConfig {
102    let total_memory = hostinfo::mem_total_bytes().unwrap_or(0);
103    // SMT siblings share an execution unit, so physical cores is the honest
104    // width for a compress pipeline; logical count is the fallback when
105    // /proc/cpuinfo carries no topology (most non-x86).
106    let max_core_allowed =
107        hostinfo::physical_core_count().unwrap_or_else(hostinfo::logical_cpu_count);
108
109    let max_core_in_flight = ((max_core_allowed as f32) * 0.90).ceil() as usize;
110    let max_core_in_compress = max_core_allowed.saturating_sub(max_core_in_flight);
111    let min_free_memory_ratio = 1.0 - resource;
112    let compression_level = 19;
113    let max_mem_allowed = ((total_memory as f32) * (1.0 - min_free_memory_ratio)) as u64;
114    let file_split_block_size = 10 * 1024 * 1024;
115    let zstd_output_buffer_size = 1 * 1024 * 1024;
116
117    let max_chunks: u32 = (max_mem_allowed / file_split_block_size) as u32;
118
119    log::info!(
120        "[strategic_config] Detekterade {} kärnor och {} MiB minne",
121        max_core_allowed,
122        // `total_memory` is BYTES. Dividing by 1024 printed KiB under a "MiB"
123        // label — off by 1024 for as long as the line has existed.
124        total_memory / (1024 * 1024)
125    );
126
127    let sc = StrategicConfig {
128        max_core_allowed,
129        max_core_in_flight,
130        max_core_in_compress,
131        max_mem_allowed,
132        min_free_memory_ratio,
133        file_split_block_size,
134        compression_level,
135        max_chunks,
136        zstd_output_buffer_size,
137    };
138    log_strategic_conf(&sc);
139    sc
140}
141
142fn strategic_confi_large() -> StrategicConfig {
143    let mut sc = strategic_config(1.0);
144
145    // Cap at the legacy LARGE_SIZE (128 slots × 10 MB) — stored in archive metadata for compat.
146    sc.max_chunks = min(sc.max_chunks as u64, 128) as u32;
147    log::info!(
148        "[strategic_config large]  max_core_in_flight={}  max_core_in_compress for zstd {} max_chunks {} ",
149        sc.max_core_in_flight,
150        sc.max_core_in_compress,
151        sc.max_chunks
152    );
153    log_strategic_conf(&sc);
154    sc
155}
156
157fn log_strategic_conf(sc: &StrategicConfig) {
158    log::info!(
159        "[strategic_config] max_core_in_flight: {} (10%)",
160        sc.max_core_in_flight
161    );
162
163    log::info!(
164        "[strategic_config] max_core_in_compress: {} (90%)",
165        sc.max_core_in_compress
166    );
167    log::info!(
168        "[strategic_config] min_free_memory_ratio: {:.0}%",
169        sc.min_free_memory_ratio * 100.0
170    );
171    log::info!(
172        "[strategic_config] compression_level: {}",
173        sc.compression_level
174    );
175
176    log::info!("[strategic_config] max_chunks: {}", sc.max_chunks);
177
178    log::info!(
179        "[strategic_config] zstd_output_buffer_size: {}",
180        sc.zstd_output_buffer_size
181    );
182}
znippy_common/common_config.rs

znippy_common/
common_config.rs
