start_command/
cgroup_memory.rs

1//! cgroup v2 memory counters of a detached docker container (issue #182).
2//!
3//! Docker's `State.OOMKilled` is container-wide and sticky (moby/moby#43564):
4//! it says that *some* process in the container was OOM-killed at some point,
5//! but not how many, not whether the container hit its own `--memory` limit or
6//! the whole host ran out, and not how close the execution came to its limit.
7//! The kernel keeps all three per cgroup
8//! (<https://docs.kernel.org/admin-guide/cgroup-v2.html#memory-interface-files>):
9//!
10//! ```text
11//! memory.events `oom`       the cgroup hit its own limit
12//! memory.events `oom_kill`  processes killed here by *any* OOM killer, so
13//!                           `oom_kill > oom` points to a host-wide (or parent
14//!                           cgroup) OOM
15//! memory.peak / memory.max  peak usage (Linux 5.19+) and limit (`max` when
16//!                           unlimited)
17//! ```
18//!
19//! The cgroup is removed when the container stops, so the values cannot be read
20//! afterwards. The detached completion watcher therefore samples them while the
21//! container runs and hands the last sample to the post-mortem, the finalizer
22//! and the recovery step. Best effort by design: no cgroup v2, a remote docker
23//! daemon or a hidden `/proc` leave the counters unknown, and a kill in the last
24//! sampling interval can be missed when the cgroup is gone before the final
25//! read.
26
27use serde::{Deserialize, Deserializer, Serialize};
28use serde_json::Value;
29
30use crate::isolation::isolation_log::shell_quote;
31
32/// Seconds between two samples of a running container's cgroup.
33pub const CGROUP_SAMPLE_INTERVAL_SECONDS: u64 = 1;
34
35/// Shell variables used by the sampler snippets.
36pub mod cgroup_shell_vars {
37    /// Last sample: `<memory.max> <memory.peak> <oom> <oom_kill>`.
38    pub const SAMPLE: &str = "__start_command_cgroup";
39    pub const SAMPLER: &str = "__start_command_cgroup_sampler";
40    pub const FILE: &str = "__start_command_cgroup_file";
41}
42
43/// Where the OOM killer that killed processes in the container came from.
44#[derive(Debug, Clone, Copy, PartialEq, Eq)]
45pub enum OomScope {
46    /// `oom_kill <= oom`: the container hit its own memory limit.
47    ContainerLimit,
48    /// `oom_kill > oom`: a host-wide or parent cgroup OOM killed processes.
49    HostOrParent,
50}
51
52impl OomScope {
53    pub fn as_str(&self) -> &'static str {
54        match self {
55            OomScope::ContainerLimit => "container-limit",
56            OomScope::HostOrParent => "host-or-parent",
57        }
58    }
59
60    pub fn note(&self) -> &'static str {
61        match self {
62            OomScope::ContainerLimit => "the container hit its own memory limit",
63            OomScope::HostOrParent => {
64                "oom_kill > oom: a host-wide or parent cgroup OOM killed processes here"
65            }
66        }
67    }
68}
69
70/// The counters of one sample. `limit_bytes` is `None` without a limit.
71#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
72#[serde(rename_all = "camelCase")]
73pub struct CgroupMemory {
74    pub limit_bytes: Option<u64>,
75    pub peak_bytes: Option<u64>,
76    pub oom_events: Option<u64>,
77    pub oom_kills: Option<u64>,
78}
79
80/// Shell functions shared by the sampler and its final read:
81///
82/// ```text
83/// __start_command_cgroup_dir NAME  print the container's cgroup v2 directory
84/// __start_command_cgroup_read DIR  print `<max> <peak> <oom> <oom_kill> DIR`
85/// ```
86///
87/// The directory comes from `/proc/<.State.Pid>/cgroup` (`0::<path>`), which
88/// covers the systemd and cgroupfs drivers, `--cgroup-parent` and rootless
89/// docker alike; the two default layouts are fallbacks. A candidate must name
90/// the container ID, so a PID reported by a remote daemon can never select an
91/// unrelated local cgroup. `START_COMMAND_CGROUP_ROOT` and
92/// `START_COMMAND_PROC_ROOT` replace `/sys/fs/cgroup` and `/proc` (tests).
93pub fn build_cgroup_functions_snippet() -> String {
94    [
95        concat!(
96            "__start_command_cgroup_dir() { __scd_id=$(docker inspect -f '{{.Id}}' \"$1\" 2>/dev/null); ",
97            "[ -n \"$__scd_id\" ] || return 1; ",
98            "__scd_pid=$(docker inspect -f '{{.State.Pid}}' \"$1\" 2>/dev/null); __scd_rel=''; ",
99            "case \"$__scd_pid\" in ''|0|*[!0-9]*) ;; ",
100            "*) __scd_rel=$(sed -n 's/^0:://p' \"${START_COMMAND_PROC_ROOT:-/proc}/$__scd_pid/cgroup\" 2>/dev/null);; esac; ",
101            "__scd_root=${START_COMMAND_CGROUP_ROOT:-/sys/fs/cgroup}; ",
102            "for __scd_dir in ${__scd_rel:+\"$__scd_root$__scd_rel\"} ",
103            "\"$__scd_root/system.slice/docker-$__scd_id.scope\" \"$__scd_root/docker/$__scd_id\"; do ",
104            "case \"$__scd_dir\" in *\"$__scd_id\"*) if [ -r \"$__scd_dir/memory.events\" ]; then ",
105            "printf '%s' \"$__scd_dir\"; return 0; fi;; esac; done; return 1; }",
106        ),
107        concat!(
108            "__start_command_cgroup_read() { __scr_events=$(cat \"$1/memory.events\" 2>/dev/null) || return 1; ",
109            "__scr_oom=$(printf '%s\\n' \"$__scr_events\" | sed -n 's/^oom //p'); ",
110            "__scr_kill=$(printf '%s\\n' \"$__scr_events\" | sed -n 's/^oom_kill //p'); ",
111            "__scr_max=$(cat \"$1/memory.max\" 2>/dev/null); __scr_peak=$(cat \"$1/memory.peak\" 2>/dev/null); ",
112            "printf '%s %s %s %s %s\\n' \"${__scr_max:--}\" \"${__scr_peak:--}\" ",
113            "\"${__scr_oom:--}\" \"${__scr_kill:--}\" \"$1\"; }",
114        ),
115    ]
116    .join("; ")
117}
118
119/// Shell fragment starting the background sampler. It writes each sample to a
120/// temp file with `mv`, so the watcher never reads a half-written line, and
121/// ends by itself once the cgroup is gone.
122pub fn build_cgroup_sampler_start_snippet(container_name: &str) -> String {
123    use cgroup_shell_vars::{FILE, SAMPLER};
124    [
125        build_cgroup_functions_snippet(),
126        format!("{}=\"${{TMPDIR:-/tmp}}/start-command-cgroup.$$\"", FILE),
127        format!("rm -f \"${f}\" \"${f}.tmp\"", f = FILE),
128        format!(
129            "( __scs_dir=$(__start_command_cgroup_dir {name}) || exit 0; \
130             while __scs_line=$(__start_command_cgroup_read \"$__scs_dir\"); do \
131             printf '%s\\n' \"$__scs_line\" > \"${f}.tmp\" && mv -f \"${f}.tmp\" \"${f}\"; \
132             sleep {interval}; done ) >/dev/null 2>&1 & {sampler}=$!",
133            name = shell_quote(container_name),
134            f = FILE,
135            interval = CGROUP_SAMPLE_INTERVAL_SECONDS,
136            sampler = SAMPLER,
137        ),
138    ]
139    .join("; ")
140}
141
142/// Shell fragment stopping the sampler once the container has exited and
143/// leaving the last sample (`<max> <peak> <oom> <oom_kill>`, or empty) in
144/// `$__start_command_cgroup`. The cgroup can outlive the main process for a
145/// moment, so it is read one last time when it still exists.
146pub fn build_cgroup_sampler_stop_snippet() -> String {
147    use cgroup_shell_vars::{FILE, SAMPLE, SAMPLER};
148    [
149        format!("kill \"${}\" 2>/dev/null", SAMPLER),
150        format!("wait \"${}\" 2>/dev/null", SAMPLER),
151        format!("{}=$(cat \"${}\" 2>/dev/null)", SAMPLE, FILE),
152        format!("__scs_dir=$(printf '%s' \"${}\" | cut -s -d ' ' -f 5-)", SAMPLE),
153        format!(
154            "if [ -n \"$__scs_dir\" ] && __scs_line=$(__start_command_cgroup_read \"$__scs_dir\"); then {}=$__scs_line; fi",
155            SAMPLE
156        ),
157        format!("rm -f \"${f}\" \"${f}.tmp\"", f = FILE),
158        format!(
159            "{s}=$(printf '%s' \"${s}\" | cut -d ' ' -f 1-4)",
160            s = SAMPLE
161        ),
162    ]
163    .join("; ")
164}
165
166/// Shell fragment appending the `Memory:` line of the post-mortem when a
167/// sample exists. `quoted_log_path` must already be shell-quoted.
168pub fn build_cgroup_memory_log_snippet(quoted_log_path: &str) -> String {
169    let sample = cgroup_shell_vars::SAMPLE;
170    // A function, so `$1`.. of the watcher script itself stay untouched.
171    format!(
172        "__start_command_cgroup_log() {{ __scm_note=''; \
173         if [ \"$4\" -gt 0 ] 2>/dev/null; then if [ \"$4\" -gt \"$3\" ] 2>/dev/null; then \
174         __scm_note=' ({host})'; \
175         else __scm_note=' ({container})'; fi; fi; \
176         printf 'Memory:     memory.max=%s memory.peak=%s oom=%s oom_kill=%s%s\\n' \
177         \"$1\" \"$2\" \"$3\" \"$4\" \"$__scm_note\"; }}; \
178         if [ -n \"${sample}\" ]; then __start_command_cgroup_log ${sample} >> {log}; fi",
179        host = OomScope::HostOrParent.note(),
180        container = OomScope::ContainerLimit.note(),
181        sample = sample,
182        log = quoted_log_path,
183    )
184}
185
186fn parse_counter(text: &str) -> Option<u64> {
187    if !text.is_empty() && text.chars().all(|c| c.is_ascii_digit()) {
188        text.parse().ok()
189    } else {
190        None
191    }
192}
193
194/// Parse a sample written by the watcher
195/// (`<memory.max> <memory.peak> <oom> <oom_kill>`); `None` without a sample.
196pub fn parse_cgroup_memory_sample(text: &str) -> Option<CgroupMemory> {
197    let fields: Vec<&str> = text.split_whitespace().collect();
198    if fields.len() < 4 {
199        return None;
200    }
201    let counters = CgroupMemory {
202        limit_bytes: parse_counter(fields[0]),
203        peak_bytes: parse_counter(fields[1]),
204        oom_events: parse_counter(fields[2]),
205        oom_kills: parse_counter(fields[3]),
206    };
207    if counters.oom_events.is_none() && counters.oom_kills.is_none() {
208        // memory.events was unreadable: the line carries no information.
209        return None;
210    }
211    Some(counters)
212}
213
214/// The `Memory:` line for a sample, exactly as the watcher's shell writes it.
215/// Used where Rust writes the post-mortem itself (the recovery step).
216pub fn format_cgroup_memory_log_line(text: &str) -> Option<String> {
217    let counters = parse_cgroup_memory_sample(text)?;
218    let fields: Vec<&str> = text.split_whitespace().collect();
219    let note = describe_cgroup_oom_scope(&counters)
220        .map(|scope| format!(" ({})", scope.note()))
221        .unwrap_or_default();
222    Some(format!(
223        "Memory:     memory.max={} memory.peak={} oom={} oom_kill={}{}",
224        fields[0], fields[1], fields[2], fields[3], note
225    ))
226}
227
228/// `None` when nothing was OOM-killed.
229pub fn describe_cgroup_oom_scope(memory: &CgroupMemory) -> Option<OomScope> {
230    let kills = memory.oom_kills.filter(|kills| *kills > 0)?;
231    Some(if kills > memory.oom_events.unwrap_or(0) {
232        OomScope::HostOrParent
233    } else {
234        OomScope::ContainerLimit
235    })
236}
237
238fn format_bytes(bytes: Option<u64>) -> String {
239    let Some(bytes) = bytes else {
240        return "unknown".to_string();
241    };
242    const UNITS: [&str; 5] = ["B", "KiB", "MiB", "GiB", "TiB"];
243    let mut value = bytes as f64;
244    let mut unit = 0;
245    while value >= 1024.0 && unit < UNITS.len() - 1 {
246        value /= 1024.0;
247        unit += 1;
248    }
249    if unit == 0 {
250        format!("{} B", bytes)
251    } else {
252        format!("{:.1} {}", value, UNITS[unit])
253    }
254}
255
256/// Human-readable counters for `--status`, e.g.
257/// `peak 255.9 MiB of 256.0 MiB limit, oom 0, oom_kill 3 (...)`.
258pub fn format_cgroup_memory(memory: &CgroupMemory) -> String {
259    let limit = match memory.limit_bytes {
260        Some(_) => format!("{} limit", format_bytes(memory.limit_bytes)),
261        None => "no limit".to_string(),
262    };
263    let count = |value: Option<u64>| value.map_or_else(|| "unknown".to_string(), |v| v.to_string());
264    let note = describe_cgroup_oom_scope(memory)
265        .map(|scope| format!(" ({})", scope.note()))
266        .unwrap_or_default();
267    format!(
268        "peak {} of {}, oom {}, oom_kill {}{}",
269        format_bytes(memory.peak_bytes),
270        limit,
271        count(memory.oom_events),
272        count(memory.oom_kills),
273        note
274    )
275}
276
277fn counter_from_value(value: Option<&Value>) -> Option<u64> {
278    match value? {
279        Value::Number(number) => number.as_u64().or_else(|| {
280            number
281                .as_f64()
282                .filter(|v| v.fract() == 0.0 && *v >= 0.0 && *v <= 9_007_199_254_740_991.0)
283                .map(|v| v as u64)
284        }),
285        Value::String(text) => parse_counter(text.trim()),
286        _ => None,
287    }
288}
289
290/// Normalize counters read back from an execution record. Lenient, like the
291/// JavaScript reader: a malformed value never makes the record unreadable.
292pub fn normalize_cgroup_memory(value: &Value) -> Option<CgroupMemory> {
293    let object = value.as_object()?;
294    Some(CgroupMemory {
295        limit_bytes: counter_from_value(object.get("limitBytes")),
296        peak_bytes: counter_from_value(object.get("peakBytes")),
297        oom_events: counter_from_value(object.get("oomEvents")),
298        oom_kills: counter_from_value(object.get("oomKills")),
299    })
300}
301
302/// `deserialize_with` helper for `ExecutionRecord::cgroup_memory`.
303pub fn deserialize_cgroup_memory<'de, D>(deserializer: D) -> Result<Option<CgroupMemory>, D::Error>
304where
305    D: Deserializer<'de>,
306{
307    let value = Option::<Value>::deserialize(deserializer)?;
308    Ok(value.as_ref().and_then(normalize_cgroup_memory))
309}
310
311#[cfg(test)]
312mod tests {
313    use super::*;
314
315    #[test]
316    fn parses_counters_and_treats_max_as_no_limit() {
317        assert_eq!(
318            parse_cgroup_memory_sample("max 1024 0 2"),
319            Some(CgroupMemory {
320                limit_bytes: None,
321                peak_bytes: Some(1024),
322                oom_events: Some(0),
323                oom_kills: Some(2),
324            })
325        );
326        assert_eq!(parse_cgroup_memory_sample("max - - -"), None);
327        assert_eq!(parse_cgroup_memory_sample("1 2 3"), None);
328    }
329
330    #[test]
331    fn normalizes_floats_strings_and_nulls() {
332        let value = serde_json::json!({
333            "limitBytes": 268435456.0,
334            "peakBytes": "1024",
335            "oomEvents": null,
336            "oomKills": -1
337        });
338        assert_eq!(
339            normalize_cgroup_memory(&value),
340            Some(CgroupMemory {
341                limit_bytes: Some(268_435_456),
342                peak_bytes: Some(1024),
343                oom_events: None,
344                oom_kills: None,
345            })
346        );
347        assert_eq!(normalize_cgroup_memory(&Value::Null), None);
348    }
349}
start_command/cgroup_memory.rs

start_command/
cgroup_memory.rs
