gam_runtime/
cgroup_memory.rs

1//! Typed Linux cgroup memory observations.
2//!
3//! `sysinfo::CGroupLimits` is intentionally not used here. Its aggregate loses
4//! the distinction between the literal cgroup-v2 `memory.max = max` token and a
5//! finite limit, derives "free" as `max - memory.current` even though
6//! `memory.current` includes reclaimable file cache, and probes a fixed cgroup
7//! path rather than the current process' hierarchy. Memory admission needs the
8//! kernel contract before any of that information is erased.
9
10use std::fmt;
11
12/// The exact syntax of one cgroup-v2 `memory.max` value.
13#[derive(Clone, Copy, Debug, PartialEq, Eq)]
14pub enum CgroupMemoryLimit {
15    /// The literal `max` token: this level imposes no hard memory ceiling.
16    Unlimited,
17    /// A finite hard ceiling in bytes. Zero is valid and authoritative.
18    Finite(u64),
19}
20
21/// Why a live cgroup memory hierarchy could not be observed safely.
22#[derive(Clone, Copy, Debug, PartialEq, Eq)]
23pub enum CgroupMemoryProbeFailureKind {
24    MalformedMembership,
25    MissingUnifiedMount,
26    MalformedMountInfo,
27    Io,
28    InvalidLimit,
29    InvalidCounter,
30    MissingCounter,
31    InconsistentCounters,
32}
33
34impl fmt::Display for CgroupMemoryProbeFailureKind {
35    fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
36        let name = match self {
37            Self::MalformedMembership => "malformed-membership",
38            Self::MissingUnifiedMount => "missing-unified-mount",
39            Self::MalformedMountInfo => "malformed-mountinfo",
40            Self::Io => "io",
41            Self::InvalidLimit => "invalid-limit",
42            Self::InvalidCounter => "invalid-counter",
43            Self::MissingCounter => "missing-counter",
44            Self::InconsistentCounters => "inconsistent-counters",
45        };
46        formatter.write_str(name)
47    }
48}
49
50/// Fail-closed evidence from an active cgroup controller probe.
51#[derive(Clone, Debug, PartialEq, Eq)]
52pub struct CgroupMemoryProbeFailure {
53    kind: CgroupMemoryProbeFailureKind,
54    path: Box<str>,
55    detail: Box<str>,
56}
57
58impl CgroupMemoryProbeFailure {
59    pub const fn kind(&self) -> CgroupMemoryProbeFailureKind {
60        self.kind
61    }
62
63    pub fn path(&self) -> &str {
64        &self.path
65    }
66
67    pub fn detail(&self) -> &str {
68        &self.detail
69    }
70}
71
72impl fmt::Display for CgroupMemoryProbeFailure {
73    fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
74        write!(
75            formatter,
76            "{} at {}: {}",
77            self.kind, self.path, self.detail
78        )
79    }
80}
81
82/// Reclaim-aware headroom under the binding finite cgroup ancestor.
83///
84/// `working_set_bytes = memory.current - inactive_file`. Only inactive file
85/// cache is credited as reclaimable; active file cache and reclaimable slab are
86/// deliberately left in the working set. The governor's separate 1/4 headroom
87/// remains available for reclaim latency, allocator slack, and untracked work.
88#[derive(Clone, Debug, PartialEq, Eq)]
89pub struct CgroupMemoryAvailability {
90    binding_path: Box<str>,
91    limit_bytes: u64,
92    current_bytes: u64,
93    inactive_file_bytes: u64,
94    working_set_bytes: u64,
95    available_bytes: u64,
96    inspected_levels: usize,
97}
98
99impl CgroupMemoryAvailability {
100    pub fn binding_path(&self) -> &str {
101        &self.binding_path
102    }
103
104    pub const fn limit_bytes(&self) -> u64 {
105        self.limit_bytes
106    }
107
108    pub const fn current_bytes(&self) -> u64 {
109        self.current_bytes
110    }
111
112    pub const fn inactive_file_bytes(&self) -> u64 {
113        self.inactive_file_bytes
114    }
115
116    pub const fn working_set_bytes(&self) -> u64 {
117        self.working_set_bytes
118    }
119
120    pub const fn available_bytes(&self) -> u64 {
121        self.available_bytes
122    }
123
124    pub const fn inspected_levels(&self) -> usize {
125        self.inspected_levels
126    }
127}
128
129#[cfg(test)]
130mod tests_fixtures {
131    use super::*;
132
133    impl CgroupMemoryProbeFailure {
134        pub(crate) fn fixture(
135            kind: CgroupMemoryProbeFailureKind,
136            path: impl Into<Box<str>>,
137            detail: impl Into<Box<str>>,
138        ) -> Self {
139            Self {
140                kind,
141                path: path.into(),
142                detail: detail.into(),
143            }
144        }
145    }
146
147    impl CgroupMemoryAvailability {
148        pub(crate) fn fixture(
149            binding_path: impl Into<Box<str>>,
150            limit_bytes: u64,
151            current_bytes: u64,
152            inactive_file_bytes: u64,
153            inspected_levels: usize,
154        ) -> Self {
155            let working_set_bytes = current_bytes
156                .checked_sub(inactive_file_bytes)
157                .expect("cgroup test fixture counters must be internally consistent");
158            Self {
159                binding_path: binding_path.into(),
160                limit_bytes,
161                current_bytes,
162                inactive_file_bytes,
163                working_set_bytes,
164                available_bytes: limit_bytes.saturating_sub(working_set_bytes),
165                inspected_levels,
166            }
167        }
168    }
169}
170
171impl fmt::Display for CgroupMemoryAvailability {
172    fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
173        write!(
174            formatter,
175            "finite cgroup ceiling at {} (limit={}, current={}, inactive_file={}, working_set={}, available={}, visible_levels={})",
176            self.binding_path,
177            self.limit_bytes,
178            self.current_bytes,
179            self.inactive_file_bytes,
180            self.working_set_bytes,
181            self.available_bytes,
182            self.inspected_levels,
183        )
184    }
185}
186
187/// The process' typed cgroup memory provenance.
188#[derive(Clone, Debug, PartialEq, Eq)]
189pub enum CgroupMemoryObservation {
190    /// No active memory controller applies on this platform/hierarchy.
191    NotPresent,
192    /// A cgroup-v2 hierarchy was found, but every visible hard limit was the
193    /// literal `max` token (or the process is at the unconstrained root).
194    V2Unbounded {
195        cgroup_path: Box<str>,
196        inspected_levels: usize,
197    },
198    /// At least one finite hard ceiling applies. This carries the ancestor with
199    /// the least reclaim-aware headroom.
200    V2Limited(CgroupMemoryAvailability),
201    /// A cgroup-v1 memory hierarchy is active. V1 exposes its unlimited state
202    /// as an architecture-dependent numeric sentinel rather than a token, so
203    /// every visible numeric ceiling participates in the minimum; an enormous
204    /// sentinel naturally loses to host availability.
205    V1Limited(CgroupMemoryAvailability),
206    /// A memory controller appears active but its semantics could not be read
207    /// exactly. Admission must fail closed rather than inherit host memory.
208    ProbeFailed(CgroupMemoryProbeFailure),
209}
210
211impl fmt::Display for CgroupMemoryObservation {
212    fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
213        match self {
214            Self::NotPresent => formatter.write_str("no active cgroup memory controller"),
215            Self::V2Unbounded {
216                cgroup_path,
217                inspected_levels,
218            } => write!(
219                formatter,
220                "unbounded cgroup-v2 hierarchy at {cgroup_path} ({inspected_levels} memory.max levels)"
221            ),
222            Self::V2Limited(observation) => write!(formatter, "cgroup-v2 {observation}"),
223            Self::V1Limited(observation) => write!(formatter, "cgroup-v1 {observation}"),
224            Self::ProbeFailed(failure) => write!(formatter, "cgroup probe failed: {failure}"),
225        }
226    }
227}
228
229#[cfg(not(target_os = "linux"))]
230pub(crate) fn detect_cgroup_memory() -> CgroupMemoryObservation {
231    CgroupMemoryObservation::NotPresent
232}
233
234#[cfg(target_os = "linux")]
235mod linux {
236    use super::*;
237    use std::ffi::OsString;
238    use std::fs;
239    use std::io;
240    use std::os::unix::ffi::OsStringExt;
241    use std::path::{Component, Path, PathBuf};
242
243    impl CgroupMemoryProbeFailure {
244        fn new(
245            kind: CgroupMemoryProbeFailureKind,
246            path: impl Into<Box<str>>,
247            detail: impl Into<Box<str>>,
248        ) -> Self {
249            Self {
250                kind,
251                path: path.into(),
252                detail: detail.into(),
253            }
254        }
255    }
256
257    impl CgroupMemoryAvailability {
258        fn from_consistent_counters(
259            binding_path: impl Into<Box<str>>,
260            limit_bytes: u64,
261            current_bytes: u64,
262            inactive_file_bytes: u64,
263            inspected_levels: usize,
264        ) -> Option<Self> {
265            let working_set_bytes = current_bytes.checked_sub(inactive_file_bytes)?;
266            Some(Self {
267                binding_path: binding_path.into(),
268                limit_bytes,
269                current_bytes,
270                inactive_file_bytes,
271                working_set_bytes,
272                available_bytes: limit_bytes.saturating_sub(working_set_bytes),
273                inspected_levels,
274            })
275        }
276    }
277
278    const PROC_SELF_CGROUP: &str = "/proc/self/cgroup";
279    const PROC_SELF_MOUNTINFO: &str = "/proc/self/mountinfo";
280
281    #[derive(Debug)]
282    struct ControllerMount {
283        root: PathBuf,
284        mount_point: PathBuf,
285    }
286
287    #[derive(Debug)]
288    struct MemoryMembership {
289        unified: Option<PathBuf>,
290        legacy: Option<PathBuf>,
291    }
292
293    pub(super) fn detect() -> CgroupMemoryObservation {
294        detect_from_proc_files(
295            Path::new(PROC_SELF_CGROUP),
296            Path::new(PROC_SELF_MOUNTINFO),
297        )
298        .unwrap_or_else(CgroupMemoryObservation::ProbeFailed)
299    }
300
301    fn detect_from_proc_files(
302        cgroup_file: &Path,
303        mountinfo_file: &Path,
304    ) -> Result<CgroupMemoryObservation, CgroupMemoryProbeFailure> {
305        let membership_text = read_required(cgroup_file)?;
306        let membership = parse_memory_membership(&membership_text, cgroup_file)?;
307        let mountinfo_text = read_required(mountinfo_file)?;
308        if let Some(legacy) = membership.legacy {
309            let mount = select_controller_mount(
310                &mountinfo_text,
311                &legacy,
312                mountinfo_file,
313                "cgroup",
314                Some("memory"),
315            )?;
316            let leaf = resolve_membership_leaf(&legacy, &mount, mountinfo_file)?;
317            return inspect_visible_v1_hierarchy(&leaf, &mount.mount_point);
318        }
319        if let Some(unified) = membership.unified {
320            let mount = select_controller_mount(
321                &mountinfo_text,
322                &unified,
323                mountinfo_file,
324                "cgroup2",
325                None,
326            )?;
327            let leaf = resolve_membership_leaf(&unified, &mount, mountinfo_file)?;
328            return inspect_visible_v2_hierarchy(&leaf, &mount.mount_point);
329        }
330        Ok(CgroupMemoryObservation::NotPresent)
331    }
332
333    fn parse_memory_membership(
334        text: &str,
335        source: &Path,
336    ) -> Result<MemoryMembership, CgroupMemoryProbeFailure> {
337        let mut unified = None;
338        let mut legacy = None;
339        for line in text.lines().filter(|line| !line.is_empty()) {
340            let mut fields = line.splitn(3, ':');
341            let hierarchy = fields.next();
342            let controllers = fields.next();
343            let path = fields.next();
344            let (Some(hierarchy), Some(controllers), Some(path)) =
345                (hierarchy, controllers, path)
346            else {
347                return Err(failure(
348                    CgroupMemoryProbeFailureKind::MalformedMembership,
349                    source,
350                    format!("invalid /proc/self/cgroup record {line:?}"),
351                ));
352            };
353            if controllers.split(',').any(|name| name == "memory") {
354                if legacy.is_some() {
355                    return Err(failure(
356                        CgroupMemoryProbeFailureKind::MalformedMembership,
357                        source,
358                        "multiple cgroup-v1 memory memberships",
359                    ));
360                }
361                let path = PathBuf::from(path);
362                validate_absolute_cgroup_path(&path, source)?;
363                legacy = Some(path);
364            }
365            if hierarchy == "0" && controllers.is_empty() {
366                if unified.is_some() {
367                    return Err(failure(
368                        CgroupMemoryProbeFailureKind::MalformedMembership,
369                        source,
370                        "multiple unified cgroup-v2 memberships",
371                    ));
372                }
373                let path = PathBuf::from(path);
374                validate_absolute_cgroup_path(&path, source)?;
375                unified = Some(path);
376            }
377        }
378        Ok(MemoryMembership { unified, legacy })
379    }
380
381    fn validate_absolute_cgroup_path(
382        path: &Path,
383        source: &Path,
384    ) -> Result<(), CgroupMemoryProbeFailure> {
385        if !path.is_absolute()
386            || path
387                .components()
388                .any(|component| matches!(component, Component::ParentDir))
389        {
390            return Err(failure(
391                CgroupMemoryProbeFailureKind::MalformedMembership,
392                source,
393                format!("invalid unified cgroup path {}", path.display()),
394            ));
395        }
396        Ok(())
397    }
398
399    fn select_controller_mount(
400        text: &str,
401        membership: &Path,
402        source: &Path,
403        filesystem: &str,
404        required_super_option: Option<&str>,
405    ) -> Result<ControllerMount, CgroupMemoryProbeFailure> {
406        let mut selected: Option<ControllerMount> = None;
407        for line in text.lines().filter(|line| !line.is_empty()) {
408            let Some((before_separator, after_separator)) = line.split_once(" - ") else {
409                return Err(failure(
410                    CgroupMemoryProbeFailureKind::MalformedMountInfo,
411                    source,
412                    format!("mountinfo record has no separator: {line:?}"),
413                ));
414            };
415            let after_fields = after_separator.split_whitespace().collect::<Vec<_>>();
416            if after_fields.first().copied() != Some(filesystem) {
417                continue;
418            }
419            if let Some(required) = required_super_option
420                && !after_fields
421                    .get(2)
422                    .is_some_and(|options| options.split(',').any(|option| option == required))
423            {
424                continue;
425            }
426            let before_fields = before_separator.split_whitespace().collect::<Vec<_>>();
427            if before_fields.len() < 6 {
428                return Err(failure(
429                    CgroupMemoryProbeFailureKind::MalformedMountInfo,
430                    source,
431                    format!("short {filesystem} mountinfo record: {line:?}"),
432                ));
433            }
434            let root = decode_mountinfo_path(before_fields[3], source)?;
435            let mount_point = decode_mountinfo_path(before_fields[4], source)?;
436            if !root.is_absolute() || !mount_point.is_absolute() {
437                return Err(failure(
438                    CgroupMemoryProbeFailureKind::MalformedMountInfo,
439                    source,
440                    format!(
441                        "{filesystem} mount paths must be absolute (root={}, mount_point={})",
442                        root.display(),
443                        mount_point.display()
444                    ),
445                ));
446            }
447            if membership.strip_prefix(&root).is_err() {
448                continue;
449            }
450            let candidate = ControllerMount { root, mount_point };
451            let candidate_depth = candidate.root.components().count();
452            let selected_depth = selected
453                .as_ref()
454                .map_or(0, |mount| mount.root.components().count());
455            if selected.is_none() || candidate_depth > selected_depth {
456                selected = Some(candidate);
457            }
458        }
459        selected.ok_or_else(|| {
460            failure(
461                CgroupMemoryProbeFailureKind::MissingUnifiedMount,
462                source,
463                format!(
464                    "no {filesystem} memory mount covers process membership {}",
465                    membership.display()
466                ),
467            )
468        })
469    }
470
471    fn resolve_membership_leaf(
472        membership: &Path,
473        mount: &ControllerMount,
474        source: &Path,
475    ) -> Result<PathBuf, CgroupMemoryProbeFailure> {
476        let relative = membership.strip_prefix(&mount.root).map_err(|_| {
477            failure(
478                CgroupMemoryProbeFailureKind::MissingUnifiedMount,
479                source,
480                format!(
481                    "cgroup path {} is outside selected mount root {}",
482                    membership.display(),
483                    mount.root.display()
484                ),
485            )
486        })?;
487        let relative = relative.strip_prefix(Path::new("/")).unwrap_or(relative);
488        Ok(mount.mount_point.join(relative))
489    }
490
491    fn decode_mountinfo_path(
492        raw: &str,
493        source: &Path,
494    ) -> Result<PathBuf, CgroupMemoryProbeFailure> {
495        let bytes = raw.as_bytes();
496        let mut decoded = Vec::with_capacity(bytes.len());
497        let mut index = 0;
498        while index < bytes.len() {
499            if bytes[index] != b'\\' {
500                decoded.push(bytes[index]);
501                index += 1;
502                continue;
503            }
504            let octal = bytes
505                .get(index + 1..index + 4)
506                .filter(|digits| digits.iter().all(|digit| (b'0'..=b'7').contains(digit)));
507            let Some(octal) = octal else {
508                return Err(failure(
509                    CgroupMemoryProbeFailureKind::MalformedMountInfo,
510                    source,
511                    format!("invalid mountinfo path escape in {raw:?}"),
512                ));
513            };
514            let value = u16::from(octal[0] - b'0') * 64
515                + u16::from(octal[1] - b'0') * 8
516                + u16::from(octal[2] - b'0');
517            let value = u8::try_from(value).map_err(|_| {
518                failure(
519                    CgroupMemoryProbeFailureKind::MalformedMountInfo,
520                    source,
521                    format!("mountinfo path escape exceeds one byte in {raw:?}"),
522                )
523            })?;
524            decoded.push(value);
525            index += 4;
526        }
527        Ok(PathBuf::from(OsString::from_vec(decoded)))
528    }
529
530    fn inspect_visible_v2_hierarchy(
531        leaf: &Path,
532        mount_point: &Path,
533    ) -> Result<CgroupMemoryObservation, CgroupMemoryProbeFailure> {
534        if !leaf.starts_with(mount_point) {
535            return Err(failure(
536                CgroupMemoryProbeFailureKind::MissingUnifiedMount,
537                leaf,
538                format!("leaf is outside mount point {}", mount_point.display()),
539            ));
540        }
541        let leaf_metadata = fs::metadata(leaf).map_err(|error| io_failure(leaf, error))?;
542        if !leaf_metadata.is_dir() {
543            return Err(failure(
544                CgroupMemoryProbeFailureKind::MissingUnifiedMount,
545                leaf,
546                "resolved process cgroup is not a directory",
547            ));
548        }
549        let mut directory = leaf.to_path_buf();
550        let mut inspected_levels = 0usize;
551        let mut binding: Option<CgroupMemoryAvailability> = None;
552        loop {
553            let max_path = directory.join("memory.max");
554            let current_path = directory.join("memory.current");
555            let stat_path = directory.join("memory.stat");
556            match read_optional(&max_path)? {
557                Some(raw_limit) => {
558                    inspected_levels = inspected_levels.saturating_add(1);
559                    match parse_limit(&raw_limit, &max_path)? {
560                        CgroupMemoryLimit::Unlimited => {}
561                        CgroupMemoryLimit::Finite(limit_bytes) => {
562                            let current_before_raw =
563                                read_optional(&current_path)?.ok_or_else(|| {
564                                    failure(
565                                        CgroupMemoryProbeFailureKind::MissingCounter,
566                                        &current_path,
567                                        "finite memory.max requires memory.current",
568                                    )
569                                })?;
570                            let current_before =
571                                parse_counter(&current_before_raw, &current_path)?;
572                            let stat_raw = read_optional(&stat_path)?.ok_or_else(|| {
573                                failure(
574                                    CgroupMemoryProbeFailureKind::MissingCounter,
575                                    &stat_path,
576                                    "finite memory.max requires memory.stat",
577                                )
578                            })?;
579                            let inactive_file_bytes =
580                                parse_stat_counter(&stat_raw, "inactive_file", &stat_path)?;
581                            // `memory.current` and `memory.stat` are live files,
582                            // not an atomic snapshot. Bracket the stat read and
583                            // use the larger current value: this is conservative
584                            // for admission and prevents an ordinary concurrent
585                            // charge/uncharge from looking like malformed data.
586                            let current_after_raw =
587                                read_optional(&current_path)?.ok_or_else(|| {
588                                    failure(
589                                        CgroupMemoryProbeFailureKind::MissingCounter,
590                                        &current_path,
591                                        "memory.current disappeared during cgroup probe",
592                                    )
593                                })?;
594                            let current_after =
595                                parse_counter(&current_after_raw, &current_path)?;
596                            let current_bytes = current_before.max(current_after);
597                            if inactive_file_bytes > current_bytes {
598                                return Err(failure(
599                                    CgroupMemoryProbeFailureKind::InconsistentCounters,
600                                    &stat_path,
601                                    format!(
602                                        "inactive_file={inactive_file_bytes} exceeds memory.current={current_bytes}"
603                                    ),
604                                ));
605                            }
606                            let candidate = CgroupMemoryAvailability::from_consistent_counters(
607                                directory.display().to_string().into_boxed_str(),
608                                limit_bytes,
609                                current_bytes,
610                                inactive_file_bytes,
611                                0,
612                            )
613                            .ok_or_else(|| {
614                                failure(
615                                    CgroupMemoryProbeFailureKind::InconsistentCounters,
616                                    &stat_path,
617                                    "memory counters became inconsistent during construction",
618                                )
619                            })?;
620                            if binding.as_ref().map_or(true, |current| {
621                                candidate.available_bytes() < current.available_bytes()
622                            }) {
623                                binding = Some(candidate);
624                            }
625                        }
626                    }
627                }
628                None => {
629                    let has_current = read_optional(&current_path)?.is_some();
630                    let has_stat = read_optional(&stat_path)?.is_some();
631                    // The cgroup-v2 root is exempt from resource control and
632                    // therefore has accounting files but no `memory.max`.
633                    // Every non-root level with controller accounting must
634                    // expose its hard-limit file; otherwise the observation is
635                    // incomplete and cannot safely inherit host capacity.
636                    if directory != mount_point && (has_current || has_stat) {
637                        return Err(failure(
638                            CgroupMemoryProbeFailureKind::MissingCounter,
639                            &max_path,
640                            "memory controller files are present but memory.max is missing",
641                        ));
642                    }
643                }
644            }
645            if directory == mount_point {
646                break;
647            }
648            let Some(parent) = directory.parent() else {
649                return Err(failure(
650                    CgroupMemoryProbeFailureKind::MissingUnifiedMount,
651                    &directory,
652                    "cgroup hierarchy ended before its mount point",
653                ));
654            };
655            directory = parent.to_path_buf();
656        }
657        if let Some(mut binding) = binding {
658            binding.inspected_levels = inspected_levels;
659            Ok(CgroupMemoryObservation::V2Limited(binding))
660        } else {
661            Ok(CgroupMemoryObservation::V2Unbounded {
662                cgroup_path: leaf.display().to_string().into_boxed_str(),
663                inspected_levels,
664            })
665        }
666    }
667
668    fn inspect_visible_v1_hierarchy(
669        leaf: &Path,
670        mount_point: &Path,
671    ) -> Result<CgroupMemoryObservation, CgroupMemoryProbeFailure> {
672        if !leaf.starts_with(mount_point) {
673            return Err(failure(
674                CgroupMemoryProbeFailureKind::MissingUnifiedMount,
675                leaf,
676                format!("leaf is outside mount point {}", mount_point.display()),
677            ));
678        }
679        let leaf_metadata = fs::metadata(leaf).map_err(|error| io_failure(leaf, error))?;
680        if !leaf_metadata.is_dir() {
681            return Err(failure(
682                CgroupMemoryProbeFailureKind::MissingUnifiedMount,
683                leaf,
684                "resolved process cgroup is not a directory",
685            ));
686        }
687
688        let hierarchy_path = leaf.join("memory.use_hierarchy");
689        let hierarchy_raw = read_required(&hierarchy_path)?;
690        let hierarchical = match parse_counter(&hierarchy_raw, &hierarchy_path)? {
691            0 => false,
692            1 => true,
693            value => {
694                return Err(failure(
695                    CgroupMemoryProbeFailureKind::InvalidCounter,
696                    &hierarchy_path,
697                    format!("memory.use_hierarchy must be 0 or 1, got {value}"),
698                ));
699            }
700        };
701
702        let mut directory = leaf.to_path_buf();
703        let mut inspected_levels = 0usize;
704        let mut binding: Option<CgroupMemoryAvailability> = None;
705        loop {
706            let limit_path = directory.join("memory.limit_in_bytes");
707            let usage_path = directory.join("memory.usage_in_bytes");
708            let stat_path = directory.join("memory.stat");
709            let limit_bytes = parse_counter(&read_required(&limit_path)?, &limit_path)?;
710            let usage_before = parse_counter(&read_required(&usage_path)?, &usage_path)?;
711            let stat_raw = read_required(&stat_path)?;
712            let inactive_key = if hierarchical {
713                "total_inactive_file"
714            } else {
715                "inactive_file"
716            };
717            let inactive_file_bytes = parse_stat_counter(&stat_raw, inactive_key, &stat_path)?;
718            let usage_after = parse_counter(&read_required(&usage_path)?, &usage_path)?;
719            let current_bytes = usage_before.max(usage_after);
720            if inactive_file_bytes > current_bytes {
721                return Err(failure(
722                    CgroupMemoryProbeFailureKind::InconsistentCounters,
723                    &stat_path,
724                    format!(
725                        "{inactive_key}={inactive_file_bytes} exceeds memory.usage_in_bytes={current_bytes}"
726                    ),
727                ));
728            }
729            inspected_levels = inspected_levels.saturating_add(1);
730            let candidate = CgroupMemoryAvailability::from_consistent_counters(
731                directory.display().to_string().into_boxed_str(),
732                limit_bytes,
733                current_bytes,
734                inactive_file_bytes,
735                0,
736            )
737            .ok_or_else(|| {
738                failure(
739                    CgroupMemoryProbeFailureKind::InconsistentCounters,
740                    &stat_path,
741                    "memory counters became inconsistent during construction",
742                )
743            })?;
744            if binding.as_ref().is_none_or(|current| {
745                candidate.available_bytes() < current.available_bytes()
746            }) {
747                binding = Some(candidate);
748            }
749
750            if !hierarchical || directory == mount_point {
751                break;
752            }
753            let Some(parent) = directory.parent() else {
754                return Err(failure(
755                    CgroupMemoryProbeFailureKind::MissingUnifiedMount,
756                    &directory,
757                    "cgroup hierarchy ended before its mount point",
758                ));
759            };
760            directory = parent.to_path_buf();
761        }
762
763        let mut binding = binding.ok_or_else(|| {
764            failure(
765                CgroupMemoryProbeFailureKind::MissingCounter,
766                leaf,
767                "cgroup-v1 memory hierarchy exposed no accounting level",
768            )
769        })?;
770        binding.inspected_levels = inspected_levels;
771        Ok(CgroupMemoryObservation::V1Limited(binding))
772    }
773
774    fn parse_limit(
775        raw: &str,
776        source: &Path,
777    ) -> Result<CgroupMemoryLimit, CgroupMemoryProbeFailure> {
778        let value = raw.trim();
779        if value == "max" {
780            return Ok(CgroupMemoryLimit::Unlimited);
781        }
782        let bytes = value.parse::<u64>().map_err(|_| {
783            failure(
784                CgroupMemoryProbeFailureKind::InvalidLimit,
785                source,
786                format!("expected literal max or an unsigned byte count, got {value:?}"),
787            )
788        })?;
789        Ok(CgroupMemoryLimit::Finite(bytes))
790    }
791
792    fn parse_counter(raw: &str, source: &Path) -> Result<u64, CgroupMemoryProbeFailure> {
793        let value = raw.trim();
794        value.parse::<u64>().map_err(|_| {
795            failure(
796                CgroupMemoryProbeFailureKind::InvalidCounter,
797                source,
798                format!("expected an unsigned byte count, got {value:?}"),
799            )
800        })
801    }
802
803    fn parse_stat_counter(
804        raw: &str,
805        key: &str,
806        source: &Path,
807    ) -> Result<u64, CgroupMemoryProbeFailure> {
808        for line in raw.lines() {
809            let mut fields = line.split_whitespace();
810            let Some(name) = fields.next() else {
811                continue;
812            };
813            if name != key {
814                continue;
815            }
816            let Some(value) = fields.next() else {
817                return Err(failure(
818                    CgroupMemoryProbeFailureKind::InvalidCounter,
819                    source,
820                    format!("memory.stat key {key:?} has no value"),
821                ));
822            };
823            if fields.next().is_some() {
824                return Err(failure(
825                    CgroupMemoryProbeFailureKind::InvalidCounter,
826                    source,
827                    format!("memory.stat key {key:?} has trailing fields"),
828                ));
829            }
830            return value.parse::<u64>().map_err(|_| {
831                failure(
832                    CgroupMemoryProbeFailureKind::InvalidCounter,
833                    source,
834                    format!("memory.stat key {key:?} is not an unsigned byte count"),
835                )
836            });
837        }
838        Err(failure(
839            CgroupMemoryProbeFailureKind::MissingCounter,
840            source,
841            format!("memory.stat is missing required key {key:?}"),
842        ))
843    }
844
845    fn read_required(path: &Path) -> Result<String, CgroupMemoryProbeFailure> {
846        fs::read_to_string(path).map_err(|error| io_failure(path, error))
847    }
848
849    fn read_optional(path: &Path) -> Result<Option<String>, CgroupMemoryProbeFailure> {
850        match fs::read_to_string(path) {
851            Ok(value) => Ok(Some(value)),
852            Err(error) if error.kind() == io::ErrorKind::NotFound => Ok(None),
853            Err(error) => Err(io_failure(path, error)),
854        }
855    }
856
857    fn io_failure(path: &Path, error: io::Error) -> CgroupMemoryProbeFailure {
858        failure(
859            CgroupMemoryProbeFailureKind::Io,
860            path,
861            error.to_string(),
862        )
863    }
864
865    fn failure(
866        kind: CgroupMemoryProbeFailureKind,
867        path: &Path,
868        detail: impl Into<Box<str>>,
869    ) -> CgroupMemoryProbeFailure {
870        CgroupMemoryProbeFailure::new(
871            kind,
872            path.display().to_string().into_boxed_str(),
873            detail,
874        )
875    }
876
877    #[cfg(test)]
878    mod tests {
879        use super::*;
880        use std::fs;
881        use tempfile::TempDir;
882
883        struct Fixture {
884            _temp: TempDir,
885            mount: PathBuf,
886            cgroup_file: PathBuf,
887            mountinfo_file: PathBuf,
888        }
889
890        impl Fixture {
891            fn new(membership: &str) -> Self {
892                let temp = TempDir::new().expect("fixture tempdir");
893                let mount = temp.path().join("cgroup2");
894                fs::create_dir_all(&mount).expect("fixture mount");
895                let cgroup_file = temp.path().join("self.cgroup");
896                fs::write(&cgroup_file, format!("0::{membership}\n"))
897                    .expect("fixture membership");
898                let mountinfo_file = temp.path().join("self.mountinfo");
899                fs::write(
900                    &mountinfo_file,
901                    format!(
902                        "29 23 0:26 / {} rw,nosuid,nodev,noexec,relatime - cgroup2 cgroup rw\n",
903                        mount.display()
904                    ),
905                )
906                .expect("fixture mountinfo");
907                Self {
908                    _temp: temp,
909                    mount,
910                    cgroup_file,
911                    mountinfo_file,
912                }
913            }
914
915            fn level(&self, relative: &str, limit: &str, current: u64, inactive: u64) {
916                let directory = self.mount.join(relative.trim_start_matches('/'));
917                fs::create_dir_all(&directory).expect("fixture level");
918                fs::write(directory.join("memory.max"), format!("{limit}\n"))
919                    .expect("fixture memory.max");
920                fs::write(directory.join("memory.current"), format!("{current}\n"))
921                    .expect("fixture memory.current");
922                fs::write(
923                    directory.join("memory.stat"),
924                    format!("anon 1\ninactive_file {inactive}\nactive_file 2\n"),
925                )
926                .expect("fixture memory.stat");
927            }
928
929            fn observe(&self) -> CgroupMemoryObservation {
930                detect_from_proc_files(&self.cgroup_file, &self.mountinfo_file)
931                    .unwrap_or_else(CgroupMemoryObservation::ProbeFailed)
932            }
933        }
934
935        #[test]
936        fn literal_max_is_typed_unbounded_even_when_current_is_large() {
937            let fixture = Fixture::new("/tenant/leaf");
938            fixture.level("tenant/leaf", "max", u64::MAX - 1, 0);
939            assert!(matches!(
940                fixture.observe(),
941                CgroupMemoryObservation::V2Unbounded {
942                    inspected_levels: 1,
943                    ..
944                }
945            ));
946        }
947
948        #[test]
949        fn cgroup_v2_root_accounting_without_memory_max_is_unbounded() {
950            let fixture = Fixture::new("/");
951            fs::write(fixture.mount.join("memory.current"), "1000\n")
952                .expect("root current");
953            fs::write(fixture.mount.join("memory.stat"), "inactive_file 700\n")
954                .expect("root stat");
955            assert!(matches!(
956                fixture.observe(),
957                CgroupMemoryObservation::V2Unbounded {
958                    inspected_levels: 0,
959                    ..
960                }
961            ));
962        }
963
964        #[test]
965        fn finite_zero_remains_authoritative() {
966            let fixture = Fixture::new("/tenant/leaf");
967            fixture.level("tenant/leaf", "0", 0, 0);
968            let CgroupMemoryObservation::V2Limited(observation) = fixture.observe() else {
969                panic!("finite zero must be a real cgroup ceiling");
970            };
971            assert_eq!(observation.limit_bytes(), 0);
972            assert_eq!(observation.available_bytes(), 0);
973        }
974
975        #[test]
976        fn inactive_file_cache_is_conservatively_reclaimable() {
977            let fixture = Fixture::new("/tenant/leaf");
978            fixture.level("tenant/leaf", "1000", 1000, 700);
979            let CgroupMemoryObservation::V2Limited(observation) = fixture.observe() else {
980                panic!("finite ceiling must bind");
981            };
982            assert_eq!(observation.working_set_bytes(), 300);
983            assert_eq!(observation.available_bytes(), 700);
984        }
985
986        #[test]
987        fn finite_working_set_exhaustion_is_zero() {
988            let fixture = Fixture::new("/tenant/leaf");
989            fixture.level("tenant/leaf", "1000", 1200, 200);
990            let CgroupMemoryObservation::V2Limited(observation) = fixture.observe() else {
991                panic!("finite ceiling must bind");
992            };
993            assert_eq!(observation.working_set_bytes(), 1000);
994            assert_eq!(observation.available_bytes(), 0);
995        }
996
997        #[test]
998        fn binding_parent_accounts_for_sibling_pressure() {
999            let fixture = Fixture::new("/tenant/leaf");
1000            fixture.level("tenant/leaf", "max", 200, 100);
1001            fixture.level("tenant", "1000", 950, 100);
1002            let CgroupMemoryObservation::V2Limited(observation) = fixture.observe() else {
1003                panic!("finite parent must bind an unlimited leaf");
1004            };
1005            assert_eq!(observation.available_bytes(), 150);
1006            assert!(observation.binding_path().ends_with("/tenant"));
1007            assert_eq!(observation.inspected_levels(), 2);
1008        }
1009
1010        #[test]
1011        fn most_specific_covering_mount_resolves_non_root_membership() {
1012            let temp = TempDir::new().expect("fixture tempdir");
1013            let broad = temp.path().join("broad");
1014            let narrow = temp.path().join("narrow");
1015            fs::create_dir_all(&broad).expect("broad mount");
1016            fs::create_dir_all(narrow.join("leaf")).expect("narrow leaf");
1017            fs::write(narrow.join("leaf/memory.max"), "512\n").expect("max");
1018            fs::write(narrow.join("leaf/memory.current"), "256\n").expect("current");
1019            fs::write(narrow.join("leaf/memory.stat"), "inactive_file 64\n")
1020                .expect("stat");
1021            let cgroup_file = temp.path().join("self.cgroup");
1022            fs::write(&cgroup_file, "0::/tenant/leaf\n").expect("membership");
1023            let mountinfo_file = temp.path().join("self.mountinfo");
1024            fs::write(
1025                &mountinfo_file,
1026                format!(
1027                    "29 23 0:26 / {} rw - cgroup2 cgroup rw\n30 23 0:26 /tenant {} rw - cgroup2 cgroup rw\n",
1028                    broad.display(),
1029                    narrow.display()
1030                ),
1031            )
1032            .expect("mountinfo");
1033            let observation = detect_from_proc_files(&cgroup_file, &mountinfo_file)
1034                .expect("typed cgroup observation");
1035            let CgroupMemoryObservation::V2Limited(observation) = observation else {
1036                panic!("narrow finite mount must bind");
1037            };
1038            assert_eq!(observation.available_bytes(), 320);
1039            assert!(observation.binding_path().ends_with("/narrow/leaf"));
1040        }
1041
1042        #[test]
1043        fn malformed_active_controller_fails_closed() {
1044            let fixture = Fixture::new("/tenant/leaf");
1045            fixture.level("tenant/leaf", "1000", 500, 100);
1046            fs::write(
1047                fixture.mount.join("tenant/leaf/memory.current"),
1048                "not-a-counter\n",
1049            )
1050            .expect("corrupt current");
1051            let CgroupMemoryObservation::ProbeFailed(failure) = fixture.observe() else {
1052                panic!("malformed active controller must fail closed");
1053            };
1054            assert_eq!(
1055                failure.kind(),
1056                CgroupMemoryProbeFailureKind::InvalidCounter
1057            );
1058        }
1059
1060        #[test]
1061        fn inconsistent_cache_counter_fails_closed() {
1062            let fixture = Fixture::new("/tenant/leaf");
1063            fixture.level("tenant/leaf", "1000", 100, 101);
1064            let CgroupMemoryObservation::ProbeFailed(failure) = fixture.observe() else {
1065                panic!("inconsistent controller counters must fail closed");
1066            };
1067            assert_eq!(
1068                failure.kind(),
1069                CgroupMemoryProbeFailureKind::InconsistentCounters
1070            );
1071        }
1072
1073        #[test]
1074        fn active_cgroup_v1_memory_controller_is_measured_exactly() {
1075            let temp = TempDir::new().expect("fixture tempdir");
1076            let mount = temp.path().join("cgroup-memory");
1077            let leaf = mount.join("legacy");
1078            fs::create_dir_all(&leaf).expect("v1 leaf");
1079            fs::write(leaf.join("memory.use_hierarchy"), "0\n").expect("hierarchy");
1080            fs::write(leaf.join("memory.limit_in_bytes"), "1024\n").expect("limit");
1081            fs::write(leaf.join("memory.usage_in_bytes"), "512\n").expect("usage");
1082            fs::write(
1083                leaf.join("memory.stat"),
1084                "inactive_file 128\ntotal_inactive_file 128\n",
1085            )
1086            .expect("stat");
1087            let cgroup_file = temp.path().join("self.cgroup");
1088            fs::write(&cgroup_file, "4:memory:/legacy\n").expect("membership");
1089            let mountinfo_file = temp.path().join("self.mountinfo");
1090            fs::write(
1091                &mountinfo_file,
1092                format!(
1093                    "31 23 0:30 / {} rw - cgroup cgroup rw,memory\n",
1094                    mount.display()
1095                ),
1096            )
1097            .expect("mountinfo");
1098            let CgroupMemoryObservation::V1Limited(observation) =
1099                detect_from_proc_files(&cgroup_file, &mountinfo_file)
1100                    .expect("v1 memory observation")
1101            else {
1102                panic!("v1 memory ceiling must be authoritative");
1103            };
1104            assert_eq!(observation.working_set_bytes(), 384);
1105            assert_eq!(observation.available_bytes(), 640);
1106        }
1107
1108        #[test]
1109        fn hybrid_hierarchy_uses_the_active_v1_memory_controller() {
1110            let fixture = Fixture::new("/tenant/leaf");
1111            let v1_mount = fixture.mount.parent().unwrap().join("cgroup-memory");
1112            let v1_leaf = v1_mount.join("legacy");
1113            fs::create_dir_all(&v1_leaf).expect("v1 leaf");
1114            fs::write(v1_leaf.join("memory.use_hierarchy"), "0\n").expect("hierarchy");
1115            fs::write(v1_leaf.join("memory.limit_in_bytes"), "2048\n").expect("limit");
1116            fs::write(v1_leaf.join("memory.usage_in_bytes"), "1024\n").expect("usage");
1117            fs::write(
1118                v1_leaf.join("memory.stat"),
1119                "inactive_file 256\ntotal_inactive_file 256\n",
1120            )
1121            .expect("stat");
1122            fs::write(
1123                &fixture.cgroup_file,
1124                "0::/tenant/leaf\n4:memory:/legacy\n",
1125            )
1126            .expect("hybrid membership");
1127            let original = fs::read_to_string(&fixture.mountinfo_file).expect("mountinfo");
1128            fs::write(
1129                &fixture.mountinfo_file,
1130                format!(
1131                    "{original}31 23 0:30 / {} rw - cgroup cgroup rw,memory\n",
1132                    v1_mount.display()
1133                ),
1134            )
1135            .expect("hybrid mountinfo");
1136            let CgroupMemoryObservation::V1Limited(observation) = fixture.observe() else {
1137                panic!("hybrid memory accounting must follow v1");
1138            };
1139            assert_eq!(observation.available_bytes(), 1280);
1140        }
1141    }
1142}
1143
1144#[cfg(target_os = "linux")]
1145pub(crate) fn detect_cgroup_memory() -> CgroupMemoryObservation {
1146    linux::detect()
1147}
1148
1149#[cfg(all(test, not(target_os = "linux")))]
1150mod non_linux_tests {
1151    use super::*;
1152
1153    #[test]
1154    fn non_linux_platform_has_no_cgroup_controller() {
1155        assert_eq!(
1156            detect_cgroup_memory(),
1157            CgroupMemoryObservation::NotPresent
1158        );
1159    }
1160}
gam_runtime/cgroup_memory.rs

gam_runtime/
cgroup_memory.rs
