gam_runtime/
cgroup_memory.rs

1//! Typed Linux cgroup memory observations.
2//!
3//! `sysinfo::CGroupLimits` is intentionally not used here. Its aggregate loses
4//! the distinction between the literal cgroup-v2 `memory.max = max` token and a
5//! finite limit, derives "free" as `max - memory.current` even though
6//! `memory.current` includes reclaimable file cache, and probes a fixed cgroup
7//! path rather than the current process' hierarchy. Memory admission needs the
8//! kernel contract before any of that information is erased.
9
10use std::fmt;
11
12/// The exact syntax of one cgroup-v2 `memory.max` value.
13#[derive(Clone, Copy, Debug, PartialEq, Eq)]
14pub enum CgroupMemoryLimit {
15    /// The literal `max` token: this level imposes no hard memory ceiling.
16    Unlimited,
17    /// A finite hard ceiling in bytes. Zero is valid and authoritative.
18    Finite(u64),
19}
20
21/// Why a live cgroup memory hierarchy could not be observed safely.
22#[derive(Clone, Copy, Debug, PartialEq, Eq)]
23pub enum CgroupMemoryProbeFailureKind {
24    MalformedMembership,
25    MissingUnifiedMount,
26    MalformedMountInfo,
27    Io,
28    InvalidLimit,
29    InvalidCounter,
30    MissingCounter,
31    InconsistentCounters,
32}
33
34impl fmt::Display for CgroupMemoryProbeFailureKind {
35    fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
36        let name = match self {
37            Self::MalformedMembership => "malformed-membership",
38            Self::MissingUnifiedMount => "missing-unified-mount",
39            Self::MalformedMountInfo => "malformed-mountinfo",
40            Self::Io => "io",
41            Self::InvalidLimit => "invalid-limit",
42            Self::InvalidCounter => "invalid-counter",
43            Self::MissingCounter => "missing-counter",
44            Self::InconsistentCounters => "inconsistent-counters",
45        };
46        formatter.write_str(name)
47    }
48}
49
50/// Fail-closed evidence from an active cgroup controller probe.
51#[derive(Clone, Debug, PartialEq, Eq)]
52pub struct CgroupMemoryProbeFailure {
53    kind: CgroupMemoryProbeFailureKind,
54    path: Box<str>,
55    detail: Box<str>,
56}
57
58impl CgroupMemoryProbeFailure {
59    pub const fn kind(&self) -> CgroupMemoryProbeFailureKind {
60        self.kind
61    }
62
63    pub fn path(&self) -> &str {
64        &self.path
65    }
66
67    pub fn detail(&self) -> &str {
68        &self.detail
69    }
70}
71
72impl fmt::Display for CgroupMemoryProbeFailure {
73    fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
74        write!(formatter, "{} at {}: {}", self.kind, self.path, self.detail)
75    }
76}
77
78/// Reclaim-aware headroom under the binding finite cgroup ancestor.
79///
80/// `working_set_bytes = memory.current - inactive_file`. Only inactive file
81/// cache is credited as reclaimable; active file cache and reclaimable slab are
82/// deliberately left in the working set. The governor's separate 1/4 headroom
83/// remains available for reclaim latency, allocator slack, and untracked work.
84#[derive(Clone, Debug, PartialEq, Eq)]
85pub struct CgroupMemoryAvailability {
86    binding_path: Box<str>,
87    limit_bytes: u64,
88    current_bytes: u64,
89    inactive_file_bytes: u64,
90    working_set_bytes: u64,
91    available_bytes: u64,
92    inspected_levels: usize,
93}
94
95impl CgroupMemoryAvailability {
96    /// Assemble one observation from counters that were read together.
97    ///
98    /// `None` when `inactive_file > current`, which is a torn read of the two
99    /// counters rather than a legal state: the working set would be negative.
100    /// This is the single construction site — the live probe, the replay used
101    /// by [`crate::test_support::simulated_cgroup_memory_environment`], and the
102    /// unit fixtures all funnel through it, so the derived `working_set` and
103    /// `available` can never disagree between a measured and a replayed
104    /// environment.
105    pub(crate) fn from_consistent_counters(
106        binding_path: impl Into<Box<str>>,
107        limit_bytes: u64,
108        current_bytes: u64,
109        inactive_file_bytes: u64,
110        inspected_levels: usize,
111    ) -> Option<Self> {
112        let working_set_bytes = current_bytes.checked_sub(inactive_file_bytes)?;
113        Some(Self {
114            binding_path: binding_path.into(),
115            limit_bytes,
116            current_bytes,
117            inactive_file_bytes,
118            working_set_bytes,
119            available_bytes: limit_bytes.saturating_sub(working_set_bytes),
120            inspected_levels,
121        })
122    }
123
124    pub fn binding_path(&self) -> &str {
125        &self.binding_path
126    }
127
128    pub const fn limit_bytes(&self) -> u64 {
129        self.limit_bytes
130    }
131
132    pub const fn current_bytes(&self) -> u64 {
133        self.current_bytes
134    }
135
136    pub const fn inactive_file_bytes(&self) -> u64 {
137        self.inactive_file_bytes
138    }
139
140    pub const fn working_set_bytes(&self) -> u64 {
141        self.working_set_bytes
142    }
143
144    pub const fn available_bytes(&self) -> u64 {
145        self.available_bytes
146    }
147
148    pub const fn inspected_levels(&self) -> usize {
149        self.inspected_levels
150    }
151}
152
153#[cfg(test)]
154mod tests_fixtures {
155    use super::*;
156
157    impl CgroupMemoryProbeFailure {
158        pub(crate) fn fixture(
159            kind: CgroupMemoryProbeFailureKind,
160            path: impl Into<Box<str>>,
161            detail: impl Into<Box<str>>,
162        ) -> Self {
163            Self {
164                kind,
165                path: path.into(),
166                detail: detail.into(),
167            }
168        }
169    }
170
171    impl CgroupMemoryAvailability {
172        pub(crate) fn fixture(
173            binding_path: impl Into<Box<str>>,
174            limit_bytes: u64,
175            current_bytes: u64,
176            inactive_file_bytes: u64,
177            inspected_levels: usize,
178        ) -> Self {
179            Self::from_consistent_counters(
180                binding_path,
181                limit_bytes,
182                current_bytes,
183                inactive_file_bytes,
184                inspected_levels,
185            )
186            .expect("cgroup test fixture counters must be internally consistent")
187        }
188    }
189}
190
191impl fmt::Display for CgroupMemoryAvailability {
192    fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
193        write!(
194            formatter,
195            "finite cgroup ceiling at {} (limit={}, current={}, inactive_file={}, working_set={}, available={}, visible_levels={})",
196            self.binding_path,
197            self.limit_bytes,
198            self.current_bytes,
199            self.inactive_file_bytes,
200            self.working_set_bytes,
201            self.available_bytes,
202            self.inspected_levels,
203        )
204    }
205}
206
207/// The process' typed cgroup memory provenance.
208#[derive(Clone, Debug, PartialEq, Eq)]
209pub enum CgroupMemoryObservation {
210    /// No active memory controller applies on this platform/hierarchy.
211    NotPresent,
212    /// A cgroup-v2 hierarchy was found, but every visible hard limit was the
213    /// literal `max` token (or the process is at the unconstrained root).
214    V2Unbounded {
215        cgroup_path: Box<str>,
216        inspected_levels: usize,
217    },
218    /// At least one finite hard ceiling applies. This carries the ancestor with
219    /// the least reclaim-aware headroom.
220    V2Limited(CgroupMemoryAvailability),
221    /// A cgroup-v1 memory hierarchy is active. V1 exposes its unlimited state
222    /// as an architecture-dependent numeric sentinel rather than a token, so
223    /// every visible numeric ceiling participates in the minimum; an enormous
224    /// sentinel naturally loses to host availability.
225    V1Limited(CgroupMemoryAvailability),
226    /// A memory controller appears active but its semantics could not be read
227    /// exactly. Admission must fail closed rather than inherit host memory.
228    ProbeFailed(CgroupMemoryProbeFailure),
229}
230
231impl fmt::Display for CgroupMemoryObservation {
232    fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
233        match self {
234            Self::NotPresent => formatter.write_str("no active cgroup memory controller"),
235            Self::V2Unbounded {
236                cgroup_path,
237                inspected_levels,
238            } => write!(
239                formatter,
240                "unbounded cgroup-v2 hierarchy at {cgroup_path} ({inspected_levels} memory.max levels)"
241            ),
242            Self::V2Limited(observation) => write!(formatter, "cgroup-v2 {observation}"),
243            Self::V1Limited(observation) => write!(formatter, "cgroup-v1 {observation}"),
244            Self::ProbeFailed(failure) => write!(formatter, "cgroup probe failed: {failure}"),
245        }
246    }
247}
248
249#[cfg(not(target_os = "linux"))]
250pub(crate) fn detect_cgroup_memory() -> CgroupMemoryObservation {
251    CgroupMemoryObservation::NotPresent
252}
253
254#[cfg(target_os = "linux")]
255mod linux {
256    use super::*;
257    use std::ffi::OsString;
258    use std::fs;
259    use std::io;
260    use std::os::unix::ffi::OsStringExt;
261    use std::path::{Component, Path, PathBuf};
262
263    impl CgroupMemoryProbeFailure {
264        fn new(
265            kind: CgroupMemoryProbeFailureKind,
266            path: impl Into<Box<str>>,
267            detail: impl Into<Box<str>>,
268        ) -> Self {
269            Self {
270                kind,
271                path: path.into(),
272                detail: detail.into(),
273            }
274        }
275    }
276
277    const PROC_SELF_CGROUP: &str = "/proc/self/cgroup";
278    const PROC_SELF_MOUNTINFO: &str = "/proc/self/mountinfo";
279
280    #[derive(Debug)]
281    struct ControllerMount {
282        root: PathBuf,
283        mount_point: PathBuf,
284    }
285
286    #[derive(Debug)]
287    struct MemoryMembership {
288        unified: Option<PathBuf>,
289        legacy: Option<PathBuf>,
290    }
291
292    pub(super) fn detect() -> CgroupMemoryObservation {
293        detect_from_proc_files(Path::new(PROC_SELF_CGROUP), Path::new(PROC_SELF_MOUNTINFO))
294            .unwrap_or_else(CgroupMemoryObservation::ProbeFailed)
295    }
296
297    fn detect_from_proc_files(
298        cgroup_file: &Path,
299        mountinfo_file: &Path,
300    ) -> Result<CgroupMemoryObservation, CgroupMemoryProbeFailure> {
301        let membership_text = read_required(cgroup_file)?;
302        let membership = parse_memory_membership(&membership_text, cgroup_file)?;
303        let mountinfo_text = read_required(mountinfo_file)?;
304        if let Some(legacy) = membership.legacy {
305            let mount = select_controller_mount(
306                &mountinfo_text,
307                &legacy,
308                mountinfo_file,
309                "cgroup",
310                Some("memory"),
311            )?;
312            let leaf = resolve_membership_leaf(&legacy, &mount, mountinfo_file)?;
313            return inspect_visible_v1_hierarchy(&leaf, &mount.mount_point);
314        }
315        if let Some(unified) = membership.unified {
316            let mount = select_controller_mount(
317                &mountinfo_text,
318                &unified,
319                mountinfo_file,
320                "cgroup2",
321                None,
322            )?;
323            let leaf = resolve_membership_leaf(&unified, &mount, mountinfo_file)?;
324            return inspect_visible_v2_hierarchy(&leaf, &mount.mount_point);
325        }
326        Ok(CgroupMemoryObservation::NotPresent)
327    }
328
329    fn parse_memory_membership(
330        text: &str,
331        source: &Path,
332    ) -> Result<MemoryMembership, CgroupMemoryProbeFailure> {
333        let mut unified = None;
334        let mut legacy = None;
335        for line in text.lines().filter(|line| !line.is_empty()) {
336            let mut fields = line.splitn(3, ':');
337            let hierarchy = fields.next();
338            let controllers = fields.next();
339            let path = fields.next();
340            let (Some(hierarchy), Some(controllers), Some(path)) = (hierarchy, controllers, path)
341            else {
342                return Err(failure(
343                    CgroupMemoryProbeFailureKind::MalformedMembership,
344                    source,
345                    format!("invalid /proc/self/cgroup record {line:?}"),
346                ));
347            };
348            if controllers.split(',').any(|name| name == "memory") {
349                if legacy.is_some() {
350                    return Err(failure(
351                        CgroupMemoryProbeFailureKind::MalformedMembership,
352                        source,
353                        "multiple cgroup-v1 memory memberships",
354                    ));
355                }
356                let path = PathBuf::from(path);
357                validate_absolute_cgroup_path(&path, source)?;
358                legacy = Some(path);
359            }
360            if hierarchy == "0" && controllers.is_empty() {
361                if unified.is_some() {
362                    return Err(failure(
363                        CgroupMemoryProbeFailureKind::MalformedMembership,
364                        source,
365                        "multiple unified cgroup-v2 memberships",
366                    ));
367                }
368                let path = PathBuf::from(path);
369                validate_absolute_cgroup_path(&path, source)?;
370                unified = Some(path);
371            }
372        }
373        Ok(MemoryMembership { unified, legacy })
374    }
375
376    fn validate_absolute_cgroup_path(
377        path: &Path,
378        source: &Path,
379    ) -> Result<(), CgroupMemoryProbeFailure> {
380        if !path.is_absolute()
381            || path
382                .components()
383                .any(|component| matches!(component, Component::ParentDir))
384        {
385            return Err(failure(
386                CgroupMemoryProbeFailureKind::MalformedMembership,
387                source,
388                format!("invalid unified cgroup path {}", path.display()),
389            ));
390        }
391        Ok(())
392    }
393
394    fn select_controller_mount(
395        text: &str,
396        membership: &Path,
397        source: &Path,
398        filesystem: &str,
399        required_super_option: Option<&str>,
400    ) -> Result<ControllerMount, CgroupMemoryProbeFailure> {
401        let mut selected: Option<ControllerMount> = None;
402        for line in text.lines().filter(|line| !line.is_empty()) {
403            let Some((before_separator, after_separator)) = line.split_once(" - ") else {
404                return Err(failure(
405                    CgroupMemoryProbeFailureKind::MalformedMountInfo,
406                    source,
407                    format!("mountinfo record has no separator: {line:?}"),
408                ));
409            };
410            let after_fields = after_separator.split_whitespace().collect::<Vec<_>>();
411            if after_fields.first().copied() != Some(filesystem) {
412                continue;
413            }
414            if let Some(required) = required_super_option
415                && !after_fields
416                    .get(2)
417                    .is_some_and(|options| options.split(',').any(|option| option == required))
418            {
419                continue;
420            }
421            let before_fields = before_separator.split_whitespace().collect::<Vec<_>>();
422            if before_fields.len() < 6 {
423                return Err(failure(
424                    CgroupMemoryProbeFailureKind::MalformedMountInfo,
425                    source,
426                    format!("short {filesystem} mountinfo record: {line:?}"),
427                ));
428            }
429            let root = decode_mountinfo_path(before_fields[3], source)?;
430            let mount_point = decode_mountinfo_path(before_fields[4], source)?;
431            if !root.is_absolute() || !mount_point.is_absolute() {
432                return Err(failure(
433                    CgroupMemoryProbeFailureKind::MalformedMountInfo,
434                    source,
435                    format!(
436                        "{filesystem} mount paths must be absolute (root={}, mount_point={})",
437                        root.display(),
438                        mount_point.display()
439                    ),
440                ));
441            }
442            if membership.strip_prefix(&root).is_err() {
443                continue;
444            }
445            let candidate = ControllerMount { root, mount_point };
446            let candidate_depth = candidate.root.components().count();
447            let selected_depth = selected
448                .as_ref()
449                .map_or(0, |mount| mount.root.components().count());
450            if selected.is_none() || candidate_depth > selected_depth {
451                selected = Some(candidate);
452            }
453        }
454        selected.ok_or_else(|| {
455            failure(
456                CgroupMemoryProbeFailureKind::MissingUnifiedMount,
457                source,
458                format!(
459                    "no {filesystem} memory mount covers process membership {}",
460                    membership.display()
461                ),
462            )
463        })
464    }
465
466    fn resolve_membership_leaf(
467        membership: &Path,
468        mount: &ControllerMount,
469        source: &Path,
470    ) -> Result<PathBuf, CgroupMemoryProbeFailure> {
471        let relative = membership.strip_prefix(&mount.root).map_err(|_| {
472            failure(
473                CgroupMemoryProbeFailureKind::MissingUnifiedMount,
474                source,
475                format!(
476                    "cgroup path {} is outside selected mount root {}",
477                    membership.display(),
478                    mount.root.display()
479                ),
480            )
481        })?;
482        let relative = relative.strip_prefix(Path::new("/")).unwrap_or(relative);
483        Ok(mount.mount_point.join(relative))
484    }
485
486    fn decode_mountinfo_path(
487        raw: &str,
488        source: &Path,
489    ) -> Result<PathBuf, CgroupMemoryProbeFailure> {
490        let bytes = raw.as_bytes();
491        let mut decoded = Vec::with_capacity(bytes.len());
492        let mut index = 0;
493        while index < bytes.len() {
494            if bytes[index] != b'\\' {
495                decoded.push(bytes[index]);
496                index += 1;
497                continue;
498            }
499            let octal = bytes
500                .get(index + 1..index + 4)
501                .filter(|digits| digits.iter().all(|digit| (b'0'..=b'7').contains(digit)));
502            let Some(octal) = octal else {
503                return Err(failure(
504                    CgroupMemoryProbeFailureKind::MalformedMountInfo,
505                    source,
506                    format!("invalid mountinfo path escape in {raw:?}"),
507                ));
508            };
509            let value = u16::from(octal[0] - b'0') * 64
510                + u16::from(octal[1] - b'0') * 8
511                + u16::from(octal[2] - b'0');
512            let value = u8::try_from(value).map_err(|_| {
513                failure(
514                    CgroupMemoryProbeFailureKind::MalformedMountInfo,
515                    source,
516                    format!("mountinfo path escape exceeds one byte in {raw:?}"),
517                )
518            })?;
519            decoded.push(value);
520            index += 4;
521        }
522        Ok(PathBuf::from(OsString::from_vec(decoded)))
523    }
524
525    fn inspect_visible_v2_hierarchy(
526        leaf: &Path,
527        mount_point: &Path,
528    ) -> Result<CgroupMemoryObservation, CgroupMemoryProbeFailure> {
529        if !leaf.starts_with(mount_point) {
530            return Err(failure(
531                CgroupMemoryProbeFailureKind::MissingUnifiedMount,
532                leaf,
533                format!("leaf is outside mount point {}", mount_point.display()),
534            ));
535        }
536        let leaf_metadata = fs::metadata(leaf).map_err(|error| io_failure(leaf, error))?;
537        if !leaf_metadata.is_dir() {
538            return Err(failure(
539                CgroupMemoryProbeFailureKind::MissingUnifiedMount,
540                leaf,
541                "resolved process cgroup is not a directory",
542            ));
543        }
544        let mut directory = leaf.to_path_buf();
545        let mut inspected_levels = 0usize;
546        let mut binding: Option<CgroupMemoryAvailability> = None;
547        loop {
548            let max_path = directory.join("memory.max");
549            let current_path = directory.join("memory.current");
550            let stat_path = directory.join("memory.stat");
551            match read_optional(&max_path)? {
552                Some(raw_limit) => {
553                    inspected_levels = inspected_levels.saturating_add(1);
554                    match parse_limit(&raw_limit, &max_path)? {
555                        CgroupMemoryLimit::Unlimited => {}
556                        CgroupMemoryLimit::Finite(limit_bytes) => {
557                            let current_before_raw =
558                                read_optional(&current_path)?.ok_or_else(|| {
559                                    failure(
560                                        CgroupMemoryProbeFailureKind::MissingCounter,
561                                        &current_path,
562                                        "finite memory.max requires memory.current",
563                                    )
564                                })?;
565                            let current_before = parse_counter(&current_before_raw, &current_path)?;
566                            let stat_raw = read_optional(&stat_path)?.ok_or_else(|| {
567                                failure(
568                                    CgroupMemoryProbeFailureKind::MissingCounter,
569                                    &stat_path,
570                                    "finite memory.max requires memory.stat",
571                                )
572                            })?;
573                            let inactive_file_bytes =
574                                parse_stat_counter(&stat_raw, "inactive_file", &stat_path)?;
575                            // `memory.current` and `memory.stat` are live files,
576                            // not an atomic snapshot. Bracket the stat read and
577                            // use the larger current value: this is conservative
578                            // for admission and prevents an ordinary concurrent
579                            // charge/uncharge from looking like malformed data.
580                            let current_after_raw =
581                                read_optional(&current_path)?.ok_or_else(|| {
582                                    failure(
583                                        CgroupMemoryProbeFailureKind::MissingCounter,
584                                        &current_path,
585                                        "memory.current disappeared during cgroup probe",
586                                    )
587                                })?;
588                            let current_after = parse_counter(&current_after_raw, &current_path)?;
589                            let current_bytes = current_before.max(current_after);
590                            if inactive_file_bytes > current_bytes {
591                                return Err(failure(
592                                    CgroupMemoryProbeFailureKind::InconsistentCounters,
593                                    &stat_path,
594                                    format!(
595                                        "inactive_file={inactive_file_bytes} exceeds memory.current={current_bytes}"
596                                    ),
597                                ));
598                            }
599                            let candidate = CgroupMemoryAvailability::from_consistent_counters(
600                                directory.display().to_string().into_boxed_str(),
601                                limit_bytes,
602                                current_bytes,
603                                inactive_file_bytes,
604                                0,
605                            )
606                            .ok_or_else(|| {
607                                failure(
608                                    CgroupMemoryProbeFailureKind::InconsistentCounters,
609                                    &stat_path,
610                                    "memory counters became inconsistent during construction",
611                                )
612                            })?;
613                            if binding.as_ref().map_or(true, |current| {
614                                candidate.available_bytes() < current.available_bytes()
615                            }) {
616                                binding = Some(candidate);
617                            }
618                        }
619                    }
620                }
621                None => {
622                    let has_current = read_optional(&current_path)?.is_some();
623                    let has_stat = read_optional(&stat_path)?.is_some();
624                    // The cgroup-v2 root is exempt from resource control and
625                    // therefore has accounting files but no `memory.max`.
626                    // Every non-root level with controller accounting must
627                    // expose its hard-limit file; otherwise the observation is
628                    // incomplete and cannot safely inherit host capacity.
629                    if directory != mount_point && (has_current || has_stat) {
630                        return Err(failure(
631                            CgroupMemoryProbeFailureKind::MissingCounter,
632                            &max_path,
633                            "memory controller files are present but memory.max is missing",
634                        ));
635                    }
636                }
637            }
638            if directory == mount_point {
639                break;
640            }
641            let Some(parent) = directory.parent() else {
642                return Err(failure(
643                    CgroupMemoryProbeFailureKind::MissingUnifiedMount,
644                    &directory,
645                    "cgroup hierarchy ended before its mount point",
646                ));
647            };
648            directory = parent.to_path_buf();
649        }
650        if let Some(mut binding) = binding {
651            binding.inspected_levels = inspected_levels;
652            Ok(CgroupMemoryObservation::V2Limited(binding))
653        } else {
654            Ok(CgroupMemoryObservation::V2Unbounded {
655                cgroup_path: leaf.display().to_string().into_boxed_str(),
656                inspected_levels,
657            })
658        }
659    }
660
661    fn inspect_visible_v1_hierarchy(
662        leaf: &Path,
663        mount_point: &Path,
664    ) -> Result<CgroupMemoryObservation, CgroupMemoryProbeFailure> {
665        if !leaf.starts_with(mount_point) {
666            return Err(failure(
667                CgroupMemoryProbeFailureKind::MissingUnifiedMount,
668                leaf,
669                format!("leaf is outside mount point {}", mount_point.display()),
670            ));
671        }
672        let leaf_metadata = fs::metadata(leaf).map_err(|error| io_failure(leaf, error))?;
673        if !leaf_metadata.is_dir() {
674            return Err(failure(
675                CgroupMemoryProbeFailureKind::MissingUnifiedMount,
676                leaf,
677                "resolved process cgroup is not a directory",
678            ));
679        }
680
681        let hierarchy_path = leaf.join("memory.use_hierarchy");
682        let hierarchy_raw = read_required(&hierarchy_path)?;
683        let hierarchical = match parse_counter(&hierarchy_raw, &hierarchy_path)? {
684            0 => false,
685            1 => true,
686            value => {
687                return Err(failure(
688                    CgroupMemoryProbeFailureKind::InvalidCounter,
689                    &hierarchy_path,
690                    format!("memory.use_hierarchy must be 0 or 1, got {value}"),
691                ));
692            }
693        };
694
695        let mut directory = leaf.to_path_buf();
696        let mut inspected_levels = 0usize;
697        let mut binding: Option<CgroupMemoryAvailability> = None;
698        loop {
699            let limit_path = directory.join("memory.limit_in_bytes");
700            let usage_path = directory.join("memory.usage_in_bytes");
701            let stat_path = directory.join("memory.stat");
702            let limit_bytes = parse_counter(&read_required(&limit_path)?, &limit_path)?;
703            let usage_before = parse_counter(&read_required(&usage_path)?, &usage_path)?;
704            let stat_raw = read_required(&stat_path)?;
705            let inactive_key = if hierarchical {
706                "total_inactive_file"
707            } else {
708                "inactive_file"
709            };
710            let inactive_file_bytes = parse_stat_counter(&stat_raw, inactive_key, &stat_path)?;
711            let usage_after = parse_counter(&read_required(&usage_path)?, &usage_path)?;
712            let current_bytes = usage_before.max(usage_after);
713            if inactive_file_bytes > current_bytes {
714                return Err(failure(
715                    CgroupMemoryProbeFailureKind::InconsistentCounters,
716                    &stat_path,
717                    format!(
718                        "{inactive_key}={inactive_file_bytes} exceeds memory.usage_in_bytes={current_bytes}"
719                    ),
720                ));
721            }
722            inspected_levels = inspected_levels.saturating_add(1);
723            let candidate = CgroupMemoryAvailability::from_consistent_counters(
724                directory.display().to_string().into_boxed_str(),
725                limit_bytes,
726                current_bytes,
727                inactive_file_bytes,
728                0,
729            )
730            .ok_or_else(|| {
731                failure(
732                    CgroupMemoryProbeFailureKind::InconsistentCounters,
733                    &stat_path,
734                    "memory counters became inconsistent during construction",
735                )
736            })?;
737            if binding
738                .as_ref()
739                .is_none_or(|current| candidate.available_bytes() < current.available_bytes())
740            {
741                binding = Some(candidate);
742            }
743
744            if !hierarchical || directory == mount_point {
745                break;
746            }
747            let Some(parent) = directory.parent() else {
748                return Err(failure(
749                    CgroupMemoryProbeFailureKind::MissingUnifiedMount,
750                    &directory,
751                    "cgroup hierarchy ended before its mount point",
752                ));
753            };
754            directory = parent.to_path_buf();
755        }
756
757        let mut binding = binding.ok_or_else(|| {
758            failure(
759                CgroupMemoryProbeFailureKind::MissingCounter,
760                leaf,
761                "cgroup-v1 memory hierarchy exposed no accounting level",
762            )
763        })?;
764        binding.inspected_levels = inspected_levels;
765        Ok(CgroupMemoryObservation::V1Limited(binding))
766    }
767
768    fn parse_limit(
769        raw: &str,
770        source: &Path,
771    ) -> Result<CgroupMemoryLimit, CgroupMemoryProbeFailure> {
772        let value = raw.trim();
773        if value == "max" {
774            return Ok(CgroupMemoryLimit::Unlimited);
775        }
776        let bytes = value.parse::<u64>().map_err(|_| {
777            failure(
778                CgroupMemoryProbeFailureKind::InvalidLimit,
779                source,
780                format!("expected literal max or an unsigned byte count, got {value:?}"),
781            )
782        })?;
783        Ok(CgroupMemoryLimit::Finite(bytes))
784    }
785
786    fn parse_counter(raw: &str, source: &Path) -> Result<u64, CgroupMemoryProbeFailure> {
787        let value = raw.trim();
788        value.parse::<u64>().map_err(|_| {
789            failure(
790                CgroupMemoryProbeFailureKind::InvalidCounter,
791                source,
792                format!("expected an unsigned byte count, got {value:?}"),
793            )
794        })
795    }
796
797    fn parse_stat_counter(
798        raw: &str,
799        key: &str,
800        source: &Path,
801    ) -> Result<u64, CgroupMemoryProbeFailure> {
802        for line in raw.lines() {
803            let mut fields = line.split_whitespace();
804            let Some(name) = fields.next() else {
805                continue;
806            };
807            if name != key {
808                continue;
809            }
810            let Some(value) = fields.next() else {
811                return Err(failure(
812                    CgroupMemoryProbeFailureKind::InvalidCounter,
813                    source,
814                    format!("memory.stat key {key:?} has no value"),
815                ));
816            };
817            if fields.next().is_some() {
818                return Err(failure(
819                    CgroupMemoryProbeFailureKind::InvalidCounter,
820                    source,
821                    format!("memory.stat key {key:?} has trailing fields"),
822                ));
823            }
824            return value.parse::<u64>().map_err(|_| {
825                failure(
826                    CgroupMemoryProbeFailureKind::InvalidCounter,
827                    source,
828                    format!("memory.stat key {key:?} is not an unsigned byte count"),
829                )
830            });
831        }
832        Err(failure(
833            CgroupMemoryProbeFailureKind::MissingCounter,
834            source,
835            format!("memory.stat is missing required key {key:?}"),
836        ))
837    }
838
839    fn read_required(path: &Path) -> Result<String, CgroupMemoryProbeFailure> {
840        fs::read_to_string(path).map_err(|error| io_failure(path, error))
841    }
842
843    fn read_optional(path: &Path) -> Result<Option<String>, CgroupMemoryProbeFailure> {
844        match fs::read_to_string(path) {
845            Ok(value) => Ok(Some(value)),
846            Err(error) if error.kind() == io::ErrorKind::NotFound => Ok(None),
847            Err(error) => Err(io_failure(path, error)),
848        }
849    }
850
851    fn io_failure(path: &Path, error: io::Error) -> CgroupMemoryProbeFailure {
852        failure(CgroupMemoryProbeFailureKind::Io, path, error.to_string())
853    }
854
855    fn failure(
856        kind: CgroupMemoryProbeFailureKind,
857        path: &Path,
858        detail: impl Into<Box<str>>,
859    ) -> CgroupMemoryProbeFailure {
860        CgroupMemoryProbeFailure::new(kind, path.display().to_string().into_boxed_str(), detail)
861    }
862
863    #[cfg(test)]
864    mod tests {
865        use super::*;
866        use std::fs;
867        use tempfile::TempDir;
868
869        struct Fixture {
870            _temp: TempDir,
871            mount: PathBuf,
872            cgroup_file: PathBuf,
873            mountinfo_file: PathBuf,
874        }
875
876        impl Fixture {
877            fn new(membership: &str) -> Self {
878                let temp = TempDir::new().expect("fixture tempdir");
879                let mount = temp.path().join("cgroup2");
880                fs::create_dir_all(&mount).expect("fixture mount");
881                let cgroup_file = temp.path().join("self.cgroup");
882                fs::write(&cgroup_file, format!("0::{membership}\n")).expect("fixture membership");
883                let mountinfo_file = temp.path().join("self.mountinfo");
884                fs::write(
885                    &mountinfo_file,
886                    format!(
887                        "29 23 0:26 / {} rw,nosuid,nodev,noexec,relatime - cgroup2 cgroup rw\n",
888                        mount.display()
889                    ),
890                )
891                .expect("fixture mountinfo");
892                Self {
893                    _temp: temp,
894                    mount,
895                    cgroup_file,
896                    mountinfo_file,
897                }
898            }
899
900            fn level(&self, relative: &str, limit: &str, current: u64, inactive: u64) {
901                let directory = self.mount.join(relative.trim_start_matches('/'));
902                fs::create_dir_all(&directory).expect("fixture level");
903                fs::write(directory.join("memory.max"), format!("{limit}\n"))
904                    .expect("fixture memory.max");
905                fs::write(directory.join("memory.current"), format!("{current}\n"))
906                    .expect("fixture memory.current");
907                fs::write(
908                    directory.join("memory.stat"),
909                    format!("anon 1\ninactive_file {inactive}\nactive_file 2\n"),
910                )
911                .expect("fixture memory.stat");
912            }
913
914            fn observe(&self) -> CgroupMemoryObservation {
915                detect_from_proc_files(&self.cgroup_file, &self.mountinfo_file)
916                    .unwrap_or_else(CgroupMemoryObservation::ProbeFailed)
917            }
918        }
919
920        #[test]
921        fn literal_max_is_typed_unbounded_even_when_current_is_large() {
922            let fixture = Fixture::new("/tenant/leaf");
923            fixture.level("tenant/leaf", "max", u64::MAX - 1, 0);
924            assert!(matches!(
925                fixture.observe(),
926                CgroupMemoryObservation::V2Unbounded {
927                    inspected_levels: 1,
928                    ..
929                }
930            ));
931        }
932
933        #[test]
934        fn cgroup_v2_root_accounting_without_memory_max_is_unbounded() {
935            let fixture = Fixture::new("/");
936            fs::write(fixture.mount.join("memory.current"), "1000\n").expect("root current");
937            fs::write(fixture.mount.join("memory.stat"), "inactive_file 700\n").expect("root stat");
938            assert!(matches!(
939                fixture.observe(),
940                CgroupMemoryObservation::V2Unbounded {
941                    inspected_levels: 0,
942                    ..
943                }
944            ));
945        }
946
947        #[test]
948        fn finite_zero_remains_authoritative() {
949            let fixture = Fixture::new("/tenant/leaf");
950            fixture.level("tenant/leaf", "0", 0, 0);
951            let CgroupMemoryObservation::V2Limited(observation) = fixture.observe() else {
952                panic!("finite zero must be a real cgroup ceiling");
953            };
954            assert_eq!(observation.limit_bytes(), 0);
955            assert_eq!(observation.available_bytes(), 0);
956        }
957
958        #[test]
959        fn inactive_file_cache_is_conservatively_reclaimable() {
960            let fixture = Fixture::new("/tenant/leaf");
961            fixture.level("tenant/leaf", "1000", 1000, 700);
962            let CgroupMemoryObservation::V2Limited(observation) = fixture.observe() else {
963                panic!("finite ceiling must bind");
964            };
965            assert_eq!(observation.working_set_bytes(), 300);
966            assert_eq!(observation.available_bytes(), 700);
967        }
968
969        #[test]
970        fn finite_working_set_exhaustion_is_zero() {
971            let fixture = Fixture::new("/tenant/leaf");
972            fixture.level("tenant/leaf", "1000", 1200, 200);
973            let CgroupMemoryObservation::V2Limited(observation) = fixture.observe() else {
974                panic!("finite ceiling must bind");
975            };
976            assert_eq!(observation.working_set_bytes(), 1000);
977            assert_eq!(observation.available_bytes(), 0);
978        }
979
980        #[test]
981        fn binding_parent_accounts_for_sibling_pressure() {
982            let fixture = Fixture::new("/tenant/leaf");
983            fixture.level("tenant/leaf", "max", 200, 100);
984            fixture.level("tenant", "1000", 950, 100);
985            let CgroupMemoryObservation::V2Limited(observation) = fixture.observe() else {
986                panic!("finite parent must bind an unlimited leaf");
987            };
988            assert_eq!(observation.available_bytes(), 150);
989            assert!(observation.binding_path().ends_with("/tenant"));
990            assert_eq!(observation.inspected_levels(), 2);
991        }
992
993        #[test]
994        fn most_specific_covering_mount_resolves_non_root_membership() {
995            let temp = TempDir::new().expect("fixture tempdir");
996            let broad = temp.path().join("broad");
997            let narrow = temp.path().join("narrow");
998            fs::create_dir_all(&broad).expect("broad mount");
999            fs::create_dir_all(narrow.join("leaf")).expect("narrow leaf");
1000            fs::write(narrow.join("leaf/memory.max"), "512\n").expect("max");
1001            fs::write(narrow.join("leaf/memory.current"), "256\n").expect("current");
1002            fs::write(narrow.join("leaf/memory.stat"), "inactive_file 64\n").expect("stat");
1003            let cgroup_file = temp.path().join("self.cgroup");
1004            fs::write(&cgroup_file, "0::/tenant/leaf\n").expect("membership");
1005            let mountinfo_file = temp.path().join("self.mountinfo");
1006            fs::write(
1007                &mountinfo_file,
1008                format!(
1009                    "29 23 0:26 / {} rw - cgroup2 cgroup rw\n30 23 0:26 /tenant {} rw - cgroup2 cgroup rw\n",
1010                    broad.display(),
1011                    narrow.display()
1012                ),
1013            )
1014            .expect("mountinfo");
1015            let observation = detect_from_proc_files(&cgroup_file, &mountinfo_file)
1016                .expect("typed cgroup observation");
1017            let CgroupMemoryObservation::V2Limited(observation) = observation else {
1018                panic!("narrow finite mount must bind");
1019            };
1020            assert_eq!(observation.available_bytes(), 320);
1021            assert!(observation.binding_path().ends_with("/narrow/leaf"));
1022        }
1023
1024        #[test]
1025        fn malformed_active_controller_fails_closed() {
1026            let fixture = Fixture::new("/tenant/leaf");
1027            fixture.level("tenant/leaf", "1000", 500, 100);
1028            fs::write(
1029                fixture.mount.join("tenant/leaf/memory.current"),
1030                "not-a-counter\n",
1031            )
1032            .expect("corrupt current");
1033            let CgroupMemoryObservation::ProbeFailed(failure) = fixture.observe() else {
1034                panic!("malformed active controller must fail closed");
1035            };
1036            assert_eq!(failure.kind(), CgroupMemoryProbeFailureKind::InvalidCounter);
1037        }
1038
1039        #[test]
1040        fn inconsistent_cache_counter_fails_closed() {
1041            let fixture = Fixture::new("/tenant/leaf");
1042            fixture.level("tenant/leaf", "1000", 100, 101);
1043            let CgroupMemoryObservation::ProbeFailed(failure) = fixture.observe() else {
1044                panic!("inconsistent controller counters must fail closed");
1045            };
1046            assert_eq!(
1047                failure.kind(),
1048                CgroupMemoryProbeFailureKind::InconsistentCounters
1049            );
1050        }
1051
1052        #[test]
1053        fn active_cgroup_v1_memory_controller_is_measured_exactly() {
1054            let temp = TempDir::new().expect("fixture tempdir");
1055            let mount = temp.path().join("cgroup-memory");
1056            let leaf = mount.join("legacy");
1057            fs::create_dir_all(&leaf).expect("v1 leaf");
1058            fs::write(leaf.join("memory.use_hierarchy"), "0\n").expect("hierarchy");
1059            fs::write(leaf.join("memory.limit_in_bytes"), "1024\n").expect("limit");
1060            fs::write(leaf.join("memory.usage_in_bytes"), "512\n").expect("usage");
1061            fs::write(
1062                leaf.join("memory.stat"),
1063                "inactive_file 128\ntotal_inactive_file 128\n",
1064            )
1065            .expect("stat");
1066            let cgroup_file = temp.path().join("self.cgroup");
1067            fs::write(&cgroup_file, "4:memory:/legacy\n").expect("membership");
1068            let mountinfo_file = temp.path().join("self.mountinfo");
1069            fs::write(
1070                &mountinfo_file,
1071                format!(
1072                    "31 23 0:30 / {} rw - cgroup cgroup rw,memory\n",
1073                    mount.display()
1074                ),
1075            )
1076            .expect("mountinfo");
1077            let CgroupMemoryObservation::V1Limited(observation) =
1078                detect_from_proc_files(&cgroup_file, &mountinfo_file)
1079                    .expect("v1 memory observation")
1080            else {
1081                panic!("v1 memory ceiling must be authoritative");
1082            };
1083            assert_eq!(observation.working_set_bytes(), 384);
1084            assert_eq!(observation.available_bytes(), 640);
1085        }
1086
1087        #[test]
1088        fn hybrid_hierarchy_uses_the_active_v1_memory_controller() {
1089            let fixture = Fixture::new("/tenant/leaf");
1090            let v1_mount = fixture.mount.parent().unwrap().join("cgroup-memory");
1091            let v1_leaf = v1_mount.join("legacy");
1092            fs::create_dir_all(&v1_leaf).expect("v1 leaf");
1093            fs::write(v1_leaf.join("memory.use_hierarchy"), "0\n").expect("hierarchy");
1094            fs::write(v1_leaf.join("memory.limit_in_bytes"), "2048\n").expect("limit");
1095            fs::write(v1_leaf.join("memory.usage_in_bytes"), "1024\n").expect("usage");
1096            fs::write(
1097                v1_leaf.join("memory.stat"),
1098                "inactive_file 256\ntotal_inactive_file 256\n",
1099            )
1100            .expect("stat");
1101            fs::write(&fixture.cgroup_file, "0::/tenant/leaf\n4:memory:/legacy\n")
1102                .expect("hybrid membership");
1103            let original = fs::read_to_string(&fixture.mountinfo_file).expect("mountinfo");
1104            fs::write(
1105                &fixture.mountinfo_file,
1106                format!(
1107                    "{original}31 23 0:30 / {} rw - cgroup cgroup rw,memory\n",
1108                    v1_mount.display()
1109                ),
1110            )
1111            .expect("hybrid mountinfo");
1112            let CgroupMemoryObservation::V1Limited(observation) = fixture.observe() else {
1113                panic!("hybrid memory accounting must follow v1");
1114            };
1115            assert_eq!(observation.available_bytes(), 1280);
1116        }
1117    }
1118}
1119
1120#[cfg(target_os = "linux")]
1121pub(crate) fn detect_cgroup_memory() -> CgroupMemoryObservation {
1122    linux::detect()
1123}
1124
1125#[cfg(all(test, not(target_os = "linux")))]
1126mod non_linux_tests {
1127    use super::*;
1128
1129    #[test]
1130    fn non_linux_platform_has_no_cgroup_controller() {
1131        assert_eq!(detect_cgroup_memory(), CgroupMemoryObservation::NotPresent);
1132    }
1133}
gam_runtime/cgroup_memory.rs

gam_runtime/
cgroup_memory.rs
