gam_runtime/
cgroup_memory.rs

1//! Typed Linux cgroup memory observations.
2//!
3//! `sysinfo::CGroupLimits` is intentionally not used here. Its aggregate loses
4//! the distinction between the literal cgroup-v2 `memory.max = max` token and a
5//! finite limit, derives "free" as `max - memory.current` even though
6//! `memory.current` includes reclaimable file cache, and probes a fixed cgroup
7//! path rather than the current process' hierarchy. Memory admission needs the
8//! kernel contract before any of that information is erased.
9
10use std::fmt;
11
12/// The exact syntax of one cgroup-v2 `memory.max` value.
13#[derive(Clone, Copy, Debug, PartialEq, Eq)]
14pub enum CgroupMemoryLimit {
15    /// The literal `max` token: this level imposes no hard memory ceiling.
16    Unlimited,
17    /// A finite hard ceiling in bytes. Zero is valid and authoritative.
18    Finite(u64),
19}
20
21/// Why a live cgroup memory hierarchy could not be observed safely.
22#[derive(Clone, Copy, Debug, PartialEq, Eq)]
23pub enum CgroupMemoryProbeFailureKind {
24    MalformedMembership,
25    MissingUnifiedMount,
26    MalformedMountInfo,
27    Io,
28    InvalidLimit,
29    InvalidCounter,
30    MissingCounter,
31    InconsistentCounters,
32}
33
34impl fmt::Display for CgroupMemoryProbeFailureKind {
35    fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
36        let name = match self {
37            Self::MalformedMembership => "malformed-membership",
38            Self::MissingUnifiedMount => "missing-unified-mount",
39            Self::MalformedMountInfo => "malformed-mountinfo",
40            Self::Io => "io",
41            Self::InvalidLimit => "invalid-limit",
42            Self::InvalidCounter => "invalid-counter",
43            Self::MissingCounter => "missing-counter",
44            Self::InconsistentCounters => "inconsistent-counters",
45        };
46        formatter.write_str(name)
47    }
48}
49
50/// Fail-closed evidence from an active cgroup controller probe.
51#[derive(Clone, Debug, PartialEq, Eq)]
52pub struct CgroupMemoryProbeFailure {
53    kind: CgroupMemoryProbeFailureKind,
54    path: Box<str>,
55    detail: Box<str>,
56}
57
58impl CgroupMemoryProbeFailure {
59    pub const fn kind(&self) -> CgroupMemoryProbeFailureKind {
60        self.kind
61    }
62
63    pub fn path(&self) -> &str {
64        &self.path
65    }
66
67    pub fn detail(&self) -> &str {
68        &self.detail
69    }
70}
71
72impl fmt::Display for CgroupMemoryProbeFailure {
73    fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
74        write!(formatter, "{} at {}: {}", self.kind, self.path, self.detail)
75    }
76}
77
78/// Reclaim-aware headroom under the binding finite cgroup ancestor.
79///
80/// `working_set_bytes = memory.current - inactive_file`. Only inactive file
81/// cache is credited as reclaimable; active file cache and reclaimable slab are
82/// deliberately left in the working set. The governor's separate 1/4 headroom
83/// remains available for reclaim latency, allocator slack, and untracked work.
84#[derive(Clone, Debug, PartialEq, Eq)]
85pub struct CgroupMemoryAvailability {
86    binding_path: Box<str>,
87    limit_bytes: u64,
88    current_bytes: u64,
89    inactive_file_bytes: u64,
90    working_set_bytes: u64,
91    available_bytes: u64,
92    inspected_levels: usize,
93}
94
95impl CgroupMemoryAvailability {
96    /// Assemble one observation from counters that were read together.
97    ///
98    /// `None` when `inactive_file > current`, which is a torn read of the two
99    /// counters rather than a legal state: the working set would be negative.
100    /// This is the single construction site — the live probe, the replay used
101    /// by [`crate::test_support::simulated_cgroup_memory_environment`], and the
102    /// unit fixtures all funnel through it, so the derived `working_set` and
103    /// `available` can never disagree between a measured and a replayed
104    /// environment.
105    pub(crate) fn from_consistent_counters(
106        binding_path: impl Into<Box<str>>,
107        limit_bytes: u64,
108        current_bytes: u64,
109        inactive_file_bytes: u64,
110        inspected_levels: usize,
111    ) -> Option<Self> {
112        let working_set_bytes = current_bytes.checked_sub(inactive_file_bytes)?;
113        Some(Self {
114            binding_path: binding_path.into(),
115            limit_bytes,
116            current_bytes,
117            inactive_file_bytes,
118            working_set_bytes,
119            available_bytes: limit_bytes.saturating_sub(working_set_bytes),
120            inspected_levels,
121        })
122    }
123
124    pub const fn limit_bytes(&self) -> u64 {
125        self.limit_bytes
126    }
127
128    pub const fn current_bytes(&self) -> u64 {
129        self.current_bytes
130    }
131
132    pub const fn available_bytes(&self) -> u64 {
133        self.available_bytes
134    }
135
136}
137
138#[cfg(test)]
139mod tests_fixtures {
140    use super::*;
141
142    impl CgroupMemoryProbeFailure {
143    }
144
145    impl CgroupMemoryAvailability {
146    }
147}
148
149impl fmt::Display for CgroupMemoryAvailability {
150    fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
151        write!(
152            formatter,
153            "finite cgroup ceiling at {} (limit={}, current={}, inactive_file={}, working_set={}, available={}, visible_levels={})",
154            self.binding_path,
155            self.limit_bytes,
156            self.current_bytes,
157            self.inactive_file_bytes,
158            self.working_set_bytes,
159            self.available_bytes,
160            self.inspected_levels,
161        )
162    }
163}
164
165/// The process' typed cgroup memory provenance.
166#[derive(Clone, Debug, PartialEq, Eq)]
167pub enum CgroupMemoryObservation {
168    /// No active memory controller applies on this platform/hierarchy.
169    NotPresent,
170    /// A cgroup-v2 hierarchy was found, but every visible hard limit was the
171    /// literal `max` token (or the process is at the unconstrained root).
172    V2Unbounded {
173        cgroup_path: Box<str>,
174        inspected_levels: usize,
175    },
176    /// At least one finite hard ceiling applies. This carries the ancestor with
177    /// the least reclaim-aware headroom.
178    V2Limited(CgroupMemoryAvailability),
179    /// A cgroup-v1 memory hierarchy is active. V1 exposes its unlimited state
180    /// as an architecture-dependent numeric sentinel rather than a token, so
181    /// every visible numeric ceiling participates in the minimum; an enormous
182    /// sentinel naturally loses to host availability.
183    V1Limited(CgroupMemoryAvailability),
184    /// A memory controller appears active but its semantics could not be read
185    /// exactly. Admission must fail closed rather than inherit host memory.
186    ProbeFailed(CgroupMemoryProbeFailure),
187}
188
189impl fmt::Display for CgroupMemoryObservation {
190    fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
191        match self {
192            Self::NotPresent => formatter.write_str("no active cgroup memory controller"),
193            Self::V2Unbounded {
194                cgroup_path,
195                inspected_levels,
196            } => write!(
197                formatter,
198                "unbounded cgroup-v2 hierarchy at {cgroup_path} ({inspected_levels} memory.max levels)"
199            ),
200            Self::V2Limited(observation) => write!(formatter, "cgroup-v2 {observation}"),
201            Self::V1Limited(observation) => write!(formatter, "cgroup-v1 {observation}"),
202            Self::ProbeFailed(failure) => write!(formatter, "cgroup probe failed: {failure}"),
203        }
204    }
205}
206
207#[cfg(not(target_os = "linux"))]
208pub(crate) fn detect_cgroup_memory() -> CgroupMemoryObservation {
209    CgroupMemoryObservation::NotPresent
210}
211
212#[cfg(target_os = "linux")]
213mod linux {
214    use super::*;
215    use std::ffi::OsString;
216    use std::fs;
217    use std::io;
218    use std::os::unix::ffi::OsStringExt;
219    use std::path::{Component, Path, PathBuf};
220
221    impl CgroupMemoryProbeFailure {
222        fn new(
223            kind: CgroupMemoryProbeFailureKind,
224            path: impl Into<Box<str>>,
225            detail: impl Into<Box<str>>,
226        ) -> Self {
227            Self {
228                kind,
229                path: path.into(),
230                detail: detail.into(),
231            }
232        }
233    }
234
235    const PROC_SELF_CGROUP: &str = "/proc/self/cgroup";
236    const PROC_SELF_MOUNTINFO: &str = "/proc/self/mountinfo";
237
238    #[derive(Debug)]
239    struct ControllerMount {
240        root: PathBuf,
241        mount_point: PathBuf,
242    }
243
244    #[derive(Debug)]
245    struct MemoryMembership {
246        unified: Option<PathBuf>,
247        legacy: Option<PathBuf>,
248    }
249
250    pub(super) fn detect() -> CgroupMemoryObservation {
251        detect_from_proc_files(Path::new(PROC_SELF_CGROUP), Path::new(PROC_SELF_MOUNTINFO))
252            .unwrap_or_else(CgroupMemoryObservation::ProbeFailed)
253    }
254
255    fn detect_from_proc_files(
256        cgroup_file: &Path,
257        mountinfo_file: &Path,
258    ) -> Result<CgroupMemoryObservation, CgroupMemoryProbeFailure> {
259        let membership_text = read_required(cgroup_file)?;
260        let membership = parse_memory_membership(&membership_text, cgroup_file)?;
261        let mountinfo_text = read_required(mountinfo_file)?;
262        if let Some(legacy) = membership.legacy {
263            let mount = select_controller_mount(
264                &mountinfo_text,
265                &legacy,
266                mountinfo_file,
267                "cgroup",
268                Some("memory"),
269            )?;
270            let leaf = resolve_membership_leaf(&legacy, &mount, mountinfo_file)?;
271            return inspect_visible_v1_hierarchy(&leaf, &mount.mount_point);
272        }
273        if let Some(unified) = membership.unified {
274            let mount = select_controller_mount(
275                &mountinfo_text,
276                &unified,
277                mountinfo_file,
278                "cgroup2",
279                None,
280            )?;
281            let leaf = resolve_membership_leaf(&unified, &mount, mountinfo_file)?;
282            return inspect_visible_v2_hierarchy(&leaf, &mount.mount_point);
283        }
284        Ok(CgroupMemoryObservation::NotPresent)
285    }
286
287    fn parse_memory_membership(
288        text: &str,
289        source: &Path,
290    ) -> Result<MemoryMembership, CgroupMemoryProbeFailure> {
291        let mut unified = None;
292        let mut legacy = None;
293        for line in text.lines().filter(|line| !line.is_empty()) {
294            let mut fields = line.splitn(3, ':');
295            let hierarchy = fields.next();
296            let controllers = fields.next();
297            let path = fields.next();
298            let (Some(hierarchy), Some(controllers), Some(path)) = (hierarchy, controllers, path)
299            else {
300                return Err(failure(
301                    CgroupMemoryProbeFailureKind::MalformedMembership,
302                    source,
303                    format!("invalid /proc/self/cgroup record {line:?}"),
304                ));
305            };
306            if controllers.split(',').any(|name| name == "memory") {
307                if legacy.is_some() {
308                    return Err(failure(
309                        CgroupMemoryProbeFailureKind::MalformedMembership,
310                        source,
311                        "multiple cgroup-v1 memory memberships",
312                    ));
313                }
314                let path = PathBuf::from(path);
315                validate_absolute_cgroup_path(&path, source)?;
316                legacy = Some(path);
317            }
318            if hierarchy == "0" && controllers.is_empty() {
319                if unified.is_some() {
320                    return Err(failure(
321                        CgroupMemoryProbeFailureKind::MalformedMembership,
322                        source,
323                        "multiple unified cgroup-v2 memberships",
324                    ));
325                }
326                let path = PathBuf::from(path);
327                validate_absolute_cgroup_path(&path, source)?;
328                unified = Some(path);
329            }
330        }
331        Ok(MemoryMembership { unified, legacy })
332    }
333
334    fn validate_absolute_cgroup_path(
335        path: &Path,
336        source: &Path,
337    ) -> Result<(), CgroupMemoryProbeFailure> {
338        if !path.is_absolute()
339            || path
340                .components()
341                .any(|component| matches!(component, Component::ParentDir))
342        {
343            return Err(failure(
344                CgroupMemoryProbeFailureKind::MalformedMembership,
345                source,
346                format!("invalid unified cgroup path {}", path.display()),
347            ));
348        }
349        Ok(())
350    }
351
352    fn select_controller_mount(
353        text: &str,
354        membership: &Path,
355        source: &Path,
356        filesystem: &str,
357        required_super_option: Option<&str>,
358    ) -> Result<ControllerMount, CgroupMemoryProbeFailure> {
359        let mut selected: Option<ControllerMount> = None;
360        for line in text.lines().filter(|line| !line.is_empty()) {
361            let Some((before_separator, after_separator)) = line.split_once(" - ") else {
362                return Err(failure(
363                    CgroupMemoryProbeFailureKind::MalformedMountInfo,
364                    source,
365                    format!("mountinfo record has no separator: {line:?}"),
366                ));
367            };
368            let after_fields = after_separator.split_whitespace().collect::<Vec<_>>();
369            if after_fields.first().copied() != Some(filesystem) {
370                continue;
371            }
372            if let Some(required) = required_super_option
373                && !after_fields
374                    .get(2)
375                    .is_some_and(|options| options.split(',').any(|option| option == required))
376            {
377                continue;
378            }
379            let before_fields = before_separator.split_whitespace().collect::<Vec<_>>();
380            if before_fields.len() < 6 {
381                return Err(failure(
382                    CgroupMemoryProbeFailureKind::MalformedMountInfo,
383                    source,
384                    format!("short {filesystem} mountinfo record: {line:?}"),
385                ));
386            }
387            let root = decode_mountinfo_path(before_fields[3], source)?;
388            let mount_point = decode_mountinfo_path(before_fields[4], source)?;
389            if !root.is_absolute() || !mount_point.is_absolute() {
390                return Err(failure(
391                    CgroupMemoryProbeFailureKind::MalformedMountInfo,
392                    source,
393                    format!(
394                        "{filesystem} mount paths must be absolute (root={}, mount_point={})",
395                        root.display(),
396                        mount_point.display()
397                    ),
398                ));
399            }
400            if membership.strip_prefix(&root).is_err() {
401                continue;
402            }
403            let candidate = ControllerMount { root, mount_point };
404            let candidate_depth = candidate.root.components().count();
405            let selected_depth = selected
406                .as_ref()
407                .map_or(0, |mount| mount.root.components().count());
408            if selected.is_none() || candidate_depth > selected_depth {
409                selected = Some(candidate);
410            }
411        }
412        selected.ok_or_else(|| {
413            failure(
414                CgroupMemoryProbeFailureKind::MissingUnifiedMount,
415                source,
416                format!(
417                    "no {filesystem} memory mount covers process membership {}",
418                    membership.display()
419                ),
420            )
421        })
422    }
423
424    fn resolve_membership_leaf(
425        membership: &Path,
426        mount: &ControllerMount,
427        source: &Path,
428    ) -> Result<PathBuf, CgroupMemoryProbeFailure> {
429        let relative = membership.strip_prefix(&mount.root).map_err(|_| {
430            failure(
431                CgroupMemoryProbeFailureKind::MissingUnifiedMount,
432                source,
433                format!(
434                    "cgroup path {} is outside selected mount root {}",
435                    membership.display(),
436                    mount.root.display()
437                ),
438            )
439        })?;
440        let relative = relative.strip_prefix(Path::new("/")).unwrap_or(relative);
441        Ok(mount.mount_point.join(relative))
442    }
443
444    fn decode_mountinfo_path(
445        raw: &str,
446        source: &Path,
447    ) -> Result<PathBuf, CgroupMemoryProbeFailure> {
448        let bytes = raw.as_bytes();
449        let mut decoded = Vec::with_capacity(bytes.len());
450        let mut index = 0;
451        while index < bytes.len() {
452            if bytes[index] != b'\\' {
453                decoded.push(bytes[index]);
454                index += 1;
455                continue;
456            }
457            let octal = bytes
458                .get(index + 1..index + 4)
459                .filter(|digits| digits.iter().all(|digit| (b'0'..=b'7').contains(digit)));
460            let Some(octal) = octal else {
461                return Err(failure(
462                    CgroupMemoryProbeFailureKind::MalformedMountInfo,
463                    source,
464                    format!("invalid mountinfo path escape in {raw:?}"),
465                ));
466            };
467            let value = u16::from(octal[0] - b'0') * 64
468                + u16::from(octal[1] - b'0') * 8
469                + u16::from(octal[2] - b'0');
470            let value = u8::try_from(value).map_err(|_| {
471                failure(
472                    CgroupMemoryProbeFailureKind::MalformedMountInfo,
473                    source,
474                    format!("mountinfo path escape exceeds one byte in {raw:?}"),
475                )
476            })?;
477            decoded.push(value);
478            index += 4;
479        }
480        Ok(PathBuf::from(OsString::from_vec(decoded)))
481    }
482
483    fn inspect_visible_v2_hierarchy(
484        leaf: &Path,
485        mount_point: &Path,
486    ) -> Result<CgroupMemoryObservation, CgroupMemoryProbeFailure> {
487        if !leaf.starts_with(mount_point) {
488            return Err(failure(
489                CgroupMemoryProbeFailureKind::MissingUnifiedMount,
490                leaf,
491                format!("leaf is outside mount point {}", mount_point.display()),
492            ));
493        }
494        let leaf_metadata = fs::metadata(leaf).map_err(|error| io_failure(leaf, error))?;
495        if !leaf_metadata.is_dir() {
496            return Err(failure(
497                CgroupMemoryProbeFailureKind::MissingUnifiedMount,
498                leaf,
499                "resolved process cgroup is not a directory",
500            ));
501        }
502        let mut directory = leaf.to_path_buf();
503        let mut inspected_levels = 0usize;
504        let mut binding: Option<CgroupMemoryAvailability> = None;
505        loop {
506            let max_path = directory.join("memory.max");
507            let current_path = directory.join("memory.current");
508            let stat_path = directory.join("memory.stat");
509            match read_optional(&max_path)? {
510                Some(raw_limit) => {
511                    inspected_levels = inspected_levels.saturating_add(1);
512                    match parse_limit(&raw_limit, &max_path)? {
513                        CgroupMemoryLimit::Unlimited => {}
514                        CgroupMemoryLimit::Finite(limit_bytes) => {
515                            let current_before_raw =
516                                read_optional(&current_path)?.ok_or_else(|| {
517                                    failure(
518                                        CgroupMemoryProbeFailureKind::MissingCounter,
519                                        &current_path,
520                                        "finite memory.max requires memory.current",
521                                    )
522                                })?;
523                            let current_before = parse_counter(&current_before_raw, &current_path)?;
524                            let stat_raw = read_optional(&stat_path)?.ok_or_else(|| {
525                                failure(
526                                    CgroupMemoryProbeFailureKind::MissingCounter,
527                                    &stat_path,
528                                    "finite memory.max requires memory.stat",
529                                )
530                            })?;
531                            let inactive_file_bytes =
532                                parse_stat_counter(&stat_raw, "inactive_file", &stat_path)?;
533                            // `memory.current` and `memory.stat` are live files,
534                            // not an atomic snapshot. Bracket the stat read and
535                            // use the larger current value: this is conservative
536                            // for admission and prevents an ordinary concurrent
537                            // charge/uncharge from looking like malformed data.
538                            let current_after_raw =
539                                read_optional(&current_path)?.ok_or_else(|| {
540                                    failure(
541                                        CgroupMemoryProbeFailureKind::MissingCounter,
542                                        &current_path,
543                                        "memory.current disappeared during cgroup probe",
544                                    )
545                                })?;
546                            let current_after = parse_counter(&current_after_raw, &current_path)?;
547                            let current_bytes = current_before.max(current_after);
548                            if inactive_file_bytes > current_bytes {
549                                return Err(failure(
550                                    CgroupMemoryProbeFailureKind::InconsistentCounters,
551                                    &stat_path,
552                                    format!(
553                                        "inactive_file={inactive_file_bytes} exceeds memory.current={current_bytes}"
554                                    ),
555                                ));
556                            }
557                            let candidate = CgroupMemoryAvailability::from_consistent_counters(
558                                directory.display().to_string().into_boxed_str(),
559                                limit_bytes,
560                                current_bytes,
561                                inactive_file_bytes,
562                                0,
563                            )
564                            .ok_or_else(|| {
565                                failure(
566                                    CgroupMemoryProbeFailureKind::InconsistentCounters,
567                                    &stat_path,
568                                    "memory counters became inconsistent during construction",
569                                )
570                            })?;
571                            if binding.as_ref().map_or(true, |current| {
572                                candidate.available_bytes() < current.available_bytes()
573                            }) {
574                                binding = Some(candidate);
575                            }
576                        }
577                    }
578                }
579                None => {
580                    let has_current = read_optional(&current_path)?.is_some();
581                    let has_stat = read_optional(&stat_path)?.is_some();
582                    // The cgroup-v2 root is exempt from resource control and
583                    // therefore has accounting files but no `memory.max`.
584                    // Every non-root level with controller accounting must
585                    // expose its hard-limit file; otherwise the observation is
586                    // incomplete and cannot safely inherit host capacity.
587                    if directory != mount_point && (has_current || has_stat) {
588                        return Err(failure(
589                            CgroupMemoryProbeFailureKind::MissingCounter,
590                            &max_path,
591                            "memory controller files are present but memory.max is missing",
592                        ));
593                    }
594                }
595            }
596            if directory == mount_point {
597                break;
598            }
599            let Some(parent) = directory.parent() else {
600                return Err(failure(
601                    CgroupMemoryProbeFailureKind::MissingUnifiedMount,
602                    &directory,
603                    "cgroup hierarchy ended before its mount point",
604                ));
605            };
606            directory = parent.to_path_buf();
607        }
608        if let Some(mut binding) = binding {
609            binding.inspected_levels = inspected_levels;
610            Ok(CgroupMemoryObservation::V2Limited(binding))
611        } else {
612            Ok(CgroupMemoryObservation::V2Unbounded {
613                cgroup_path: leaf.display().to_string().into_boxed_str(),
614                inspected_levels,
615            })
616        }
617    }
618
619    fn inspect_visible_v1_hierarchy(
620        leaf: &Path,
621        mount_point: &Path,
622    ) -> Result<CgroupMemoryObservation, CgroupMemoryProbeFailure> {
623        if !leaf.starts_with(mount_point) {
624            return Err(failure(
625                CgroupMemoryProbeFailureKind::MissingUnifiedMount,
626                leaf,
627                format!("leaf is outside mount point {}", mount_point.display()),
628            ));
629        }
630        let leaf_metadata = fs::metadata(leaf).map_err(|error| io_failure(leaf, error))?;
631        if !leaf_metadata.is_dir() {
632            return Err(failure(
633                CgroupMemoryProbeFailureKind::MissingUnifiedMount,
634                leaf,
635                "resolved process cgroup is not a directory",
636            ));
637        }
638
639        let hierarchy_path = leaf.join("memory.use_hierarchy");
640        let hierarchy_raw = read_required(&hierarchy_path)?;
641        let hierarchical = match parse_counter(&hierarchy_raw, &hierarchy_path)? {
642            0 => false,
643            1 => true,
644            value => {
645                return Err(failure(
646                    CgroupMemoryProbeFailureKind::InvalidCounter,
647                    &hierarchy_path,
648                    format!("memory.use_hierarchy must be 0 or 1, got {value}"),
649                ));
650            }
651        };
652
653        let mut directory = leaf.to_path_buf();
654        let mut inspected_levels = 0usize;
655        let mut binding: Option<CgroupMemoryAvailability> = None;
656        loop {
657            let limit_path = directory.join("memory.limit_in_bytes");
658            let usage_path = directory.join("memory.usage_in_bytes");
659            let stat_path = directory.join("memory.stat");
660            let limit_bytes = parse_counter(&read_required(&limit_path)?, &limit_path)?;
661            let usage_before = parse_counter(&read_required(&usage_path)?, &usage_path)?;
662            let stat_raw = read_required(&stat_path)?;
663            let inactive_key = if hierarchical {
664                "total_inactive_file"
665            } else {
666                "inactive_file"
667            };
668            let inactive_file_bytes = parse_stat_counter(&stat_raw, inactive_key, &stat_path)?;
669            let usage_after = parse_counter(&read_required(&usage_path)?, &usage_path)?;
670            let current_bytes = usage_before.max(usage_after);
671            if inactive_file_bytes > current_bytes {
672                return Err(failure(
673                    CgroupMemoryProbeFailureKind::InconsistentCounters,
674                    &stat_path,
675                    format!(
676                        "{inactive_key}={inactive_file_bytes} exceeds memory.usage_in_bytes={current_bytes}"
677                    ),
678                ));
679            }
680            inspected_levels = inspected_levels.saturating_add(1);
681            let candidate = CgroupMemoryAvailability::from_consistent_counters(
682                directory.display().to_string().into_boxed_str(),
683                limit_bytes,
684                current_bytes,
685                inactive_file_bytes,
686                0,
687            )
688            .ok_or_else(|| {
689                failure(
690                    CgroupMemoryProbeFailureKind::InconsistentCounters,
691                    &stat_path,
692                    "memory counters became inconsistent during construction",
693                )
694            })?;
695            if binding
696                .as_ref()
697                .is_none_or(|current| candidate.available_bytes() < current.available_bytes())
698            {
699                binding = Some(candidate);
700            }
701
702            if !hierarchical || directory == mount_point {
703                break;
704            }
705            let Some(parent) = directory.parent() else {
706                return Err(failure(
707                    CgroupMemoryProbeFailureKind::MissingUnifiedMount,
708                    &directory,
709                    "cgroup hierarchy ended before its mount point",
710                ));
711            };
712            directory = parent.to_path_buf();
713        }
714
715        let mut binding = binding.ok_or_else(|| {
716            failure(
717                CgroupMemoryProbeFailureKind::MissingCounter,
718                leaf,
719                "cgroup-v1 memory hierarchy exposed no accounting level",
720            )
721        })?;
722        binding.inspected_levels = inspected_levels;
723        Ok(CgroupMemoryObservation::V1Limited(binding))
724    }
725
726    fn parse_limit(
727        raw: &str,
728        source: &Path,
729    ) -> Result<CgroupMemoryLimit, CgroupMemoryProbeFailure> {
730        let value = raw.trim();
731        if value == "max" {
732            return Ok(CgroupMemoryLimit::Unlimited);
733        }
734        let bytes = value.parse::<u64>().map_err(|_| {
735            failure(
736                CgroupMemoryProbeFailureKind::InvalidLimit,
737                source,
738                format!("expected literal max or an unsigned byte count, got {value:?}"),
739            )
740        })?;
741        Ok(CgroupMemoryLimit::Finite(bytes))
742    }
743
744    fn parse_counter(raw: &str, source: &Path) -> Result<u64, CgroupMemoryProbeFailure> {
745        let value = raw.trim();
746        value.parse::<u64>().map_err(|_| {
747            failure(
748                CgroupMemoryProbeFailureKind::InvalidCounter,
749                source,
750                format!("expected an unsigned byte count, got {value:?}"),
751            )
752        })
753    }
754
755    fn parse_stat_counter(
756        raw: &str,
757        key: &str,
758        source: &Path,
759    ) -> Result<u64, CgroupMemoryProbeFailure> {
760        for line in raw.lines() {
761            let mut fields = line.split_whitespace();
762            let Some(name) = fields.next() else {
763                continue;
764            };
765            if name != key {
766                continue;
767            }
768            let Some(value) = fields.next() else {
769                return Err(failure(
770                    CgroupMemoryProbeFailureKind::InvalidCounter,
771                    source,
772                    format!("memory.stat key {key:?} has no value"),
773                ));
774            };
775            if fields.next().is_some() {
776                return Err(failure(
777                    CgroupMemoryProbeFailureKind::InvalidCounter,
778                    source,
779                    format!("memory.stat key {key:?} has trailing fields"),
780                ));
781            }
782            return value.parse::<u64>().map_err(|_| {
783                failure(
784                    CgroupMemoryProbeFailureKind::InvalidCounter,
785                    source,
786                    format!("memory.stat key {key:?} is not an unsigned byte count"),
787                )
788            });
789        }
790        Err(failure(
791            CgroupMemoryProbeFailureKind::MissingCounter,
792            source,
793            format!("memory.stat is missing required key {key:?}"),
794        ))
795    }
796
797    fn read_required(path: &Path) -> Result<String, CgroupMemoryProbeFailure> {
798        fs::read_to_string(path).map_err(|error| io_failure(path, error))
799    }
800
801    fn read_optional(path: &Path) -> Result<Option<String>, CgroupMemoryProbeFailure> {
802        match fs::read_to_string(path) {
803            Ok(value) => Ok(Some(value)),
804            Err(error) if error.kind() == io::ErrorKind::NotFound => Ok(None),
805            Err(error) => Err(io_failure(path, error)),
806        }
807    }
808
809    fn io_failure(path: &Path, error: io::Error) -> CgroupMemoryProbeFailure {
810        failure(CgroupMemoryProbeFailureKind::Io, path, error.to_string())
811    }
812
813    fn failure(
814        kind: CgroupMemoryProbeFailureKind,
815        path: &Path,
816        detail: impl Into<Box<str>>,
817    ) -> CgroupMemoryProbeFailure {
818        CgroupMemoryProbeFailure::new(kind, path.display().to_string().into_boxed_str(), detail)
819    }
820
821    #[cfg(test)]
822    mod tests {
823        use super::*;
824        use std::fs;
825        use tempfile::TempDir;
826
827        struct Fixture {
828            _temp: TempDir,
829            mount: PathBuf,
830            cgroup_file: PathBuf,
831            mountinfo_file: PathBuf,
832        }
833
834        impl Fixture {
835            fn new(membership: &str) -> Self {
836                let temp = TempDir::new().expect("fixture tempdir");
837                let mount = temp.path().join("cgroup2");
838                fs::create_dir_all(&mount).expect("fixture mount");
839                let cgroup_file = temp.path().join("self.cgroup");
840                fs::write(&cgroup_file, format!("0::{membership}\n")).expect("fixture membership");
841                let mountinfo_file = temp.path().join("self.mountinfo");
842                fs::write(
843                    &mountinfo_file,
844                    format!(
845                        "29 23 0:26 / {} rw,nosuid,nodev,noexec,relatime - cgroup2 cgroup rw\n",
846                        mount.display()
847                    ),
848                )
849                .expect("fixture mountinfo");
850                Self {
851                    _temp: temp,
852                    mount,
853                    cgroup_file,
854                    mountinfo_file,
855                }
856            }
857
858            fn level(&self, relative: &str, limit: &str, current: u64, inactive: u64) {
859                let directory = self.mount.join(relative.trim_start_matches('/'));
860                fs::create_dir_all(&directory).expect("fixture level");
861                fs::write(directory.join("memory.max"), format!("{limit}\n"))
862                    .expect("fixture memory.max");
863                fs::write(directory.join("memory.current"), format!("{current}\n"))
864                    .expect("fixture memory.current");
865                fs::write(
866                    directory.join("memory.stat"),
867                    format!("anon 1\ninactive_file {inactive}\nactive_file 2\n"),
868                )
869                .expect("fixture memory.stat");
870            }
871
872            fn observe(&self) -> CgroupMemoryObservation {
873                detect_from_proc_files(&self.cgroup_file, &self.mountinfo_file)
874                    .unwrap_or_else(CgroupMemoryObservation::ProbeFailed)
875            }
876        }
877
878        #[test]
879        fn literal_max_is_typed_unbounded_even_when_current_is_large() {
880            let fixture = Fixture::new("/tenant/leaf");
881            fixture.level("tenant/leaf", "max", u64::MAX - 1, 0);
882            assert!(matches!(
883                fixture.observe(),
884                CgroupMemoryObservation::V2Unbounded {
885                    inspected_levels: 1,
886                    ..
887                }
888            ));
889        }
890
891        #[test]
892        fn cgroup_v2_root_accounting_without_memory_max_is_unbounded() {
893            let fixture = Fixture::new("/");
894            fs::write(fixture.mount.join("memory.current"), "1000\n").expect("root current");
895            fs::write(fixture.mount.join("memory.stat"), "inactive_file 700\n").expect("root stat");
896            assert!(matches!(
897                fixture.observe(),
898                CgroupMemoryObservation::V2Unbounded {
899                    inspected_levels: 0,
900                    ..
901                }
902            ));
903        }
904
905        #[test]
906        fn finite_zero_remains_authoritative() {
907            let fixture = Fixture::new("/tenant/leaf");
908            fixture.level("tenant/leaf", "0", 0, 0);
909            let CgroupMemoryObservation::V2Limited(observation) = fixture.observe() else {
910                panic!("finite zero must be a real cgroup ceiling");
911            };
912            assert_eq!(observation.limit_bytes(), 0);
913            assert_eq!(observation.available_bytes(), 0);
914        }
915
916        #[test]
917        fn malformed_active_controller_fails_closed() {
918            let fixture = Fixture::new("/tenant/leaf");
919            fixture.level("tenant/leaf", "1000", 500, 100);
920            fs::write(
921                fixture.mount.join("tenant/leaf/memory.current"),
922                "not-a-counter\n",
923            )
924            .expect("corrupt current");
925            let CgroupMemoryObservation::ProbeFailed(failure) = fixture.observe() else {
926                panic!("malformed active controller must fail closed");
927            };
928            assert_eq!(failure.kind(), CgroupMemoryProbeFailureKind::InvalidCounter);
929        }
930
931        #[test]
932        fn inconsistent_cache_counter_fails_closed() {
933            let fixture = Fixture::new("/tenant/leaf");
934            fixture.level("tenant/leaf", "1000", 100, 101);
935            let CgroupMemoryObservation::ProbeFailed(failure) = fixture.observe() else {
936                panic!("inconsistent controller counters must fail closed");
937            };
938            assert_eq!(
939                failure.kind(),
940                CgroupMemoryProbeFailureKind::InconsistentCounters
941            );
942        }
943
944        #[test]
945        fn hybrid_hierarchy_uses_the_active_v1_memory_controller() {
946            let fixture = Fixture::new("/tenant/leaf");
947            let v1_mount = fixture.mount.parent().unwrap().join("cgroup-memory");
948            let v1_leaf = v1_mount.join("legacy");
949            fs::create_dir_all(&v1_leaf).expect("v1 leaf");
950            fs::write(v1_leaf.join("memory.use_hierarchy"), "0\n").expect("hierarchy");
951            fs::write(v1_leaf.join("memory.limit_in_bytes"), "2048\n").expect("limit");
952            fs::write(v1_leaf.join("memory.usage_in_bytes"), "1024\n").expect("usage");
953            fs::write(
954                v1_leaf.join("memory.stat"),
955                "inactive_file 256\ntotal_inactive_file 256\n",
956            )
957            .expect("stat");
958            fs::write(&fixture.cgroup_file, "0::/tenant/leaf\n4:memory:/legacy\n")
959                .expect("hybrid membership");
960            let original = fs::read_to_string(&fixture.mountinfo_file).expect("mountinfo");
961            fs::write(
962                &fixture.mountinfo_file,
963                format!(
964                    "{original}31 23 0:30 / {} rw - cgroup cgroup rw,memory\n",
965                    v1_mount.display()
966                ),
967            )
968            .expect("hybrid mountinfo");
969            let CgroupMemoryObservation::V1Limited(observation) = fixture.observe() else {
970                panic!("hybrid memory accounting must follow v1");
971            };
972            assert_eq!(observation.available_bytes(), 1280);
973        }
974    }
975}
976
977#[cfg(target_os = "linux")]
978pub(crate) fn detect_cgroup_memory() -> CgroupMemoryObservation {
979    linux::detect()
980}
981
982#[cfg(all(test, not(target_os = "linux")))]
983mod non_linux_tests {
984    use super::*;
985
986    #[test]
987    fn non_linux_platform_has_no_cgroup_controller() {
988        assert_eq!(detect_cgroup_memory(), CgroupMemoryObservation::NotPresent);
989    }
990}
gam_runtime/cgroup_memory.rs

gam_runtime/
cgroup_memory.rs
