1use crate::KernelError;
15use std::collections::HashMap;
16use std::ffi::{OsStr, OsString};
17use std::path::{Path, PathBuf};
18use std::process::{Command, Stdio};
19use std::sync::{LazyLock, Mutex, PoisonError};
20use std::time::Duration;
21
22const PROBE_BYTES: u64 = 64 * 1024 * 1024;
24const PROBE_TIMEOUT: Duration = Duration::from_secs(5);
25
26#[derive(Debug, Clone, Copy, PartialEq, Eq)]
27pub enum TreeMemoryBackend {
28 Cgroup,
29 Watchdog,
30 Off,
31}
32
33impl TreeMemoryBackend {
34 pub fn name(self) -> &'static str {
35 match self {
36 TreeMemoryBackend::Cgroup => "cgroup",
37 TreeMemoryBackend::Watchdog => "watchdog",
38 TreeMemoryBackend::Off => "off",
39 }
40 }
41}
42
43pub fn kernel_tree_memory_limit(
46 env_value: Option<&str>,
47 per_process_mb: u64,
48 override_mb: Option<u64>,
49) -> Result<u64, KernelError> {
50 let error = || KernelError::new("RLM maxTreeMemoryMb must be a non-negative integer");
51 let fallback = per_process_mb.checked_mul(2).ok_or_else(error)?;
52 let value = match override_mb {
53 Some(value) => value,
54 None => match env_value.map(jsrt::string::trim) {
55 Some(text) if !text.is_empty() && text.bytes().all(|byte| byte.is_ascii_digit()) => {
56 text.parse::<u64>().unwrap_or(u64::MAX)
57 }
58 _ => fallback,
59 },
60 };
61 if value > jsrt::number::MAX_SAFE_INTEGER as u64 {
62 return Err(error());
63 }
64 Ok(value)
65}
66
67pub fn tree_memory_backend(
71 cap_mb: u64,
72 requested: Option<&str>,
73 cgroup_available: impl FnOnce() -> bool,
74) -> TreeMemoryBackend {
75 if cap_mb == 0 || !cfg!(target_os = "linux") {
76 return TreeMemoryBackend::Off;
77 }
78 let choice = requested
79 .map(|text| jsrt::string::trim(text).to_lowercase())
80 .unwrap_or_default();
81 match choice.as_str() {
82 "off" => TreeMemoryBackend::Off,
83 "watchdog" => TreeMemoryBackend::Watchdog,
84 _ if cgroup_available() => TreeMemoryBackend::Cgroup,
85 _ => TreeMemoryBackend::Watchdog,
86 }
87}
88
89static PROBES: LazyLock<Mutex<HashMap<OsString, bool>>> = LazyLock::new(Mutex::default);
90
91pub fn cgroup_scope_available(program: &OsStr) -> bool {
95 if !cfg!(target_os = "linux") {
96 return false;
97 }
98 let mut probes = PROBES.lock().unwrap_or_else(PoisonError::into_inner);
99 if let Some(known) = probes.get(program) {
100 return *known;
101 }
102 let available = probe(program);
103 probes.insert(program.to_os_string(), available);
104 available
105}
106
107fn probe(program: &OsStr) -> bool {
108 let mut args: Vec<String> = cgroup_scope_command(PROBE_BYTES / (1024 * 1024))[1..].to_vec();
109 args.extend([
110 "/bin/sh".into(),
111 "-c".into(),
112 r#"cat "/sys/fs/cgroup$(cut -d: -f3 /proc/self/cgroup)/memory.max""#.into(),
113 ]);
114 let bound = signals::child::Bound {
115 timeout: Some(PROBE_TIMEOUT),
116 ..Default::default()
117 };
118 let mut command = Command::new(program);
119 command.args(&args).stdout(Stdio::piped()).stderr(Stdio::null());
120 signals::child::run(&mut command, None, bound).is_ok_and(|out| {
121 out.success() && jsrt::string::trim(&String::from_utf8_lossy(&out.stdout)) == PROBE_BYTES.to_string()
122 })
123}
124
125pub fn cgroup_scope_command(cap_mb: u64) -> Vec<String> {
128 [
129 "systemd-run",
130 "--user",
131 "--scope",
132 "--quiet",
133 "--collect",
134 "-p",
135 &format!("MemoryMax={cap_mb}M"),
136 "-p",
137 "MemorySwapMax=0",
138 "-p",
141 "OOMPolicy=continue",
142 "--",
143 ]
144 .map(String::from)
145 .to_vec()
146}
147
148pub fn tree_memory_limit_message(cap_mb: u64, backend: TreeMemoryBackend) -> String {
150 let how = if backend == TreeMemoryBackend::Cgroup {
151 "cgroup MemoryMax"
152 } else {
153 "host watchdog"
154 };
155 format!(
156 "RLM kernel process tree exceeded its memory limit ({cap_mb} MiB total across the kernel and every process its cells started, ULTRON_RLM_MAX_TREE_MEMORY_MB, enforced by {how}) and was stopped; the next cell starts a fresh kernel without earlier Python state"
157 )
158}
159
160pub fn process_cgroup_dir(pid: i32) -> Option<PathBuf> {
162 let text = std::fs::read_to_string(format!("/proc/{pid}/cgroup")).ok()?;
163 cgroup_dir_of(&text)
164}
165
166pub fn is_foreign_scope(dir: &Path) -> bool {
171 let Some(own) = process_cgroup_dir(std::process::id() as i32) else {
172 return false;
173 };
174 dir.to_string_lossy().ends_with(".scope") && !own.starts_with(dir)
175}
176
177pub fn cgroup_dir_of(text: &str) -> Option<PathBuf> {
179 let line = text.split('\n').find(|line| line.starts_with("0::"))?;
180 Some(PathBuf::from(format!("/sys/fs/cgroup{}", &line[3..])))
181}
182
183pub fn cgroup_oom_kills(dir: &Path) -> Option<u64> {
186 let text = std::fs::read_to_string(dir.join("memory.events")).ok()?;
187 Some(
188 text.lines()
189 .find_map(|line| line.strip_prefix("oom_kill "))
190 .and_then(|count| {
191 count
192 .bytes()
193 .all(|byte| byte.is_ascii_digit())
194 .then(|| count.parse().ok())
195 .flatten()
196 })
197 .unwrap_or(0),
198 )
199}
200
201pub fn cgroup_pids(dir: &Path) -> Vec<i32> {
203 let mut pids = Vec::new();
204 collect_cgroup_pids(dir, &mut pids);
205 pids
206}
207
208fn collect_cgroup_pids(dir: &Path, pids: &mut Vec<i32>) {
209 let Ok(text) = std::fs::read_to_string(dir.join("cgroup.procs")) else {
210 return;
211 };
212 pids.extend(
213 text.split('\n')
214 .filter(|line| !line.is_empty() && line.bytes().all(|byte| byte.is_ascii_digit()))
215 .filter_map(|line| line.parse::<i32>().ok()),
216 );
217 let Ok(entries) = std::fs::read_dir(dir) else {
218 return;
219 };
220 for entry in entries.flatten() {
221 if entry.file_type().is_ok_and(|kind| kind.is_dir()) {
222 collect_cgroup_pids(&entry.path(), pids);
223 }
224 }
225}
226
227fn sum_kib(text: &str, fields: &[&str]) -> u64 {
229 let kib: u64 = text
230 .lines()
231 .filter_map(|line| {
232 let (name, rest) = line.split_once(':')?;
233 if !fields.contains(&name) {
234 return None;
235 }
236 let value = rest
237 .strip_prefix([' ', '\t'])?
238 .trim_start_matches([' ', '\t'])
239 .strip_suffix(" kB")?;
240 (!value.is_empty() && value.bytes().all(|byte| byte.is_ascii_digit()))
241 .then(|| value.parse::<u64>().ok())
242 .flatten()
243 })
244 .sum();
245 kib * 1024
246}
247
248pub fn private_rss_bytes(status: &str) -> u64 {
250 sum_kib(status, &["RssAnon", "RssShmem", "VmSwap"])
251}
252
253pub fn proportional_bytes(rollup: &str) -> Option<u64> {
256 if !rollup.lines().any(|line| line.starts_with("Pss_Anon:")) {
257 return None;
258 }
259 Some(sum_kib(rollup, &["Pss_Anon", "Pss_Shmem", "SwapPss"]))
260}
261
262pub fn process_memory_bytes(pid: i32) -> u64 {
264 let status = || {
265 std::fs::read_to_string(format!("/proc/{pid}/status"))
266 .map(|text| private_rss_bytes(&text))
267 .unwrap_or(0)
268 };
269 match std::fs::read_to_string(format!("/proc/{pid}/smaps_rollup")) {
270 Err(_) => status(),
271 Ok(rollup) => match proportional_bytes(&rollup) {
272 Some(bytes) => bytes,
273 None if rollup.trim().is_empty() => 0,
274 None => status(),
275 },
276 }
277}
278
279pub fn tree_memory_usage(root: i32) -> (u64, Vec<i32>) {
281 let descendants = crate::process::descendant_pids(root);
282 let bytes = process_memory_bytes(root) + descendants.iter().map(|pid| process_memory_bytes(*pid)).sum::<u64>();
283 (bytes, descendants)
284}
285
286pub fn watchdog_poll_ms(bytes: u64, cap_bytes: u64) -> u64 {
288 let (bytes, cap) = (bytes as f64, cap_bytes as f64);
289 if bytes >= cap * 0.7 {
290 100
291 } else if bytes >= cap * 0.4 {
292 250
293 } else {
294 500
295 }
296}
297
298#[cfg(test)]
299mod tests {
300 use super::*;
301
302 #[test]
307 fn kernel_tree_memory_never_treats_the_hosts_cgroup_as_the_kernels() {
308 let Some(own) = process_cgroup_dir(std::process::id() as i32) else {
309 return;
310 };
311 assert!(!is_foreign_scope(&own));
312 for ancestor in own.ancestors() {
313 assert!(!is_foreign_scope(ancestor), "{}", ancestor.display());
314 }
315 assert!(is_foreign_scope(&own.with_file_name("run-u-kernel.scope")));
316 assert!(!is_foreign_scope(&own.with_file_name("run-u-kernel.slice")));
317 }
318
319 #[test]
320 fn kernel_tree_memory_text_parsers() {
321 let rollup = "Rss: 9000 kB\nPss_Anon: 100 kB\nPss_File: 7 kB\nPss_Shmem: 20 kB\nSwapPss: 3 kB\n";
322 assert_eq!(proportional_bytes(rollup), Some(123 * 1024));
323 assert_eq!(proportional_bytes("Rss: 10 kB\n"), None);
324 let status = "Name:\tpython3\nRssAnon:\t 40 kB\nRssFile:\t 99 kB\nRssShmem:\t 2 kB\nVmSwap:\t 1 kB\n";
325 assert_eq!(private_rss_bytes(status), 43 * 1024);
326 assert_eq!(
327 cgroup_dir_of("1:name=x:/a\n0::/user.slice/run-u1.scope\n"),
328 Some(PathBuf::from("/sys/fs/cgroup/user.slice/run-u1.scope"))
329 );
330 assert_eq!(cgroup_dir_of("1:cpu:/a\n"), None);
331 }
332}