Skip to main content

rlmctl_core/guard/
policy.rs

1//! Pure policy state machine: the self-healing circuit breaker at the heart of
2//! the freeze guard.
3//!
4//! Contract: [`PolicyEngine::tick`] is pure given `(now_ms, sample, targets,
5//! live_cgroups)` plus the engine's own internal state. It performs **no**
6//! syscalls and reads **no** clock; `now_ms` (monotonic milliseconds) is
7//! injected by the caller. That is what makes the whole escalation/recovery
8//! ladder unit-testable without root.
9
10use std::collections::{BTreeMap, HashMap, HashSet};
11
12use super::resolve::{Coverage, Resolution, Verdict};
13use super::types::{Action, Intervention, Level, Sample, Target};
14use common::GuardConfig;
15
16/// PSI `full` avg10 (%) that, on its own, forces at least the High level. Mirrors
17/// the design doc's "or `full.avg10 >= 3`" High trigger.
18const FULL_HIGH_RISE: f64 = 3.0;
19/// Rate-limit window for `Notify` actions (ms): at most one notification a minute.
20const NOTIFY_INTERVAL_MS: u64 = 60_000;
21
22/// Escalation gate (ms) after an action that only partly covered its app.
23/// PSI avg10 is a 10 s average, so re-measuring after 1 s still sees the old
24/// pressure; waiting at least this long stops a partial action from
25/// cascading into several more within seconds.
26pub const PARTIAL_GATE_MS: u64 = 3_000;
27/// Most apps the guard holds (frozen or capped) at the same time.
28pub const MAX_HELD_APPS: usize = 3;
29/// Growth rate (bytes/s of `memory.current`) an app must reach to be picked
30/// as the one causing pressure. Below it, the largest app is picked instead.
31pub const MIN_GROWTH_BPS: f64 = 1_048_576.0;
32/// Most consecutive ticks victim selection waits for growth data when every
33/// eligible cgroup is newly seen. After that the largest app is picked, so a
34/// stream of short-lived cgroups cannot keep the guard from acting.
35pub const MAX_COLD_DEFER_TICKS: u32 = 3;
36
37/// True when the host is actually short of memory: below the hard floor, or
38/// below `act_below_available_pct` percent of RAM. A stall confined to one
39/// cgroup's memory.max leaves MemAvailable high, so it never passes this gate.
40pub fn is_scarce(s: &Sample, t: &common::GuardTrigger) -> bool {
41    s.mem_available_mb < t.mem_available_floor_mb
42        || (s.mem_total_mb > 0
43            && s.mem_available_mb.saturating_mul(100)
44                < s.mem_total_mb.saturating_mul(t.act_below_available_pct))
45}
46
47/// Smoothed `memory.current` growth for one cgroup.
48struct Growth {
49    last_bytes: u64,
50    last_ms: u64,
51    /// EWMA of bytes per second (negative while shrinking).
52    rate_bps: f64,
53    /// True once a second sample has been seen, so `rate_bps` is a measured
54    /// rate and not the zero placeholder of a first sighting.
55    warm: bool,
56}
57
58/// Self-healing circuit-breaker policy engine.
59///
60/// On a memory spike it drives the ladder *notify, freeze (short), auto-thaw,
61/// if still high a soft cap, once calm is sustained a lift*, never issuing a kill. All
62/// of that lives in [`tick`](Self::tick); the struct just holds the state needed
63/// to make decisions stable across ticks (hysteresis, cooldowns, growth).
64///
65/// The unit of choice is an app: every cgroup of the chosen app is frozen or
66/// capped together as one escalation step.
67pub struct PolicyEngine {
68    cfg: GuardConfig,
69    /// Current pressure level (carried across ticks so hysteresis works).
70    level: Level,
71    /// Active interventions keyed by resolved cgroup path, with the
72    /// [`Resolution`] (so `Thaw`/`LiftCap` can be built without re-resolving)
73    /// and the app the cgroup was acted on as.
74    interventions: HashMap<String, (Intervention, Resolution, String)>,
75    /// Last time each app was frozen. Drives the per-app freeze cooldown that
76    /// decides freeze-vs-cap, and is intentionally kept after a thaw.
77    last_freeze_ms: HashMap<String, u64>,
78    /// Per-cgroup `memory.current` growth estimate.
79    growth: HashMap<String, Growth>,
80    /// When the level last became `Calm` (None while not calm). Gates cap lifts.
81    calm_since_ms: Option<u64>,
82    /// When we last emitted a new freeze/cap: the global escalation gate.
83    /// `None` means "never acted", so the gate is open on the first action.
84    last_action_ms: Option<u64>,
85    /// Whether the most recent escalation acted under `Coverage::Partial`.
86    /// When true the gate is shortened to [`PARTIAL_GATE_MS`] (capped at the
87    /// freeze hold) so the guard can re-assess sooner, but never instantly.
88    last_action_partial: bool,
89    /// When we last emitted a `Notify`; drives notification rate-limiting.
90    /// `None` means "never notified", so the first eligible notify fires.
91    last_notify_ms: Option<u64>,
92    /// Consecutive ticks on which selection deferred for lack of growth
93    /// data. Bounded by [`MAX_COLD_DEFER_TICKS`].
94    cold_defer_ticks: u32,
95}
96
97impl PolicyEngine {
98    pub fn new(cfg: GuardConfig) -> Self {
99        Self {
100            cfg,
101            level: Level::Calm,
102            interventions: HashMap::new(),
103            last_freeze_ms: HashMap::new(),
104            growth: HashMap::new(),
105            calm_since_ms: None,
106            last_action_ms: None,
107            last_action_partial: false,
108            last_notify_ms: None,
109            cold_defer_ticks: 0,
110        }
111    }
112
113    /// Advance the state machine one tick and return the actions to apply.
114    ///
115    /// `targets` are the cgroups eligible for action this tick (already
116    /// filtered for uid, protect list and min RSS by the Sampler).
117    ///
118    /// `live_cgroups` is the subset of the engine's intervened cgroups that
119    /// still hold a process (see `sampler::live_cgroups`), with no min-RSS
120    /// or protect filtering applied; it is deliberately independent of
121    /// `targets`. Pruning checks liveness against
122    /// this set, not against `targets`: `memory.high` also bounds
123    /// file-backed pages, so capping
124    /// a mapped-file-heavy process can push its `rss_kb` below the min-RSS
125    /// floor on the very next tick, dropping it out of `targets` even though
126    /// the cgroup is very much still alive. Pruning against `targets` there
127    /// would lift the cap while pressure is still Critical and immediately
128    /// re-trigger it. Victim *selection* deliberately keeps using `targets`.
129    pub fn tick(
130        &mut self,
131        now_ms: u64,
132        sample: Sample,
133        targets: &[Target],
134        live_cgroups: &HashSet<String>,
135    ) -> Vec<Action> {
136        // 1. Disabled guard is inert.
137        if !self.cfg.enabled {
138            return Vec::new();
139        }
140
141        let mut actions = Vec::new();
142
143        // 2. Recompute the level with hysteresis and track how long we've been calm.
144        self.level = self.next_level(sample);
145        match self.level {
146            Level::Calm => {
147                // Start the calm clock on the *transition* into calm, then leave it.
148                if self.calm_since_ms.is_none() {
149                    self.calm_since_ms = Some(now_ms);
150                }
151                self.cold_defer_ticks = 0;
152            }
153            _ => self.calm_since_ms = None,
154        }
155
156        // 3. Track memory.current growth per cgroup.
157        self.update_growth(now_ms, targets);
158
159        // 4. Prune interventions whose cgroup is no longer live (see
160        //    `live_cgroups` doc above). LiftCap doubles as "tear down the
161        //    cap", so it's the right cleanup for both frozen and capped dead
162        //    cgroups; the effector's LiftCap tolerates a missing cgroup.
163        let dead: Vec<String> = self
164            .interventions
165            .keys()
166            .filter(|cg| !live_cgroups.contains(cg.as_str()))
167            .cloned()
168            .collect();
169        for cg in dead {
170            let (_, res, _) = self.interventions.remove(&cg).expect("just found key");
171            actions.push(Action::LiftCap { res });
172        }
173
174        // 5. Recover: auto-thaw held freezes, and lift caps once calm has held.
175        let freeze_hold_ms = self.cfg.timing.freeze_hold_secs.saturating_mul(1000);
176        let calm_hold_ms = self.cfg.timing.calm_hold_secs.saturating_mul(1000);
177        let mut recovered = Vec::new();
178        // Apps thawed on this tick must not be re-targeted by escalation in
179        // the same tick; they need a re-measure first.
180        let mut thawed_apps: HashSet<String> = HashSet::new();
181        for (cg, (intervention, res, app)) in &self.interventions {
182            match *intervention {
183                Intervention::Frozen { since_ms } => {
184                    if now_ms.saturating_sub(since_ms) >= freeze_hold_ms {
185                        actions.push(Action::Thaw { res: res.clone() });
186                        recovered.push(cg.clone());
187                        thawed_apps.insert(app.clone());
188                    }
189                }
190                Intervention::Capped { .. } => {
191                    // Only lift a cap when pressure is calm *and* has stayed calm
192                    // long enough; this prevents re-capping churn. `calm_since_ms` is
193                    // the transition timestamp, so this measures *sustained* calm.
194                    if self.level == Level::Calm {
195                        if let Some(calm_since) = self.calm_since_ms {
196                            if now_ms.saturating_sub(calm_since) >= calm_hold_ms {
197                                actions.push(Action::LiftCap { res: res.clone() });
198                                recovered.push(cg.clone());
199                            }
200                        }
201                    }
202                }
203            }
204        }
205        for cg in recovered {
206            // Note: last_freeze_ms is intentionally retained for cooldown logic.
207            self.interventions.remove(&cg);
208        }
209
210        // 6. Escalate, but only when apps feel pressure, memory is actually
211        //    short (see `is_scarce`), the gate is open, and fewer than
212        //    MAX_HELD_APPS apps are already held.
213        let mut victim_name: Option<String> = None;
214        if matches!(self.level, Level::High | Level::Critical)
215            && is_scarce(&sample, &self.cfg.trigger)
216            && self.held_apps().len() < MAX_HELD_APPS
217        {
218            // Global gate: after acting on one app, wait a freeze-hold before
219            // acting again so we re-measure instead of cascading. After a
220            // partial action wait a shorter, but never zero, interval.
221            let gate_ms = if self.last_action_partial {
222                PARTIAL_GATE_MS.min(freeze_hold_ms)
223            } else {
224                freeze_hold_ms
225            };
226            let gate_open = match self.last_action_ms {
227                None => true,
228                Some(last) => now_ms.saturating_sub(last) >= gate_ms,
229            };
230            if gate_open {
231                if let Some((app, members)) = self.select_app(targets, &thawed_apps) {
232                    let cap_only = members
233                        .iter()
234                        .any(|m| m.resolution.verdict == Verdict::CapOnly);
235                    let partial = members
236                        .iter()
237                        .any(|m| m.resolution.coverage == Coverage::Partial);
238                    let cooldown_ms = self.cfg.timing.freeze_cooldown_secs.saturating_mul(1000);
239                    let in_cooldown = self
240                        .last_freeze_ms
241                        .get(&app)
242                        .is_some_and(|&last| now_ms.saturating_sub(last) < cooldown_ms);
243                    // CapOnly never freezes; a recently frozen app that is
244                    // still hot escalates to a soft cap.
245                    let freeze = !cap_only && !in_cooldown;
246
247                    for m in members {
248                        let res = m.resolution.clone();
249                        let intervention = if freeze {
250                            actions.push(Action::Freeze {
251                                res: res.clone(),
252                                name: app.clone(),
253                            });
254                            Intervention::Frozen { since_ms: now_ms }
255                        } else {
256                            actions.push(Action::Cap {
257                                res: res.clone(),
258                                name: app.clone(),
259                            });
260                            Intervention::Capped { since_ms: now_ms }
261                        };
262                        self.interventions
263                            .insert(res.cgroup.clone(), (intervention, res, app.clone()));
264                    }
265                    if freeze {
266                        self.last_freeze_ms.insert(app.clone(), now_ms);
267                    }
268                    self.last_action_ms = Some(now_ms);
269                    self.last_action_partial = partial;
270                    victim_name = Some(app);
271                }
272            }
273        }
274
275        // 7. Notify (rate-limited) while there's anything to report.
276        let notify_due = match self.last_notify_ms {
277            None => true,
278            Some(last) => now_ms.saturating_sub(last) >= NOTIFY_INTERVAL_MS,
279        };
280        if self.cfg.notify
281            && matches!(self.level, Level::Warn | Level::High | Level::Critical)
282            && notify_due
283        {
284            let message = match &victim_name {
285                Some(name) => format!(
286                    "rlm-guard: memory pressure {:?}, acting on {}",
287                    self.level, name
288                ),
289                None => format!("rlm-guard: memory pressure {:?}", self.level),
290            };
291            actions.push(Action::Notify { message });
292            self.last_notify_ms = Some(now_ms);
293        }
294
295        actions
296    }
297
298    /// True when the caller should gather targets for the next tick: the
299    /// level would be above Calm, or memory is already scarce. Scanning while
300    /// scarce keeps growth rates warm, so a sudden drop below the floor can
301    /// pick the app that is growing instead of the largest one. A disabled
302    /// engine never wants them.
303    pub fn wants_candidates(&self, sample: Sample) -> bool {
304        self.cfg.enabled
305            && (self.next_level(sample) != Level::Calm || is_scarce(&sample, &self.cfg.trigger))
306    }
307
308    /// The pressure level as of the last tick.
309    pub fn level(&self) -> Level {
310        self.level
311    }
312
313    /// Currently active interventions (cgroup path and intervention), sorted by
314    /// cgroup path so callers get a deterministic order.
315    pub fn interventions(&self) -> Vec<(String, Intervention)> {
316        let mut out: Vec<(String, Intervention)> = self
317            .interventions
318            .iter()
319            .map(|(cg, (iv, _, _))| (cg.clone(), *iv))
320            .collect();
321        out.sort_by(|(a, _), (b, _)| a.cmp(b));
322        out
323    }
324
325    /// Cgroup paths the engine currently holds an intervention on, frozen
326    /// *or* capped. Interventions are already keyed by cgroup, so this is
327    /// just the key set. For external callers (e.g.
328    /// `RulesEnforcer::reconcile`, D1) that must not fight/revert an active
329    /// guard action: rewriting `memory.high` on a cgroup the engine just
330    /// capped would silently no-op the cap and leave `PolicyEngine` holding
331    /// a stale `Capped` intervention that then blocks victim re-selection
332    /// for the rest of the pressure episode.
333    pub fn intervened_cgroups(&self) -> Vec<String> {
334        self.interventions.keys().cloned().collect()
335    }
336
337    /// Distinct apps with at least one active intervention.
338    fn held_apps(&self) -> HashSet<&str> {
339        self.interventions
340            .values()
341            .map(|(_, _, app)| app.as_str())
342            .collect()
343    }
344
345    /// Compute the next level from the current level + a fresh sample, applying
346    /// rise/fall hysteresis. The fall threshold is half the rise threshold, and
347    /// we only ever *step down* when below the fall threshold, so a sample that
348    /// sits between fall and rise leaves the level unchanged (no flapping).
349    fn next_level(&self, s: Sample) -> Level {
350        let t = &self.cfg.trigger;
351        let floor = t.mem_available_floor_mb;
352
353        // Rise predicates (cross the upper threshold to enter a level).
354        let warn_rise = s.some_avg10 >= t.psi_some_warn;
355        let high_rise = s.some_avg10 >= t.psi_some_high || s.full_avg10 >= FULL_HIGH_RISE;
356        let crit_rise = s.full_avg10 >= t.psi_full_critical || s.mem_available_mb < floor;
357
358        // Stay predicates (above the lower/fall threshold: keep the level).
359        let warn_stay = s.some_avg10 >= t.psi_some_warn / 2.0;
360        let high_stay =
361            s.some_avg10 >= t.psi_some_high / 2.0 || s.full_avg10 >= FULL_HIGH_RISE / 2.0;
362        let crit_stay = s.full_avg10 >= t.psi_full_critical / 2.0 || s.mem_available_mb < floor;
363
364        // Highest level we're allowed to be at, given current level + hysteresis.
365        // For each tier: enter if its rise fires; otherwise remain if we're
366        // already at/above it and its stay predicate still holds.
367        let at_critical = self.level == Level::Critical;
368        let at_high = matches!(self.level, Level::High | Level::Critical);
369        let at_warn = matches!(self.level, Level::Warn | Level::High | Level::Critical);
370
371        if crit_rise || (at_critical && crit_stay) {
372            Level::Critical
373        } else if high_rise || (at_high && high_stay) {
374            Level::High
375        } else if warn_rise || (at_warn && warn_stay) {
376            Level::Warn
377        } else {
378            Level::Calm
379        }
380    }
381
382    /// Update each target cgroup's `memory.current` growth rate (an EWMA
383    /// with weight 0.5 per tick) and forget cgroups no longer present.
384    fn update_growth(&mut self, now_ms: u64, targets: &[Target]) {
385        let mut seen = HashSet::new();
386        for t in targets {
387            let Some(cur) = t.current_bytes else {
388                continue;
389            };
390            let cg = &t.resolution.cgroup;
391            seen.insert(cg.clone());
392            match self.growth.get_mut(cg) {
393                Some(g) if now_ms > g.last_ms => {
394                    let dt = (now_ms - g.last_ms) as f64 / 1000.0;
395                    let inst = (cur as f64 - g.last_bytes as f64) / dt;
396                    g.rate_bps = 0.5 * g.rate_bps + 0.5 * inst;
397                    g.last_bytes = cur;
398                    g.last_ms = now_ms;
399                    g.warm = true;
400                }
401                Some(_) => {}
402                None => {
403                    self.growth.insert(
404                        cg.clone(),
405                        Growth {
406                            last_bytes: cur,
407                            last_ms: now_ms,
408                            rate_bps: 0.0,
409                            warm: false,
410                        },
411                    );
412                }
413            }
414        }
415        self.growth.retain(|cg, _| seen.contains(cg));
416    }
417
418    /// Pick the app to act on and its member targets. Eligible apps are not
419    /// held, not thawed this tick (`blocked`), have a member at or above the
420    /// min-RSS floor, and have no member cgroup under an intervention. The
421    /// app whose cgroups grow fastest wins when that growth is at least
422    /// [`MIN_GROWTH_BPS`]; otherwise the largest app. Ties go to the
423    /// lexicographically smaller app name, so the choice is deterministic.
424    ///
425    /// Cold start: when no eligible cgroup has a measured growth rate yet but
426    /// at least one was first seen this tick, nothing is picked. The next
427    /// tick (one sample interval later) has rates, so an idle large app is
428    /// not frozen in place of a smaller one that is growing. The deferral
429    /// lasts one tick at most per new cgroup, and at most
430    /// [`MAX_COLD_DEFER_TICKS`] ticks in a row; after that the largest app is
431    /// picked. If no eligible cgroup reports `memory.current` at all, growth
432    /// can never be measured and the largest app is picked at once.
433    fn select_app<'a>(
434        &mut self,
435        targets: &'a [Target],
436        blocked: &HashSet<String>,
437    ) -> Option<(String, Vec<&'a Target>)> {
438        let min_rss_kb = self.cfg.selection.min_rss_mb.saturating_mul(1024);
439        let held = self.held_apps();
440        let mut groups: BTreeMap<&str, Vec<&Target>> = BTreeMap::new();
441        for t in targets {
442            groups.entry(t.app.as_str()).or_default().push(t);
443        }
444        let eligible: Vec<(&str, Vec<&Target>)> = groups
445            .into_iter()
446            .filter(|(app, ms)| {
447                !held.contains(app)
448                    && !blocked.contains(*app)
449                    && ms.iter().any(|m| m.rss_kb >= min_rss_kb)
450                    && ms
451                        .iter()
452                        .all(|m| !self.interventions.contains_key(&m.resolution.cgroup))
453            })
454            .collect();
455        let any_warm = eligible.iter().any(|(_, ms)| {
456            ms.iter().any(|m| {
457                self.growth
458                    .get(&m.resolution.cgroup)
459                    .is_some_and(|g| g.warm)
460            })
461        });
462        let any_cold = eligible.iter().any(|(_, ms)| {
463            ms.iter().any(|m| {
464                self.growth
465                    .get(&m.resolution.cgroup)
466                    .is_some_and(|g| !g.warm)
467            })
468        });
469        if !any_warm && any_cold && self.cold_defer_ticks < MAX_COLD_DEFER_TICKS {
470            self.cold_defer_ticks += 1;
471            return None;
472        }
473        self.cold_defer_ticks = 0;
474        let growth = |ms: &[&Target]| -> f64 {
475            ms.iter()
476                .map(|m| {
477                    self.growth
478                        .get(&m.resolution.cgroup)
479                        .map_or(0.0, |g| g.rate_bps.max(0.0))
480                })
481                .sum()
482        };
483        let size = |ms: &[&Target]| -> u64 {
484            ms.iter()
485                .map(|m| m.current_bytes.unwrap_or(m.rss_kb.saturating_mul(1024)))
486                .sum()
487        };
488        let pick = eligible
489            .iter()
490            .filter(|(_, ms)| growth(ms) >= MIN_GROWTH_BPS)
491            .max_by(|a, b| {
492                growth(&a.1)
493                    .total_cmp(&growth(&b.1))
494                    .then_with(|| b.0.cmp(a.0))
495            })
496            .or_else(|| {
497                eligible
498                    .iter()
499                    .max_by(|a, b| size(&a.1).cmp(&size(&b.1)).then_with(|| b.0.cmp(a.0)))
500            })?;
501        Some((pick.0.to_string(), pick.1.clone()))
502    }
503}
504
505#[cfg(test)]
506mod tests {
507    use super::super::resolve::Mechanism;
508    use super::super::types::PsiSource;
509    use super::*;
510
511    /// Default config = the documented zero-config defaults.
512    fn cfg() -> GuardConfig {
513        GuardConfig::default()
514    }
515
516    fn sample(some: f64, full: f64, avail_mb: u64) -> Sample {
517        Sample {
518            some_avg10: some,
519            full_avg10: full,
520            mem_available_mb: avail_mb,
521            mem_total_mb: 16_000,
522            source: PsiSource::AppSlice,
523        }
524    }
525
526    /// Build a default (unprotected, full-coverage) resolution for `cg`.
527    fn res(cg: &str) -> Resolution {
528        Resolution {
529            cgroup: cg.into(),
530            unit: Some(format!("{}.scope", cg.rsplit('/').next().unwrap())),
531            verdict: Verdict::Freeze,
532            coverage: Coverage::Full,
533            mechanism: Mechanism::Unit,
534        }
535    }
536
537    const MIB: u64 = 1024 * 1024;
538
539    /// Build a `Target` for app `app` in cgroup `cg`, with `mb` MB resident
540    /// and the same amount charged to the cgroup.
541    fn target(app: &str, cg: &str, mb: u64) -> Target {
542        Target {
543            app: app.into(),
544            resolution: res(cg),
545            rss_kb: mb * 1024,
546            current_bytes: Some(mb * MIB),
547        }
548    }
549
550    /// Shorthand: one app per scope, `/app.slice/app-<name>-<pid>.scope`.
551    fn proc(pid: u32, name: &str, rss_mb: u64) -> Target {
552        target(name, &format!("/app.slice/app-{name}-{pid}.scope"), rss_mb)
553    }
554
555    fn proc_at(_pid: u32, name: &str, rss_mb: u64, cg: &str) -> Target {
556        target(name, cg, rss_mb)
557    }
558
559    /// A comfortably-calm sample (no pressure, lots of memory).
560    fn calm() -> Sample {
561        sample(0.0, 0.0, 8000)
562    }
563
564    /// High PSI while only 12.5% of RAM is available: a real shortage.
565    fn high() -> Sample {
566        sample(50.0, 0.0, 2_000)
567    }
568
569    #[test]
570    fn high_pressure_with_plenty_of_free_memory_never_escalates() {
571        let mut e = PolicyEngine::new(cfg());
572        let procs = vec![proc(2, "chrome", 4000)];
573        // Half of RAM available: this stall is local to some memory.max, not a shortage.
574        let a = e.tick(0, sample(80.0, 20.0, 8_000), &procs, &live_from(&procs));
575        assert_eq!(e.level, Level::Critical);
576        assert!(
577            !a.iter()
578                .any(|x| matches!(x, Action::Freeze { .. } | Action::Cap { .. })),
579            "escalated without scarcity: {a:?}"
580        );
581    }
582
583    #[test]
584    fn is_scarce_uses_floor_or_percentage() {
585        let t = common::GuardTrigger::default(); // floor 400 MB, 20%
586        assert!(is_scarce(&sample(0.0, 0.0, 300), &t));
587        assert!(is_scarce(&sample(0.0, 0.0, 3_000), &t)); // 18.75%
588        assert!(!is_scarce(&sample(0.0, 0.0, 3_300), &t)); // 20.6%
589        let unknown = Sample {
590            mem_total_mb: 0,
591            mem_available_mb: u64::MAX,
592            ..sample(0.0, 0.0, 0)
593        };
594        assert!(
595            !is_scarce(&unknown, &t),
596            "unreadable meminfo must not enable actions"
597        );
598    }
599
600    fn freeze_targets(actions: &[Action]) -> Vec<String> {
601        actions
602            .iter()
603            .filter_map(|a| match a {
604                Action::Freeze { res, .. } => Some(res.cgroup.clone()),
605                _ => None,
606            })
607            .collect()
608    }
609
610    fn has_cap_target(actions: &[Action], cg: &str) -> bool {
611        actions
612            .iter()
613            .any(|a| matches!(a, Action::Cap { res, .. } if res.cgroup == cg))
614    }
615
616    fn has_freeze_target(actions: &[Action], cg: &str) -> bool {
617        actions
618            .iter()
619            .any(|a| matches!(a, Action::Freeze { res, .. } if res.cgroup == cg))
620    }
621
622    fn has_thaw_target(actions: &[Action], cg: &str) -> bool {
623        actions
624            .iter()
625            .any(|a| matches!(a, Action::Thaw { res } if res.cgroup == cg))
626    }
627
628    fn has_liftcap_target(actions: &[Action], cg: &str) -> bool {
629        actions
630            .iter()
631            .any(|a| matches!(a, Action::LiftCap { res } if res.cgroup == cg))
632    }
633
634    /// Give every target a measured (zero) growth rate, as if the engine had
635    /// already scanned them on an earlier tick. Tests of the ladder use this
636    /// so the cold-start deferral does not shift their first action.
637    fn prime(e: &mut PolicyEngine, ts: &[Target]) {
638        for t in ts {
639            e.growth.insert(
640                t.resolution.cgroup.clone(),
641                Growth {
642                    last_bytes: t.current_bytes.unwrap_or(0),
643                    last_ms: 0,
644                    rate_bps: 0.0,
645                    warm: true,
646                },
647            );
648        }
649    }
650
651    /// Default `live_cgroups` for tests that aren't specifically exercising
652    /// the liveness-vs-eligibility distinction: every target's cgroup.
653    fn live_from(ts: &[Target]) -> std::collections::HashSet<String> {
654        ts.iter().map(|t| t.resolution.cgroup.clone()).collect()
655    }
656
657    #[test]
658    fn calm_yields_no_actions() {
659        let mut e = PolicyEngine::new(cfg());
660        let procs = vec![proc(100, "firefox", 2000)];
661        let actions = e.tick(1000, calm(), &procs, &live_from(&procs));
662        assert!(actions.is_empty(), "calm produced actions: {actions:?}");
663        assert_eq!(e.level, Level::Calm);
664    }
665
666    #[test]
667    fn full_signal_has_fall_hysteresis() {
668        // Enter High purely via PSI `full` (some stays low): full=4.0 >= FULL_HIGH_RISE(3.0).
669        let mut e = PolicyEngine::new(cfg());
670        let procs = vec![proc(100, "firefox", 4000)];
671        e.tick(1_000, sample(0.0, 4.0, 8000), &procs, &live_from(&procs));
672        assert_eq!(e.level, Level::High, "full=4.0 should enter High");
673
674        // full drifts to 2.0, between the fall (1.5) and rise (3.0) thresholds.
675        // With hysteresis it must HOLD High, not flap back to Calm.
676        e.tick(2_000, sample(0.0, 2.0, 8000), &procs, &live_from(&procs));
677        assert_eq!(
678            e.level,
679            Level::High,
680            "full=2.0 (between fall and rise) must hold High, not flap"
681        );
682
683        // full drops below the fall threshold (1.0 < 1.5): now it may step down.
684        e.tick(3_000, sample(0.0, 1.0, 8000), &procs, &live_from(&procs));
685        assert_eq!(
686            e.level,
687            Level::Calm,
688            "full below fall threshold drops to Calm"
689        );
690    }
691
692    #[test]
693    fn disabled_engine_is_inert() {
694        let mut c = cfg();
695        c.enabled = false;
696        let mut e = PolicyEngine::new(c);
697        let procs = vec![proc(100, "firefox", 4000)];
698        assert!(e.tick(1000, high(), &procs, &live_from(&procs)).is_empty());
699    }
700
701    #[test]
702    fn high_freezes_largest_eligible_process() {
703        let mut e = PolicyEngine::new(cfg());
704        let procs = vec![
705            proc(1, "small", 300),
706            proc(2, "biggest", 4000),
707            proc(3, "medium", 1000),
708        ];
709        prime(&mut e, &procs);
710        let actions = e.tick(1000, high(), &procs, &live_from(&procs));
711        // Only the single largest hog is frozen, not the smaller ones.
712        assert_eq!(
713            freeze_targets(&actions),
714            vec!["/app.slice/app-biggest-2.scope"]
715        );
716        assert_eq!(e.level, Level::High);
717    }
718
719    #[test]
720    fn process_below_min_rss_is_never_selected() {
721        let mut e = PolicyEngine::new(cfg());
722        // Both below the 200 MB default floor.
723        let procs = vec![proc(1, "tiny", 50), proc(2, "small", 150)];
724        let actions = e.tick(1000, high(), &procs, &live_from(&procs));
725        assert!(
726            freeze_targets(&actions).is_empty(),
727            "froze a sub-min-rss process: {actions:?}"
728        );
729    }
730
731    #[test]
732    fn frozen_process_thaws_after_freeze_hold() {
733        let mut e = PolicyEngine::new(cfg()); // freeze_hold = 5s
734        let procs = vec![proc(2, "hog", 4000)];
735        prime(&mut e, &procs);
736        let cg = "/app.slice/app-hog-2.scope";
737
738        let a0 = e.tick(0, high(), &procs, &live_from(&procs));
739        assert_eq!(freeze_targets(&a0), vec![cg]);
740
741        // Before the hold elapses: no thaw yet (and escalation gate keeps it quiet).
742        let a1 = e.tick(4_000, high(), &procs, &live_from(&procs));
743        assert!(!has_thaw_target(&a1, cg), "thawed too early: {a1:?}");
744
745        // At/after 5s the freeze auto-thaws.
746        let a2 = e.tick(5_000, high(), &procs, &live_from(&procs));
747        assert!(has_thaw_target(&a2, cg), "expected thaw at hold: {a2:?}");
748        assert!(e.interventions().is_empty());
749    }
750
751    #[test]
752    fn still_high_within_cooldown_caps_instead_of_refreezing() {
753        let mut e = PolicyEngine::new(cfg()); // hold=5s, cooldown=60s
754        let procs = vec![proc(2, "hog", 4000)];
755        prime(&mut e, &procs);
756        let cg = "/app.slice/app-hog-2.scope";
757
758        // Freeze at t=0.
759        assert_eq!(
760            freeze_targets(&e.tick(0, high(), &procs, &live_from(&procs))),
761            vec![cg]
762        );
763        // Auto-thaw at t=5s.
764        assert!(has_thaw_target(
765            &e.tick(5_000, high(), &procs, &live_from(&procs)),
766            cg
767        ));
768
769        // Still high, and within the 60s freeze cooldown: Cap, not re-Freeze.
770        // t must clear the escalation gate (>= last_action 5000 + 5000 hold).
771        let a = e.tick(10_000, high(), &procs, &live_from(&procs));
772        assert!(
773            has_cap_target(&a, cg),
774            "expected cap within cooldown: {a:?}"
775        );
776        assert!(freeze_targets(&a).is_empty(), "should not re-freeze: {a:?}");
777        assert!(matches!(
778            e.interventions().as_slice(),
779            [(c, Intervention::Capped { .. })] if c == cg
780        ));
781    }
782
783    #[test]
784    fn hysteresis_holds_level_between_fall_and_rise() {
785        let mut e = PolicyEngine::new(cfg());
786        let procs = vec![proc(2, "hog", 4000)];
787        let cg = "/app.slice/app-hog-2.scope";
788
789        // Rise to High.
790        e.tick(0, high(), &procs, &live_from(&procs));
791        assert_eq!(e.level, Level::High);
792
793        // some=20 is below rise(30) but above fall(15): stay High, no lift.
794        let a = e.tick(20_000, sample(20.0, 0.0, 8000), &procs, &live_from(&procs));
795        assert_eq!(e.level, Level::High, "dropped out of High prematurely");
796        // A thaw here is expected (the freeze hold elapsed), but the cap must not
797        // be lifted while we're still High.
798        assert!(
799            !has_liftcap_target(&a, cg),
800            "should not lift while still High: {a:?}"
801        );
802
803        // Drop below the fall threshold (some < 15 and full < 3): fall to Warn.
804        e.tick(21_000, sample(12.0, 0.0, 8000), &procs, &live_from(&procs));
805        assert_eq!(e.level, Level::Warn);
806    }
807
808    #[test]
809    fn capped_process_lifted_only_after_sustained_calm() {
810        let mut e = PolicyEngine::new(cfg()); // calm_hold = 30s
811        let procs = vec![proc(2, "hog", 4000)];
812        prime(&mut e, &procs);
813        let cg = "/app.slice/app-hog-2.scope";
814
815        // Drive a freeze, thaw, then a cap (still hot within cooldown).
816        e.tick(0, high(), &procs, &live_from(&procs));
817        e.tick(5_000, high(), &procs, &live_from(&procs)); // thaw
818        let a = e.tick(10_000, high(), &procs, &live_from(&procs)); // cap
819        assert!(has_cap_target(&a, cg));
820
821        // Calm starts at t=15s. Before 30s of calm: no lift.
822        let a1 = e.tick(15_000, calm(), &procs, &live_from(&procs));
823        assert!(
824            !has_liftcap_target(&a1, cg),
825            "lifted before calm sustained: {a1:?}"
826        );
827        let a2 = e.tick(44_000, calm(), &procs, &live_from(&procs)); // 29s of calm
828        assert!(
829            !has_liftcap_target(&a2, cg),
830            "lifted just before hold: {a2:?}"
831        );
832
833        // 30s of sustained calm lifts the cap.
834        let a3 = e.tick(45_000, calm(), &procs, &live_from(&procs));
835        assert!(
836            has_liftcap_target(&a3, cg),
837            "expected lift after calm hold: {a3:?}"
838        );
839        assert!(e.interventions().is_empty());
840    }
841
842    #[test]
843    fn cap_lift_resets_if_calm_is_interrupted() {
844        let mut e = PolicyEngine::new(cfg());
845        let procs = vec![proc(2, "hog", 4000)];
846        prime(&mut e, &procs);
847        let cg = "/app.slice/app-hog-2.scope";
848        e.tick(0, high(), &procs, &live_from(&procs));
849        e.tick(5_000, high(), &procs, &live_from(&procs));
850        assert!(has_cap_target(
851            &e.tick(10_000, high(), &procs, &live_from(&procs)),
852            cg
853        ));
854
855        e.tick(15_000, calm(), &procs, &live_from(&procs)); // calm clock starts
856        e.tick(20_000, high(), &procs, &live_from(&procs)); // pressure returns, calm clock cleared
857                                                            // New calm window starts at 25s; at 50s only 25s have passed, so no lift.
858        e.tick(25_000, calm(), &procs, &live_from(&procs));
859        let a = e.tick(50_000, calm(), &procs, &live_from(&procs));
860        assert!(
861            !has_liftcap_target(&a, cg),
862            "calm clock should have reset: {a:?}"
863        );
864    }
865
866    #[test]
867    fn escalation_gate_limits_to_one_freeze_per_hold_window() {
868        let mut e = PolicyEngine::new(cfg());
869        let procs = vec![proc(1, "hog-a", 4000), proc(2, "hog-b", 3000)];
870        prime(&mut e, &procs);
871        let cg_a = "/app.slice/app-hog-a-1.scope";
872        let cg_b = "/app.slice/app-hog-b-2.scope";
873
874        // First High tick freezes hog-a.
875        let a0 = e.tick(0, high(), &procs, &live_from(&procs));
876        assert_eq!(freeze_targets(&a0), vec![cg_a]);
877
878        // Second High tick within the 5s hold: gate closed, no new freeze.
879        let a1 = e.tick(2_000, high(), &procs, &live_from(&procs));
880        assert!(
881            freeze_targets(&a1).is_empty(),
882            "gate should suppress second freeze: {a1:?}"
883        );
884
885        // After the gate reopens, the next hog can be frozen.
886        let a2 = e.tick(5_000, high(), &procs, &live_from(&procs));
887        assert_eq!(freeze_targets(&a2), vec![cg_b]);
888    }
889
890    #[test]
891    fn dead_pid_is_pruned_with_liftcap() {
892        let mut e = PolicyEngine::new(cfg());
893        let procs = vec![proc(2, "hog", 4000)];
894        prime(&mut e, &procs);
895        let cg = "/app.slice/app-hog-2.scope";
896
897        // Freeze the hog's cgroup.
898        assert_eq!(
899            freeze_targets(&e.tick(0, high(), &procs, &live_from(&procs))),
900            vec![cg]
901        );
902        assert_eq!(e.interventions().len(), 1);
903
904        // Next tick the process (and its resolution) has vanished, so LiftCap
905        // cleanup, intervention dropped.
906        let a = e.tick(1_000, calm(), &[], &live_from(&[]));
907        assert!(
908            has_liftcap_target(&a, cg),
909            "expected LiftCap for dead cgroup: {a:?}"
910        );
911        assert!(e.interventions().is_empty());
912    }
913
914    /// D2 regression: a cgroup absent from `procs` (e.g. the cap evicted
915    /// enough file pages to drop the process below `min_rss_mb`) but still
916    /// present in `live_cgroups` must NOT be pruned: the cgroup is real and
917    /// alive, just not currently eligible for (re-)selection. A cgroup
918    /// absent from *both* sets must still be pruned with a LiftCap.
919    #[test]
920    fn intervention_survives_in_live_cgroups_but_absent_from_procs() {
921        let mut e = PolicyEngine::new(cfg());
922        let procs = vec![proc(2, "hog", 4000)];
923        prime(&mut e, &procs);
924        let cg = "/app.slice/app-hog-2.scope";
925
926        // Freeze the hog's cgroup.
927        assert_eq!(
928            freeze_targets(&e.tick(0, high(), &procs, &live_from(&procs))),
929            vec![cg]
930        );
931        assert_eq!(e.interventions().len(), 1);
932
933        // Next tick: the process no longer appears in `procs` (as if the cap
934        // evicted its file pages below the min-RSS floor), but its cgroup is
935        // still in `live_cgroups`, so it must NOT be pruned.
936        let live: std::collections::HashSet<String> = [cg.to_string()].into();
937        let a1 = e.tick(1_000, calm(), &[], &live);
938        assert!(
939            !has_liftcap_target(&a1, cg),
940            "must not prune a cgroup still present in live_cgroups: {a1:?}"
941        );
942        assert_eq!(
943            e.interventions().len(),
944            1,
945            "intervention must survive while the cgroup is live"
946        );
947
948        // Now the cgroup is gone from both sets entirely, so it is pruned.
949        let a2 = e.tick(2_000, calm(), &[], &std::collections::HashSet::new());
950        assert!(
951            has_liftcap_target(&a2, cg),
952            "expected LiftCap once absent from live_cgroups too: {a2:?}"
953        );
954        assert!(e.interventions().is_empty());
955    }
956
957    #[test]
958    fn interventions_reflect_state_sorted_by_cgroup() {
959        let mut e = PolicyEngine::new(cfg());
960        // pid 5 ("a") is the larger hog; pid 3 ("b") the smaller. We build a
961        // Capped "a" and a "b" capped after a freeze, both active, then check
962        // ordering + content. "/app.slice/app-a-5.scope" sorts before
963        // "/app.slice/app-b-3.scope" lexicographically.
964        let procs = vec![proc(5, "a", 4000), proc(3, "b", 3500)];
965        prime(&mut e, &procs);
966        let cg_a = "/app.slice/app-a-5.scope";
967        let cg_b = "/app.slice/app-b-3.scope";
968
969        // Walk both cgroups down the freeze, thaw, (still hot) cap ladder
970        // so two Capped interventions coexist. Caps persist while High (never
971        // auto-thaw), which is what lets two interventions overlap under
972        // default timing.
973        assert_eq!(
974            freeze_targets(&e.tick(0, high(), &procs, &live_from(&procs))),
975            vec![cg_a]
976        ); // freeze a
977        let a1 = e.tick(5_000, high(), &procs, &live_from(&procs)); // thaw a, freeze b
978        assert!(has_thaw_target(&a1, cg_a));
979        assert_eq!(freeze_targets(&a1), vec![cg_b]);
980        let a2 = e.tick(10_000, high(), &procs, &live_from(&procs)); // thaw b, cap a (in cooldown)
981        assert!(has_thaw_target(&a2, cg_b));
982        assert!(has_cap_target(&a2, cg_a));
983        let a3 = e.tick(15_000, high(), &procs, &live_from(&procs)); // cap b (in cooldown)
984        assert!(has_cap_target(&a3, cg_b));
985
986        let ivs = e.interventions();
987        assert_eq!(ivs.len(), 2, "expected a + b both Capped: {ivs:?}");
988        // Sorted ascending by cgroup path.
989        assert_eq!(ivs[0].0, cg_a);
990        assert_eq!(ivs[1].0, cg_b);
991        assert!(ivs
992            .iter()
993            .all(|(_, iv)| matches!(iv, Intervention::Capped { .. })));
994    }
995
996    #[test]
997    fn critical_via_mem_floor_triggers_action() {
998        let mut e = PolicyEngine::new(cfg());
999        let procs = vec![proc(2, "hog", 4000)];
1000        prime(&mut e, &procs);
1001        // No PSI pressure, but MemAvailable below the 400 MB floor, so Critical.
1002        let a = e.tick(0, sample(0.0, 0.0, 100), &procs, &live_from(&procs));
1003        assert_eq!(e.level, Level::Critical);
1004        assert_eq!(freeze_targets(&a), vec!["/app.slice/app-hog-2.scope"]);
1005    }
1006
1007    #[test]
1008    fn notify_emitted_and_rate_limited() {
1009        let mut e = PolicyEngine::new(cfg());
1010        let procs = vec![proc(2, "hog", 4000)];
1011
1012        // Warn level: some>=10 but below high; just notify, no freeze.
1013        let a0 = e.tick(0, sample(12.0, 0.0, 8000), &procs, &live_from(&procs));
1014        assert!(
1015            a0.iter().any(|x| matches!(x, Action::Notify { .. })),
1016            "expected a notify at Warn: {a0:?}"
1017        );
1018        assert!(freeze_targets(&a0).is_empty());
1019
1020        // Within 60s: no second notify.
1021        let a1 = e.tick(30_000, sample(12.0, 0.0, 8000), &procs, &live_from(&procs));
1022        assert!(
1023            !a1.iter().any(|x| matches!(x, Action::Notify { .. })),
1024            "notify should be rate-limited: {a1:?}"
1025        );
1026
1027        // After 60s: notify again.
1028        let a2 = e.tick(60_000, sample(12.0, 0.0, 8000), &procs, &live_from(&procs));
1029        assert!(a2.iter().any(|x| matches!(x, Action::Notify { .. })));
1030    }
1031
1032    #[test]
1033    fn notify_disabled_suppresses_notifications() {
1034        let mut c = cfg();
1035        c.notify = false;
1036        let mut e = PolicyEngine::new(c);
1037        let procs = vec![proc(2, "hog", 4000)];
1038        let a = e.tick(0, sample(12.0, 0.0, 8000), &procs, &live_from(&procs));
1039        assert!(!a.iter().any(|x| matches!(x, Action::Notify { .. })));
1040    }
1041
1042    // ---- Task 5 new behaviors --------------------------------------------
1043
1044    #[test]
1045    fn caponly_verdict_never_freezes() {
1046        let mut e = PolicyEngine::new(cfg());
1047        let mut p = proc_at(2, "script", 4000, "/app.slice/app-alacritty-9.scope");
1048        let r = &mut p.resolution;
1049        r.verdict = Verdict::CapOnly;
1050        r.coverage = Coverage::Partial;
1051        let procs = [p];
1052        prime(&mut e, &procs);
1053        let a = e.tick(0, high(), &procs, &live_from(&procs));
1054        assert!(
1055            freeze_targets(&a).is_empty(),
1056            "CapOnly must not freeze: {a:?}"
1057        );
1058        assert!(has_cap_target(&a, "/app.slice/app-alacritty-9.scope"));
1059    }
1060
1061    #[test]
1062    fn partial_action_waits_at_least_three_seconds() {
1063        let mut e = PolicyEngine::new(cfg());
1064        let mut term = target("python3", "/app.slice/term.scope", 4000);
1065        term.resolution.verdict = Verdict::CapOnly;
1066        term.resolution.coverage = Coverage::Partial;
1067        let ts = vec![term, target("hog", "/app.slice/hog.scope", 3000)];
1068        prime(&mut e, &ts);
1069        assert!(has_cap_target(
1070            &e.tick(0, high(), &ts, &live_from(&ts)),
1071            "/app.slice/term.scope"
1072        ));
1073        assert!(freeze_targets(&e.tick(1_000, high(), &ts, &live_from(&ts))).is_empty());
1074        assert!(freeze_targets(&e.tick(2_000, high(), &ts, &live_from(&ts))).is_empty());
1075        assert!(has_freeze_target(
1076            &e.tick(3_000, high(), &ts, &live_from(&ts)),
1077            "/app.slice/hog.scope"
1078        ));
1079    }
1080
1081    #[test]
1082    fn two_scopes_of_one_app_are_acted_on_together() {
1083        let mut e = PolicyEngine::new(cfg());
1084        let ts = vec![
1085            target("chrome", "/app.slice/app-chrome-1.scope", 1500),
1086            target("chrome", "/app.slice/app-chrome-2.scope", 900),
1087            target("hog", "/app.slice/app-hog-3.scope", 2000),
1088        ];
1089        prime(&mut e, &ts);
1090        let mut frozen = freeze_targets(&e.tick(0, high(), &ts, &live_from(&ts)));
1091        frozen.sort();
1092        assert_eq!(
1093            frozen,
1094            vec![
1095                "/app.slice/app-chrome-1.scope",
1096                "/app.slice/app-chrome-2.scope"
1097            ]
1098        );
1099        let a1 = e.tick(1_000, high(), &ts, &live_from(&ts));
1100        assert!(
1101            freeze_targets(&a1).is_empty(),
1102            "one app is one escalation step: {a1:?}"
1103        );
1104    }
1105
1106    #[test]
1107    fn fastest_growing_app_is_chosen_over_largest() {
1108        let mut e = PolicyEngine::new(cfg());
1109        let warn = sample(12.0, 0.0, 2_000);
1110        let t0 = vec![
1111            target("firefox", "/app.slice/ff.scope", 4000),
1112            target("script", "/app.slice/sh.scope", 1000),
1113        ];
1114        assert!(freeze_targets(&e.tick(0, warn, &t0, &live_from(&t0))).is_empty());
1115        let t1 = vec![
1116            target("firefox", "/app.slice/ff.scope", 4000),
1117            target("script", "/app.slice/sh.scope", 1600),
1118        ];
1119        assert_eq!(
1120            freeze_targets(&e.tick(1_000, high(), &t1, &live_from(&t1))),
1121            vec!["/app.slice/sh.scope"]
1122        );
1123    }
1124
1125    #[test]
1126    fn largest_app_is_the_fallback_when_nothing_grows() {
1127        let mut e = PolicyEngine::new(cfg());
1128        let ts = vec![
1129            target("small", "/app.slice/s.scope", 300),
1130            target("big", "/app.slice/b.scope", 3000),
1131        ];
1132        e.tick(0, sample(12.0, 0.0, 2_000), &ts, &live_from(&ts));
1133        assert_eq!(
1134            freeze_targets(&e.tick(1_000, high(), &ts, &live_from(&ts))),
1135            vec!["/app.slice/b.scope"]
1136        );
1137    }
1138
1139    #[test]
1140    fn at_most_three_apps_are_held_at_once() {
1141        let mut e = PolicyEngine::new(cfg());
1142        let ts: Vec<Target> = (0..5u64)
1143            .map(|i| {
1144                target(
1145                    &format!("app{i}"),
1146                    &format!("/app.slice/a{i}.scope"),
1147                    1000 + i * 100,
1148                )
1149            })
1150            .collect();
1151        for step in 0..40u64 {
1152            e.tick(step * 5_000, high(), &ts, &live_from(&ts));
1153            assert!(
1154                e.interventions().len() <= MAX_HELD_APPS,
1155                "held {:?}",
1156                e.interventions()
1157            );
1158        }
1159    }
1160
1161    #[test]
1162    fn held_limit_counts_apps_not_cgroups() {
1163        let mut e = PolicyEngine::new(cfg());
1164        let ts = vec![
1165            target("a", "/app.slice/a1.scope", 2000),
1166            target("a", "/app.slice/a2.scope", 2000),
1167            target("b", "/app.slice/b.scope", 1500),
1168            target("c", "/app.slice/c.scope", 1200),
1169            target("d", "/app.slice/d.scope", 1100),
1170        ];
1171        for step in 0..40u64 {
1172            e.tick(step * 5_000, high(), &ts, &live_from(&ts));
1173        }
1174        let held: Vec<String> = e.interventions().into_iter().map(|(cg, _)| cg).collect();
1175        assert_eq!(
1176            held,
1177            vec![
1178                "/app.slice/a1.scope",
1179                "/app.slice/a2.scope",
1180                "/app.slice/b.scope",
1181                "/app.slice/c.scope"
1182            ],
1183            "4 cgroups across 3 apps are allowed, a 4th app is refused"
1184        );
1185    }
1186
1187    #[test]
1188    fn candidates_are_wanted_above_calm_or_when_scarce() {
1189        let e = PolicyEngine::new(cfg());
1190        assert!(!e.wants_candidates(calm()));
1191        assert!(e.wants_candidates(sample(12.0, 0.0, 8_000)));
1192        // Calm PSI but under 20% of RAM available: scan so growth stays warm.
1193        assert!(e.wants_candidates(sample(0.0, 0.0, 3_000)));
1194    }
1195
1196    /// Regression (final review I1): available memory drops below the floor
1197    /// in one step with no stall first, so the Critical tick is the first
1198    /// scan. An idle 6 GB app must not be frozen in place of a 3 GB app
1199    /// that is growing: the first tick defers, the next picks the grower.
1200    #[test]
1201    fn cold_start_defers_then_picks_grower_not_largest() {
1202        let mut e = PolicyEngine::new(cfg());
1203        for step in 0..5u64 {
1204            e.tick(step * 1_000, calm(), &[], &HashSet::new());
1205        }
1206        let crit = sample(0.0, 0.0, 300);
1207        let t0 = vec![
1208            target("chrome", "/app.slice/chrome.scope", 6000),
1209            target("hog", "/app.slice/hog.scope", 3000),
1210        ];
1211        let a0 = e.tick(5_000, crit, &t0, &live_from(&t0));
1212        assert_eq!(e.level, Level::Critical);
1213        assert!(
1214            freeze_targets(&a0).is_empty(),
1215            "no growth data yet, must defer: {a0:?}"
1216        );
1217        let t1 = vec![
1218            target("chrome", "/app.slice/chrome.scope", 6000),
1219            target("hog", "/app.slice/hog.scope", 3200),
1220        ];
1221        assert_eq!(
1222            freeze_targets(&e.tick(6_000, crit, &t1, &live_from(&t1))),
1223            vec!["/app.slice/hog.scope"]
1224        );
1225    }
1226
1227    /// With scarce-but-calm ticks feeding growth, the grower is picked on
1228    /// the very first Critical tick.
1229    #[test]
1230    fn scarce_calm_ticks_warm_growth_for_first_critical_tick() {
1231        let mut e = PolicyEngine::new(cfg());
1232        let scarce_calm = sample(0.0, 0.0, 3_000);
1233        assert!(e.wants_candidates(scarce_calm));
1234        let t0 = vec![
1235            target("chrome", "/app.slice/chrome.scope", 6000),
1236            target("hog", "/app.slice/hog.scope", 2000),
1237        ];
1238        assert!(freeze_targets(&e.tick(0, scarce_calm, &t0, &live_from(&t0))).is_empty());
1239        let t1 = vec![
1240            target("chrome", "/app.slice/chrome.scope", 6000),
1241            target("hog", "/app.slice/hog.scope", 3000),
1242        ];
1243        assert_eq!(
1244            freeze_targets(&e.tick(1_000, sample(0.0, 0.0, 300), &t1, &live_from(&t1))),
1245            vec!["/app.slice/hog.scope"]
1246        );
1247    }
1248
1249    /// A new cgroup on every tick never gets a measured growth rate. The
1250    /// deferral is bounded, so the guard still acts by the fourth tick and
1251    /// falls back to the largest eligible app.
1252    #[test]
1253    fn cold_start_deferral_is_bounded_when_cgroups_keep_changing() {
1254        let mut e = PolicyEngine::new(cfg());
1255        let crit = sample(0.0, 0.0, 300);
1256        for tick in 0..4u64 {
1257            let ts = vec![
1258                target(
1259                    &format!("big{tick}"),
1260                    &format!("/app.slice/b{tick}.scope"),
1261                    3000,
1262                ),
1263                target(
1264                    &format!("small{tick}"),
1265                    &format!("/app.slice/s{tick}.scope"),
1266                    1000,
1267                ),
1268            ];
1269            let frozen = freeze_targets(&e.tick(tick * 1_000, crit, &ts, &live_from(&ts)));
1270            if tick < u64::from(MAX_COLD_DEFER_TICKS) {
1271                assert!(frozen.is_empty(), "tick {tick} should defer: {frozen:?}");
1272            } else {
1273                assert_eq!(frozen, vec![format!("/app.slice/b{tick}.scope")]);
1274            }
1275        }
1276    }
1277
1278    /// Without any memory.current reading, growth can never be measured, so
1279    /// the guard does not wait and falls back to the largest app.
1280    #[test]
1281    fn no_current_bytes_does_not_defer() {
1282        let mut e = PolicyEngine::new(cfg());
1283        let mut big = target("big", "/app.slice/b.scope", 3000);
1284        big.current_bytes = None;
1285        let mut small = target("small", "/app.slice/s.scope", 1000);
1286        small.current_bytes = None;
1287        let ts = vec![big, small];
1288        assert_eq!(
1289            freeze_targets(&e.tick(0, high(), &ts, &live_from(&ts))),
1290            vec!["/app.slice/b.scope"]
1291        );
1292    }
1293}