Skip to main content

rlmctl_core/guard/
policy.rs

1//! Pure policy state machine: the self-healing circuit breaker at the heart of
2//! the freeze guard.
3//!
4//! Contract: [`PolicyEngine::tick`] is pure given `(now_ms, sample, targets,
5//! live_cgroups)` plus the engine's own internal state. It performs **no**
6//! syscalls and reads **no** clock; `now_ms` (monotonic milliseconds) is
7//! injected by the caller. That is what makes the whole escalation/recovery
8//! ladder unit-testable without root.
9
10use std::collections::{BTreeMap, HashMap, HashSet};
11
12use super::resolve::{Coverage, Resolution, Verdict};
13use super::types::{Action, Intervention, Level, Sample, Target};
14use common::GuardConfig;
15
16/// PSI `full` avg10 (%) that, on its own, forces at least the High level. Mirrors
17/// the design doc's "or `full.avg10 >= 3`" High trigger.
18const FULL_HIGH_RISE: f64 = 3.0;
19
20/// Escalation gate (ms) after an action that only partly covered its app.
21/// PSI avg10 is a 10 s average, so re-measuring after 1 s still sees the old
22/// pressure; waiting at least this long stops a partial action from
23/// cascading into several more within seconds.
24pub const PARTIAL_GATE_MS: u64 = 3_000;
25/// Most apps the guard holds (frozen or capped) at the same time.
26pub const MAX_HELD_APPS: usize = 3;
27/// Growth rate (bytes/s of `memory.current`) an app must reach to be picked
28/// as the one causing pressure. Below it, the largest app is picked instead.
29pub const MIN_GROWTH_BPS: f64 = 1_048_576.0;
30/// Most consecutive ticks victim selection waits for growth data when every
31/// eligible cgroup is newly seen. After that the largest app is picked, so a
32/// stream of short-lived cgroups cannot keep the guard from acting.
33pub const MAX_COLD_DEFER_TICKS: u32 = 3;
34
35/// The level a sample reaches on its own, from the rise thresholds alone
36/// (no hysteresis): Critical when PSI `full` reaches `psi_full_critical` or
37/// free memory is below `mem_available_floor_mb`, High when `some` reaches
38/// `psi_some_high` or `full` reaches [`FULL_HIGH_RISE`], Warn when `some`
39/// reaches `psi_some_warn`. The engine enters a level on exactly these rules;
40/// callers that only show the pressure (the GUI) use this to agree with it.
41pub fn rise_level(s: &Sample, t: &common::GuardTrigger) -> Level {
42    if s.full_avg10 >= t.psi_full_critical || s.mem_available_mb < t.mem_available_floor_mb {
43        Level::Critical
44    } else if s.some_avg10 >= t.psi_some_high || s.full_avg10 >= FULL_HIGH_RISE {
45        Level::High
46    } else if s.some_avg10 >= t.psi_some_warn {
47        Level::Warn
48    } else {
49        Level::Calm
50    }
51}
52
53/// True when the host is actually short of memory: below the hard floor, or
54/// below `act_below_available_pct` percent of RAM. A stall confined to one
55/// cgroup's memory.max leaves MemAvailable high, so it never passes this gate.
56pub fn is_scarce(s: &Sample, t: &common::GuardTrigger) -> bool {
57    s.mem_available_mb < t.mem_available_floor_mb
58        || (s.mem_total_mb > 0
59            && s.mem_available_mb.saturating_mul(100)
60                < s.mem_total_mb.saturating_mul(t.act_below_available_pct))
61}
62
63/// Smoothed `memory.current` growth for one cgroup.
64struct Growth {
65    last_bytes: u64,
66    last_ms: u64,
67    /// EWMA of bytes per second (negative while shrinking).
68    rate_bps: f64,
69    /// True once a second sample has been seen, so `rate_bps` is a measured
70    /// rate and not the zero placeholder of a first sighting.
71    warm: bool,
72}
73
74/// Self-healing circuit-breaker policy engine.
75///
76/// On a memory spike it drives the ladder *freeze (short), auto-thaw,
77/// if still high a soft cap, once calm is sustained a lift*, never issuing a kill. All
78/// of that lives in [`tick`](Self::tick); the struct just holds the state needed
79/// to make decisions stable across ticks (hysteresis, cooldowns, growth).
80///
81/// The unit of choice is an app: every cgroup of the chosen app is frozen or
82/// capped together as one escalation step.
83pub struct PolicyEngine {
84    cfg: GuardConfig,
85    /// Current pressure level (carried across ticks so hysteresis works).
86    level: Level,
87    /// Active interventions keyed by resolved cgroup path, with the
88    /// [`Resolution`] (so `Thaw`/`LiftCap` can be built without re-resolving)
89    /// and the app the cgroup was acted on as.
90    interventions: HashMap<String, (Intervention, Resolution, String)>,
91    /// Last time each app was frozen. Drives the per-app freeze cooldown that
92    /// decides freeze-vs-cap, and is intentionally kept after a thaw.
93    last_freeze_ms: HashMap<String, u64>,
94    /// Per-cgroup `memory.current` growth estimate.
95    growth: HashMap<String, Growth>,
96    /// When the level last became `Calm` (None while not calm). Gates cap lifts.
97    calm_since_ms: Option<u64>,
98    /// When we last emitted a new freeze/cap: the global escalation gate.
99    /// `None` means "never acted", so the gate is open on the first action.
100    last_action_ms: Option<u64>,
101    /// Whether the most recent escalation acted under `Coverage::Partial`.
102    /// When true the gate is shortened to [`PARTIAL_GATE_MS`] (capped at the
103    /// freeze hold) so the guard can re-assess sooner, but never instantly.
104    last_action_partial: bool,
105    /// Consecutive ticks on which selection deferred for lack of growth
106    /// data. Bounded by [`MAX_COLD_DEFER_TICKS`].
107    cold_defer_ticks: u32,
108}
109
110impl PolicyEngine {
111    pub fn new(cfg: GuardConfig) -> Self {
112        Self {
113            cfg,
114            level: Level::Calm,
115            interventions: HashMap::new(),
116            last_freeze_ms: HashMap::new(),
117            growth: HashMap::new(),
118            calm_since_ms: None,
119            last_action_ms: None,
120            last_action_partial: false,
121            cold_defer_ticks: 0,
122        }
123    }
124
125    /// Advance the state machine one tick and return the actions to apply.
126    ///
127    /// `targets` are the cgroups eligible for action this tick (already
128    /// filtered for uid, protect list and min RSS by the Sampler).
129    ///
130    /// `live_cgroups` is the subset of the engine's intervened cgroups that
131    /// still hold a process (see `sampler::live_cgroups`), with no min-RSS
132    /// or protect filtering applied; it is deliberately independent of
133    /// `targets`. Pruning checks liveness against
134    /// this set, not against `targets`: `memory.high` also bounds
135    /// file-backed pages, so capping
136    /// a mapped-file-heavy process can push its `rss_kb` below the min-RSS
137    /// floor on the very next tick, dropping it out of `targets` even though
138    /// the cgroup is very much still alive. Pruning against `targets` there
139    /// would lift the cap while pressure is still Critical and immediately
140    /// re-trigger it. Victim *selection* deliberately keeps using `targets`.
141    pub fn tick(
142        &mut self,
143        now_ms: u64,
144        sample: Sample,
145        targets: &[Target],
146        live_cgroups: &HashSet<String>,
147    ) -> Vec<Action> {
148        // 1. Disabled guard is inert.
149        if !self.cfg.enabled {
150            return Vec::new();
151        }
152
153        let mut actions = Vec::new();
154
155        // 2. Recompute the level with hysteresis and track how long we've been calm.
156        self.level = self.next_level(sample);
157        match self.level {
158            Level::Calm => {
159                // Start the calm clock on the *transition* into calm, then leave it.
160                if self.calm_since_ms.is_none() {
161                    self.calm_since_ms = Some(now_ms);
162                }
163                self.cold_defer_ticks = 0;
164            }
165            _ => self.calm_since_ms = None,
166        }
167
168        // 3. Track memory.current growth per cgroup.
169        self.update_growth(now_ms, targets);
170
171        // 4. Prune interventions whose cgroup is no longer live (see
172        //    `live_cgroups` doc above). LiftCap doubles as "tear down the
173        //    cap", so it's the right cleanup for both frozen and capped dead
174        //    cgroups; the effector's LiftCap tolerates a missing cgroup.
175        let dead: Vec<String> = self
176            .interventions
177            .keys()
178            .filter(|cg| !live_cgroups.contains(cg.as_str()))
179            .cloned()
180            .collect();
181        for cg in dead {
182            let (_, res, _) = self.interventions.remove(&cg).expect("just found key");
183            actions.push(Action::LiftCap { res });
184        }
185
186        // 5. Recover: auto-thaw held freezes, and lift caps once calm has held.
187        let freeze_hold_ms = self.cfg.timing.freeze_hold_secs.saturating_mul(1000);
188        let calm_hold_ms = self.cfg.timing.calm_hold_secs.saturating_mul(1000);
189        let mut recovered = Vec::new();
190        // Apps thawed on this tick must not be re-targeted by escalation in
191        // the same tick; they need a re-measure first.
192        let mut thawed_apps: HashSet<String> = HashSet::new();
193        for (cg, (intervention, res, app)) in &self.interventions {
194            match *intervention {
195                Intervention::Frozen { since_ms } => {
196                    if now_ms.saturating_sub(since_ms) >= freeze_hold_ms {
197                        actions.push(Action::Thaw { res: res.clone() });
198                        recovered.push(cg.clone());
199                        thawed_apps.insert(app.clone());
200                    }
201                }
202                Intervention::Capped { .. } => {
203                    // Only lift a cap when pressure is calm *and* has stayed calm
204                    // long enough; this prevents re-capping churn. `calm_since_ms` is
205                    // the transition timestamp, so this measures *sustained* calm.
206                    if self.level == Level::Calm {
207                        if let Some(calm_since) = self.calm_since_ms {
208                            if now_ms.saturating_sub(calm_since) >= calm_hold_ms {
209                                actions.push(Action::LiftCap { res: res.clone() });
210                                recovered.push(cg.clone());
211                            }
212                        }
213                    }
214                }
215            }
216        }
217        for cg in recovered {
218            // Note: last_freeze_ms is intentionally retained for cooldown logic.
219            self.interventions.remove(&cg);
220        }
221
222        // 6. Escalate, but only when apps feel pressure, memory is actually
223        //    short (see `is_scarce`), the gate is open, and fewer than
224        //    MAX_HELD_APPS apps are already held.
225        if matches!(self.level, Level::High | Level::Critical)
226            && is_scarce(&sample, &self.cfg.trigger)
227            && self.held_apps().len() < MAX_HELD_APPS
228        {
229            // Global gate: after acting on one app, wait a freeze-hold before
230            // acting again so we re-measure instead of cascading. After a
231            // partial action wait a shorter, but never zero, interval.
232            let gate_ms = if self.last_action_partial {
233                PARTIAL_GATE_MS.min(freeze_hold_ms)
234            } else {
235                freeze_hold_ms
236            };
237            let gate_open = match self.last_action_ms {
238                None => true,
239                Some(last) => now_ms.saturating_sub(last) >= gate_ms,
240            };
241            if gate_open {
242                if let Some((app, members)) = self.select_app(targets, &thawed_apps) {
243                    let cap_only = members
244                        .iter()
245                        .any(|m| m.resolution.verdict == Verdict::CapOnly);
246                    let partial = members
247                        .iter()
248                        .any(|m| m.resolution.coverage == Coverage::Partial);
249                    let cooldown_ms = self.cfg.timing.freeze_cooldown_secs.saturating_mul(1000);
250                    let in_cooldown = self
251                        .last_freeze_ms
252                        .get(&app)
253                        .is_some_and(|&last| now_ms.saturating_sub(last) < cooldown_ms);
254                    // CapOnly never freezes; a recently frozen app that is
255                    // still hot escalates to a soft cap.
256                    let freeze = !cap_only && !in_cooldown;
257
258                    for m in members {
259                        let res = m.resolution.clone();
260                        let intervention = if freeze {
261                            actions.push(Action::Freeze {
262                                res: res.clone(),
263                                name: app.clone(),
264                            });
265                            Intervention::Frozen { since_ms: now_ms }
266                        } else {
267                            actions.push(Action::Cap {
268                                res: res.clone(),
269                                name: app.clone(),
270                            });
271                            Intervention::Capped { since_ms: now_ms }
272                        };
273                        self.interventions
274                            .insert(res.cgroup.clone(), (intervention, res, app.clone()));
275                    }
276                    if freeze {
277                        self.last_freeze_ms.insert(app.clone(), now_ms);
278                    }
279                    self.last_action_ms = Some(now_ms);
280                    self.last_action_partial = partial;
281                }
282            }
283        }
284
285        actions
286    }
287
288    /// True when the caller should gather targets for the next tick: the
289    /// level would be above Calm, or memory is already scarce. Scanning while
290    /// scarce keeps growth rates warm, so a sudden drop below the floor can
291    /// pick the app that is growing instead of the largest one. A disabled
292    /// engine never wants them.
293    pub fn wants_candidates(&self, sample: Sample) -> bool {
294        self.cfg.enabled
295            && (self.next_level(sample) != Level::Calm || is_scarce(&sample, &self.cfg.trigger))
296    }
297
298    /// The pressure level as of the last tick.
299    pub fn level(&self) -> Level {
300        self.level
301    }
302
303    /// Currently active interventions (cgroup path and intervention), sorted by
304    /// cgroup path so callers get a deterministic order.
305    pub fn interventions(&self) -> Vec<(String, Intervention)> {
306        let mut out: Vec<(String, Intervention)> = self
307            .interventions
308            .iter()
309            .map(|(cg, (iv, _, _))| (cg.clone(), *iv))
310            .collect();
311        out.sort_by(|(a, _), (b, _)| a.cmp(b));
312        out
313    }
314
315    /// Cgroup paths the engine currently holds an intervention on, frozen
316    /// *or* capped. Interventions are already keyed by cgroup, so this is
317    /// just the key set. For external callers (e.g.
318    /// `RulesEnforcer::reconcile`, D1) that must not fight/revert an active
319    /// guard action: rewriting `memory.high` on a cgroup the engine just
320    /// capped would silently no-op the cap and leave `PolicyEngine` holding
321    /// a stale `Capped` intervention that then blocks victim re-selection
322    /// for the rest of the pressure episode.
323    pub fn intervened_cgroups(&self) -> Vec<String> {
324        self.interventions.keys().cloned().collect()
325    }
326
327    /// Distinct apps with at least one active intervention.
328    fn held_apps(&self) -> HashSet<&str> {
329        self.interventions
330            .values()
331            .map(|(_, _, app)| app.as_str())
332            .collect()
333    }
334
335    /// Compute the next level from the current level + a fresh sample, applying
336    /// rise/fall hysteresis. The fall threshold is half the rise threshold, and
337    /// we only ever *step down* when below the fall threshold, so a sample that
338    /// sits between fall and rise leaves the level unchanged (no flapping).
339    fn next_level(&self, s: Sample) -> Level {
340        let t = &self.cfg.trigger;
341        let floor = t.mem_available_floor_mb;
342
343        // Rise predicates (cross the upper threshold to enter a level).
344        let rise = rise_level(&s, t);
345        let crit_rise = rise == Level::Critical;
346        let high_rise = crit_rise || rise == Level::High;
347        let warn_rise = high_rise || rise == Level::Warn;
348
349        // Stay predicates (above the lower/fall threshold: keep the level).
350        let warn_stay = s.some_avg10 >= t.psi_some_warn / 2.0;
351        let high_stay =
352            s.some_avg10 >= t.psi_some_high / 2.0 || s.full_avg10 >= FULL_HIGH_RISE / 2.0;
353        let crit_stay = s.full_avg10 >= t.psi_full_critical / 2.0 || s.mem_available_mb < floor;
354
355        // Highest level we're allowed to be at, given current level + hysteresis.
356        // For each tier: enter if its rise fires; otherwise remain if we're
357        // already at/above it and its stay predicate still holds.
358        let at_critical = self.level == Level::Critical;
359        let at_high = matches!(self.level, Level::High | Level::Critical);
360        let at_warn = matches!(self.level, Level::Warn | Level::High | Level::Critical);
361
362        if crit_rise || (at_critical && crit_stay) {
363            Level::Critical
364        } else if high_rise || (at_high && high_stay) {
365            Level::High
366        } else if warn_rise || (at_warn && warn_stay) {
367            Level::Warn
368        } else {
369            Level::Calm
370        }
371    }
372
373    /// Update each target cgroup's `memory.current` growth rate (an EWMA
374    /// with weight 0.5 per tick) and forget cgroups no longer present.
375    fn update_growth(&mut self, now_ms: u64, targets: &[Target]) {
376        let mut seen = HashSet::new();
377        for t in targets {
378            let Some(cur) = t.current_bytes else {
379                continue;
380            };
381            let cg = &t.resolution.cgroup;
382            seen.insert(cg.clone());
383            match self.growth.get_mut(cg) {
384                Some(g) if now_ms > g.last_ms => {
385                    let dt = (now_ms - g.last_ms) as f64 / 1000.0;
386                    let inst = (cur as f64 - g.last_bytes as f64) / dt;
387                    g.rate_bps = 0.5 * g.rate_bps + 0.5 * inst;
388                    g.last_bytes = cur;
389                    g.last_ms = now_ms;
390                    g.warm = true;
391                }
392                Some(_) => {}
393                None => {
394                    self.growth.insert(
395                        cg.clone(),
396                        Growth {
397                            last_bytes: cur,
398                            last_ms: now_ms,
399                            rate_bps: 0.0,
400                            warm: false,
401                        },
402                    );
403                }
404            }
405        }
406        self.growth.retain(|cg, _| seen.contains(cg));
407    }
408
409    /// Pick the app to act on and its member targets. Eligible apps are not
410    /// held, not thawed this tick (`blocked`), have a member at or above the
411    /// min-RSS floor, and have no member cgroup under an intervention. The
412    /// app whose cgroups grow fastest wins when that growth is at least
413    /// [`MIN_GROWTH_BPS`]; otherwise the largest app. Ties go to the
414    /// lexicographically smaller app name, so the choice is deterministic.
415    ///
416    /// Cold start: when no eligible cgroup has a measured growth rate yet but
417    /// at least one was first seen this tick, nothing is picked. The next
418    /// tick (one sample interval later) has rates, so an idle large app is
419    /// not frozen in place of a smaller one that is growing. The deferral
420    /// lasts one tick at most per new cgroup, and at most
421    /// [`MAX_COLD_DEFER_TICKS`] ticks in a row; after that the largest app is
422    /// picked. If no eligible cgroup reports `memory.current` at all, growth
423    /// can never be measured and the largest app is picked at once.
424    fn select_app<'a>(
425        &mut self,
426        targets: &'a [Target],
427        blocked: &HashSet<String>,
428    ) -> Option<(String, Vec<&'a Target>)> {
429        let min_rss_kb = self.cfg.selection.min_rss_mb.saturating_mul(1024);
430        let held = self.held_apps();
431        let mut groups: BTreeMap<&str, Vec<&Target>> = BTreeMap::new();
432        for t in targets {
433            groups.entry(t.app.as_str()).or_default().push(t);
434        }
435        let eligible: Vec<(&str, Vec<&Target>)> = groups
436            .into_iter()
437            .filter(|(app, ms)| {
438                !held.contains(app)
439                    && !blocked.contains(*app)
440                    && ms.iter().any(|m| m.rss_kb >= min_rss_kb)
441                    && ms
442                        .iter()
443                        .all(|m| !self.interventions.contains_key(&m.resolution.cgroup))
444            })
445            .collect();
446        let any_warm = eligible.iter().any(|(_, ms)| {
447            ms.iter().any(|m| {
448                self.growth
449                    .get(&m.resolution.cgroup)
450                    .is_some_and(|g| g.warm)
451            })
452        });
453        let any_cold = eligible.iter().any(|(_, ms)| {
454            ms.iter().any(|m| {
455                self.growth
456                    .get(&m.resolution.cgroup)
457                    .is_some_and(|g| !g.warm)
458            })
459        });
460        if !any_warm && any_cold && self.cold_defer_ticks < MAX_COLD_DEFER_TICKS {
461            self.cold_defer_ticks += 1;
462            return None;
463        }
464        self.cold_defer_ticks = 0;
465        let growth = |ms: &[&Target]| -> f64 {
466            ms.iter()
467                .map(|m| {
468                    self.growth
469                        .get(&m.resolution.cgroup)
470                        .map_or(0.0, |g| g.rate_bps.max(0.0))
471                })
472                .sum()
473        };
474        let size = |ms: &[&Target]| -> u64 {
475            ms.iter()
476                .map(|m| m.current_bytes.unwrap_or(m.rss_kb.saturating_mul(1024)))
477                .sum()
478        };
479        let pick = eligible
480            .iter()
481            .filter(|(_, ms)| growth(ms) >= MIN_GROWTH_BPS)
482            .max_by(|a, b| {
483                growth(&a.1)
484                    .total_cmp(&growth(&b.1))
485                    .then_with(|| b.0.cmp(a.0))
486            })
487            .or_else(|| {
488                eligible
489                    .iter()
490                    .max_by(|a, b| size(&a.1).cmp(&size(&b.1)).then_with(|| b.0.cmp(a.0)))
491            })?;
492        Some((pick.0.to_string(), pick.1.clone()))
493    }
494}
495
496#[cfg(test)]
497mod tests {
498    use super::super::resolve::Mechanism;
499    use super::super::types::PsiSource;
500    use super::*;
501
502    /// Default config = the documented zero-config defaults.
503    fn cfg() -> GuardConfig {
504        GuardConfig::default()
505    }
506
507    fn sample(some: f64, full: f64, avail_mb: u64) -> Sample {
508        Sample {
509            some_avg10: some,
510            full_avg10: full,
511            mem_available_mb: avail_mb,
512            mem_total_mb: 16_000,
513            source: PsiSource::AppSlice,
514        }
515    }
516
517    /// Build a default (unprotected, full-coverage) resolution for `cg`.
518    fn res(cg: &str) -> Resolution {
519        Resolution {
520            cgroup: cg.into(),
521            unit: Some(format!("{}.scope", cg.rsplit('/').next().unwrap())),
522            verdict: Verdict::Freeze,
523            coverage: Coverage::Full,
524            mechanism: Mechanism::Unit,
525        }
526    }
527
528    const MIB: u64 = 1024 * 1024;
529
530    /// Build a `Target` for app `app` in cgroup `cg`, with `mb` MB resident
531    /// and the same amount charged to the cgroup.
532    fn target(app: &str, cg: &str, mb: u64) -> Target {
533        Target {
534            app: app.into(),
535            resolution: res(cg),
536            rss_kb: mb * 1024,
537            current_bytes: Some(mb * MIB),
538        }
539    }
540
541    /// Shorthand: one app per scope, `/app.slice/app-<name>-<pid>.scope`.
542    fn proc(pid: u32, name: &str, rss_mb: u64) -> Target {
543        target(name, &format!("/app.slice/app-{name}-{pid}.scope"), rss_mb)
544    }
545
546    fn proc_at(_pid: u32, name: &str, rss_mb: u64, cg: &str) -> Target {
547        target(name, cg, rss_mb)
548    }
549
550    /// A comfortably-calm sample (no pressure, lots of memory).
551    fn calm() -> Sample {
552        sample(0.0, 0.0, 8000)
553    }
554
555    /// High PSI while only 12.5% of RAM is available: a real shortage.
556    fn high() -> Sample {
557        sample(50.0, 0.0, 2_000)
558    }
559
560    #[test]
561    fn high_pressure_with_plenty_of_free_memory_never_escalates() {
562        let mut e = PolicyEngine::new(cfg());
563        let procs = vec![proc(2, "chrome", 4000)];
564        // Half of RAM available: this stall is local to some memory.max, not a shortage.
565        let a = e.tick(0, sample(80.0, 20.0, 8_000), &procs, &live_from(&procs));
566        assert_eq!(e.level, Level::Critical);
567        assert!(
568            !a.iter()
569                .any(|x| matches!(x, Action::Freeze { .. } | Action::Cap { .. })),
570            "escalated without scarcity: {a:?}"
571        );
572    }
573
574    #[test]
575    fn is_scarce_uses_floor_or_percentage() {
576        let t = common::GuardTrigger::default(); // floor 400 MB, 20%
577        assert!(is_scarce(&sample(0.0, 0.0, 300), &t));
578        assert!(is_scarce(&sample(0.0, 0.0, 3_000), &t)); // 18.75%
579        assert!(!is_scarce(&sample(0.0, 0.0, 3_300), &t)); // 20.6%
580        let unknown = Sample {
581            mem_total_mb: 0,
582            mem_available_mb: u64::MAX,
583            ..sample(0.0, 0.0, 0)
584        };
585        assert!(
586            !is_scarce(&unknown, &t),
587            "unreadable meminfo must not enable actions"
588        );
589    }
590
591    fn freeze_targets(actions: &[Action]) -> Vec<String> {
592        actions
593            .iter()
594            .filter_map(|a| match a {
595                Action::Freeze { res, .. } => Some(res.cgroup.clone()),
596                _ => None,
597            })
598            .collect()
599    }
600
601    fn has_cap_target(actions: &[Action], cg: &str) -> bool {
602        actions
603            .iter()
604            .any(|a| matches!(a, Action::Cap { res, .. } if res.cgroup == cg))
605    }
606
607    fn has_freeze_target(actions: &[Action], cg: &str) -> bool {
608        actions
609            .iter()
610            .any(|a| matches!(a, Action::Freeze { res, .. } if res.cgroup == cg))
611    }
612
613    fn has_thaw_target(actions: &[Action], cg: &str) -> bool {
614        actions
615            .iter()
616            .any(|a| matches!(a, Action::Thaw { res } if res.cgroup == cg))
617    }
618
619    fn has_liftcap_target(actions: &[Action], cg: &str) -> bool {
620        actions
621            .iter()
622            .any(|a| matches!(a, Action::LiftCap { res } if res.cgroup == cg))
623    }
624
625    /// Give every target a measured (zero) growth rate, as if the engine had
626    /// already scanned them on an earlier tick. Tests of the ladder use this
627    /// so the cold-start deferral does not shift their first action.
628    fn prime(e: &mut PolicyEngine, ts: &[Target]) {
629        for t in ts {
630            e.growth.insert(
631                t.resolution.cgroup.clone(),
632                Growth {
633                    last_bytes: t.current_bytes.unwrap_or(0),
634                    last_ms: 0,
635                    rate_bps: 0.0,
636                    warm: true,
637                },
638            );
639        }
640    }
641
642    /// Default `live_cgroups` for tests that aren't specifically exercising
643    /// the liveness-vs-eligibility distinction: every target's cgroup.
644    fn live_from(ts: &[Target]) -> std::collections::HashSet<String> {
645        ts.iter().map(|t| t.resolution.cgroup.clone()).collect()
646    }
647
648    #[test]
649    fn rise_level_follows_the_engine_rise_rules() {
650        let t = common::GuardTrigger::default();
651        assert_eq!(rise_level(&sample(0.0, 0.0, 8000), &t), Level::Calm);
652        assert_eq!(rise_level(&sample(12.0, 0.0, 8000), &t), Level::Warn);
653        assert_eq!(rise_level(&sample(5.0, 4.0, 2000), &t), Level::High);
654        assert_eq!(rise_level(&sample(31.0, 0.0, 8000), &t), Level::High);
655        assert_eq!(rise_level(&sample(0.0, 10.0, 8000), &t), Level::Critical);
656        // Below the free-memory floor is Critical whatever PSI says.
657        assert_eq!(rise_level(&sample(0.0, 0.0, 300), &t), Level::Critical);
658        // An unreadable MemAvailable is never below the floor.
659        assert_eq!(rise_level(&sample(0.0, 0.0, u64::MAX), &t), Level::Calm);
660        // From Calm, one tick lands on the same level.
661        for s in [
662            sample(5.0, 4.0, 2000),
663            sample(0.0, 0.0, 300),
664            sample(12.0, 0.0, 8000),
665        ] {
666            let e = PolicyEngine::new(cfg());
667            assert_eq!(e.next_level(s), rise_level(&s, &t), "{s:?}");
668        }
669    }
670
671    #[test]
672    fn calm_yields_no_actions() {
673        let mut e = PolicyEngine::new(cfg());
674        let procs = vec![proc(100, "firefox", 2000)];
675        let actions = e.tick(1000, calm(), &procs, &live_from(&procs));
676        assert!(actions.is_empty(), "calm produced actions: {actions:?}");
677        assert_eq!(e.level, Level::Calm);
678    }
679
680    #[test]
681    fn full_signal_has_fall_hysteresis() {
682        // Enter High purely via PSI `full` (some stays low): full=4.0 >= FULL_HIGH_RISE(3.0).
683        let mut e = PolicyEngine::new(cfg());
684        let procs = vec![proc(100, "firefox", 4000)];
685        e.tick(1_000, sample(0.0, 4.0, 8000), &procs, &live_from(&procs));
686        assert_eq!(e.level, Level::High, "full=4.0 should enter High");
687
688        // full drifts to 2.0, between the fall (1.5) and rise (3.0) thresholds.
689        // With hysteresis it must HOLD High, not flap back to Calm.
690        e.tick(2_000, sample(0.0, 2.0, 8000), &procs, &live_from(&procs));
691        assert_eq!(
692            e.level,
693            Level::High,
694            "full=2.0 (between fall and rise) must hold High, not flap"
695        );
696
697        // full drops below the fall threshold (1.0 < 1.5): now it may step down.
698        e.tick(3_000, sample(0.0, 1.0, 8000), &procs, &live_from(&procs));
699        assert_eq!(
700            e.level,
701            Level::Calm,
702            "full below fall threshold drops to Calm"
703        );
704    }
705
706    #[test]
707    fn disabled_engine_is_inert() {
708        let mut c = cfg();
709        c.enabled = false;
710        let mut e = PolicyEngine::new(c);
711        let procs = vec![proc(100, "firefox", 4000)];
712        assert!(e.tick(1000, high(), &procs, &live_from(&procs)).is_empty());
713    }
714
715    #[test]
716    fn high_freezes_largest_eligible_process() {
717        let mut e = PolicyEngine::new(cfg());
718        let procs = vec![
719            proc(1, "small", 300),
720            proc(2, "biggest", 4000),
721            proc(3, "medium", 1000),
722        ];
723        prime(&mut e, &procs);
724        let actions = e.tick(1000, high(), &procs, &live_from(&procs));
725        // Only the single largest hog is frozen, not the smaller ones.
726        assert_eq!(
727            freeze_targets(&actions),
728            vec!["/app.slice/app-biggest-2.scope"]
729        );
730        assert_eq!(e.level, Level::High);
731    }
732
733    #[test]
734    fn process_below_min_rss_is_never_selected() {
735        let mut e = PolicyEngine::new(cfg());
736        // Both below the 200 MB default floor.
737        let procs = vec![proc(1, "tiny", 50), proc(2, "small", 150)];
738        let actions = e.tick(1000, high(), &procs, &live_from(&procs));
739        assert!(
740            freeze_targets(&actions).is_empty(),
741            "froze a sub-min-rss process: {actions:?}"
742        );
743    }
744
745    #[test]
746    fn frozen_process_thaws_after_freeze_hold() {
747        let mut e = PolicyEngine::new(cfg()); // freeze_hold = 5s
748        let procs = vec![proc(2, "hog", 4000)];
749        prime(&mut e, &procs);
750        let cg = "/app.slice/app-hog-2.scope";
751
752        let a0 = e.tick(0, high(), &procs, &live_from(&procs));
753        assert_eq!(freeze_targets(&a0), vec![cg]);
754
755        // Before the hold elapses: no thaw yet (and escalation gate keeps it quiet).
756        let a1 = e.tick(4_000, high(), &procs, &live_from(&procs));
757        assert!(!has_thaw_target(&a1, cg), "thawed too early: {a1:?}");
758
759        // At/after 5s the freeze auto-thaws.
760        let a2 = e.tick(5_000, high(), &procs, &live_from(&procs));
761        assert!(has_thaw_target(&a2, cg), "expected thaw at hold: {a2:?}");
762        assert!(e.interventions().is_empty());
763    }
764
765    #[test]
766    fn still_high_within_cooldown_caps_instead_of_refreezing() {
767        let mut e = PolicyEngine::new(cfg()); // hold=5s, cooldown=60s
768        let procs = vec![proc(2, "hog", 4000)];
769        prime(&mut e, &procs);
770        let cg = "/app.slice/app-hog-2.scope";
771
772        // Freeze at t=0.
773        assert_eq!(
774            freeze_targets(&e.tick(0, high(), &procs, &live_from(&procs))),
775            vec![cg]
776        );
777        // Auto-thaw at t=5s.
778        assert!(has_thaw_target(
779            &e.tick(5_000, high(), &procs, &live_from(&procs)),
780            cg
781        ));
782
783        // Still high, and within the 60s freeze cooldown: Cap, not re-Freeze.
784        // t must clear the escalation gate (>= last_action 5000 + 5000 hold).
785        let a = e.tick(10_000, high(), &procs, &live_from(&procs));
786        assert!(
787            has_cap_target(&a, cg),
788            "expected cap within cooldown: {a:?}"
789        );
790        assert!(freeze_targets(&a).is_empty(), "should not re-freeze: {a:?}");
791        assert!(matches!(
792            e.interventions().as_slice(),
793            [(c, Intervention::Capped { .. })] if c == cg
794        ));
795    }
796
797    #[test]
798    fn hysteresis_holds_level_between_fall_and_rise() {
799        let mut e = PolicyEngine::new(cfg());
800        let procs = vec![proc(2, "hog", 4000)];
801        let cg = "/app.slice/app-hog-2.scope";
802
803        // Rise to High.
804        e.tick(0, high(), &procs, &live_from(&procs));
805        assert_eq!(e.level, Level::High);
806
807        // some=20 is below rise(30) but above fall(15): stay High, no lift.
808        let a = e.tick(20_000, sample(20.0, 0.0, 8000), &procs, &live_from(&procs));
809        assert_eq!(e.level, Level::High, "dropped out of High prematurely");
810        // A thaw here is expected (the freeze hold elapsed), but the cap must not
811        // be lifted while we're still High.
812        assert!(
813            !has_liftcap_target(&a, cg),
814            "should not lift while still High: {a:?}"
815        );
816
817        // Drop below the fall threshold (some < 15 and full < 3): fall to Warn.
818        e.tick(21_000, sample(12.0, 0.0, 8000), &procs, &live_from(&procs));
819        assert_eq!(e.level, Level::Warn);
820    }
821
822    #[test]
823    fn capped_process_lifted_only_after_sustained_calm() {
824        let mut e = PolicyEngine::new(cfg()); // calm_hold = 30s
825        let procs = vec![proc(2, "hog", 4000)];
826        prime(&mut e, &procs);
827        let cg = "/app.slice/app-hog-2.scope";
828
829        // Drive a freeze, thaw, then a cap (still hot within cooldown).
830        e.tick(0, high(), &procs, &live_from(&procs));
831        e.tick(5_000, high(), &procs, &live_from(&procs)); // thaw
832        let a = e.tick(10_000, high(), &procs, &live_from(&procs)); // cap
833        assert!(has_cap_target(&a, cg));
834
835        // Calm starts at t=15s. Before 30s of calm: no lift.
836        let a1 = e.tick(15_000, calm(), &procs, &live_from(&procs));
837        assert!(
838            !has_liftcap_target(&a1, cg),
839            "lifted before calm sustained: {a1:?}"
840        );
841        let a2 = e.tick(44_000, calm(), &procs, &live_from(&procs)); // 29s of calm
842        assert!(
843            !has_liftcap_target(&a2, cg),
844            "lifted just before hold: {a2:?}"
845        );
846
847        // 30s of sustained calm lifts the cap.
848        let a3 = e.tick(45_000, calm(), &procs, &live_from(&procs));
849        assert!(
850            has_liftcap_target(&a3, cg),
851            "expected lift after calm hold: {a3:?}"
852        );
853        assert!(e.interventions().is_empty());
854    }
855
856    #[test]
857    fn cap_lift_resets_if_calm_is_interrupted() {
858        let mut e = PolicyEngine::new(cfg());
859        let procs = vec![proc(2, "hog", 4000)];
860        prime(&mut e, &procs);
861        let cg = "/app.slice/app-hog-2.scope";
862        e.tick(0, high(), &procs, &live_from(&procs));
863        e.tick(5_000, high(), &procs, &live_from(&procs));
864        assert!(has_cap_target(
865            &e.tick(10_000, high(), &procs, &live_from(&procs)),
866            cg
867        ));
868
869        e.tick(15_000, calm(), &procs, &live_from(&procs)); // calm clock starts
870        e.tick(20_000, high(), &procs, &live_from(&procs)); // pressure returns, calm clock cleared
871                                                            // New calm window starts at 25s; at 50s only 25s have passed, so no lift.
872        e.tick(25_000, calm(), &procs, &live_from(&procs));
873        let a = e.tick(50_000, calm(), &procs, &live_from(&procs));
874        assert!(
875            !has_liftcap_target(&a, cg),
876            "calm clock should have reset: {a:?}"
877        );
878    }
879
880    #[test]
881    fn escalation_gate_limits_to_one_freeze_per_hold_window() {
882        let mut e = PolicyEngine::new(cfg());
883        let procs = vec![proc(1, "hog-a", 4000), proc(2, "hog-b", 3000)];
884        prime(&mut e, &procs);
885        let cg_a = "/app.slice/app-hog-a-1.scope";
886        let cg_b = "/app.slice/app-hog-b-2.scope";
887
888        // First High tick freezes hog-a.
889        let a0 = e.tick(0, high(), &procs, &live_from(&procs));
890        assert_eq!(freeze_targets(&a0), vec![cg_a]);
891
892        // Second High tick within the 5s hold: gate closed, no new freeze.
893        let a1 = e.tick(2_000, high(), &procs, &live_from(&procs));
894        assert!(
895            freeze_targets(&a1).is_empty(),
896            "gate should suppress second freeze: {a1:?}"
897        );
898
899        // After the gate reopens, the next hog can be frozen.
900        let a2 = e.tick(5_000, high(), &procs, &live_from(&procs));
901        assert_eq!(freeze_targets(&a2), vec![cg_b]);
902    }
903
904    #[test]
905    fn dead_pid_is_pruned_with_liftcap() {
906        let mut e = PolicyEngine::new(cfg());
907        let procs = vec![proc(2, "hog", 4000)];
908        prime(&mut e, &procs);
909        let cg = "/app.slice/app-hog-2.scope";
910
911        // Freeze the hog's cgroup.
912        assert_eq!(
913            freeze_targets(&e.tick(0, high(), &procs, &live_from(&procs))),
914            vec![cg]
915        );
916        assert_eq!(e.interventions().len(), 1);
917
918        // Next tick the process (and its resolution) has vanished, so LiftCap
919        // cleanup, intervention dropped.
920        let a = e.tick(1_000, calm(), &[], &live_from(&[]));
921        assert!(
922            has_liftcap_target(&a, cg),
923            "expected LiftCap for dead cgroup: {a:?}"
924        );
925        assert!(e.interventions().is_empty());
926    }
927
928    /// D2 regression: a cgroup absent from `procs` (e.g. the cap evicted
929    /// enough file pages to drop the process below `min_rss_mb`) but still
930    /// present in `live_cgroups` must NOT be pruned: the cgroup is real and
931    /// alive, just not currently eligible for (re-)selection. A cgroup
932    /// absent from *both* sets must still be pruned with a LiftCap.
933    #[test]
934    fn intervention_survives_in_live_cgroups_but_absent_from_procs() {
935        let mut e = PolicyEngine::new(cfg());
936        let procs = vec![proc(2, "hog", 4000)];
937        prime(&mut e, &procs);
938        let cg = "/app.slice/app-hog-2.scope";
939
940        // Freeze the hog's cgroup.
941        assert_eq!(
942            freeze_targets(&e.tick(0, high(), &procs, &live_from(&procs))),
943            vec![cg]
944        );
945        assert_eq!(e.interventions().len(), 1);
946
947        // Next tick: the process no longer appears in `procs` (as if the cap
948        // evicted its file pages below the min-RSS floor), but its cgroup is
949        // still in `live_cgroups`, so it must NOT be pruned.
950        let live: std::collections::HashSet<String> = [cg.to_string()].into();
951        let a1 = e.tick(1_000, calm(), &[], &live);
952        assert!(
953            !has_liftcap_target(&a1, cg),
954            "must not prune a cgroup still present in live_cgroups: {a1:?}"
955        );
956        assert_eq!(
957            e.interventions().len(),
958            1,
959            "intervention must survive while the cgroup is live"
960        );
961
962        // Now the cgroup is gone from both sets entirely, so it is pruned.
963        let a2 = e.tick(2_000, calm(), &[], &std::collections::HashSet::new());
964        assert!(
965            has_liftcap_target(&a2, cg),
966            "expected LiftCap once absent from live_cgroups too: {a2:?}"
967        );
968        assert!(e.interventions().is_empty());
969    }
970
971    #[test]
972    fn interventions_reflect_state_sorted_by_cgroup() {
973        let mut e = PolicyEngine::new(cfg());
974        // pid 5 ("a") is the larger hog; pid 3 ("b") the smaller. We build a
975        // Capped "a" and a "b" capped after a freeze, both active, then check
976        // ordering + content. "/app.slice/app-a-5.scope" sorts before
977        // "/app.slice/app-b-3.scope" lexicographically.
978        let procs = vec![proc(5, "a", 4000), proc(3, "b", 3500)];
979        prime(&mut e, &procs);
980        let cg_a = "/app.slice/app-a-5.scope";
981        let cg_b = "/app.slice/app-b-3.scope";
982
983        // Walk both cgroups down the freeze, thaw, (still hot) cap ladder
984        // so two Capped interventions coexist. Caps persist while High (never
985        // auto-thaw), which is what lets two interventions overlap under
986        // default timing.
987        assert_eq!(
988            freeze_targets(&e.tick(0, high(), &procs, &live_from(&procs))),
989            vec![cg_a]
990        ); // freeze a
991        let a1 = e.tick(5_000, high(), &procs, &live_from(&procs)); // thaw a, freeze b
992        assert!(has_thaw_target(&a1, cg_a));
993        assert_eq!(freeze_targets(&a1), vec![cg_b]);
994        let a2 = e.tick(10_000, high(), &procs, &live_from(&procs)); // thaw b, cap a (in cooldown)
995        assert!(has_thaw_target(&a2, cg_b));
996        assert!(has_cap_target(&a2, cg_a));
997        let a3 = e.tick(15_000, high(), &procs, &live_from(&procs)); // cap b (in cooldown)
998        assert!(has_cap_target(&a3, cg_b));
999
1000        let ivs = e.interventions();
1001        assert_eq!(ivs.len(), 2, "expected a + b both Capped: {ivs:?}");
1002        // Sorted ascending by cgroup path.
1003        assert_eq!(ivs[0].0, cg_a);
1004        assert_eq!(ivs[1].0, cg_b);
1005        assert!(ivs
1006            .iter()
1007            .all(|(_, iv)| matches!(iv, Intervention::Capped { .. })));
1008    }
1009
1010    #[test]
1011    fn critical_via_mem_floor_triggers_action() {
1012        let mut e = PolicyEngine::new(cfg());
1013        let procs = vec![proc(2, "hog", 4000)];
1014        prime(&mut e, &procs);
1015        // No PSI pressure, but MemAvailable below the 400 MB floor, so Critical.
1016        let a = e.tick(0, sample(0.0, 0.0, 100), &procs, &live_from(&procs));
1017        assert_eq!(e.level, Level::Critical);
1018        assert_eq!(freeze_targets(&a), vec!["/app.slice/app-hog-2.scope"]);
1019    }
1020
1021    // ---- Task 5 new behaviors --------------------------------------------
1022
1023    #[test]
1024    fn caponly_verdict_never_freezes() {
1025        let mut e = PolicyEngine::new(cfg());
1026        let mut p = proc_at(2, "script", 4000, "/app.slice/app-alacritty-9.scope");
1027        let r = &mut p.resolution;
1028        r.verdict = Verdict::CapOnly;
1029        r.coverage = Coverage::Partial;
1030        let procs = [p];
1031        prime(&mut e, &procs);
1032        let a = e.tick(0, high(), &procs, &live_from(&procs));
1033        assert!(
1034            freeze_targets(&a).is_empty(),
1035            "CapOnly must not freeze: {a:?}"
1036        );
1037        assert!(has_cap_target(&a, "/app.slice/app-alacritty-9.scope"));
1038    }
1039
1040    #[test]
1041    fn partial_action_waits_at_least_three_seconds() {
1042        let mut e = PolicyEngine::new(cfg());
1043        let mut term = target("python3", "/app.slice/term.scope", 4000);
1044        term.resolution.verdict = Verdict::CapOnly;
1045        term.resolution.coverage = Coverage::Partial;
1046        let ts = vec![term, target("hog", "/app.slice/hog.scope", 3000)];
1047        prime(&mut e, &ts);
1048        assert!(has_cap_target(
1049            &e.tick(0, high(), &ts, &live_from(&ts)),
1050            "/app.slice/term.scope"
1051        ));
1052        assert!(freeze_targets(&e.tick(1_000, high(), &ts, &live_from(&ts))).is_empty());
1053        assert!(freeze_targets(&e.tick(2_000, high(), &ts, &live_from(&ts))).is_empty());
1054        assert!(has_freeze_target(
1055            &e.tick(3_000, high(), &ts, &live_from(&ts)),
1056            "/app.slice/hog.scope"
1057        ));
1058    }
1059
1060    #[test]
1061    fn two_scopes_of_one_app_are_acted_on_together() {
1062        let mut e = PolicyEngine::new(cfg());
1063        let ts = vec![
1064            target("chrome", "/app.slice/app-chrome-1.scope", 1500),
1065            target("chrome", "/app.slice/app-chrome-2.scope", 900),
1066            target("hog", "/app.slice/app-hog-3.scope", 2000),
1067        ];
1068        prime(&mut e, &ts);
1069        let mut frozen = freeze_targets(&e.tick(0, high(), &ts, &live_from(&ts)));
1070        frozen.sort();
1071        assert_eq!(
1072            frozen,
1073            vec![
1074                "/app.slice/app-chrome-1.scope",
1075                "/app.slice/app-chrome-2.scope"
1076            ]
1077        );
1078        let a1 = e.tick(1_000, high(), &ts, &live_from(&ts));
1079        assert!(
1080            freeze_targets(&a1).is_empty(),
1081            "one app is one escalation step: {a1:?}"
1082        );
1083    }
1084
1085    #[test]
1086    fn fastest_growing_app_is_chosen_over_largest() {
1087        let mut e = PolicyEngine::new(cfg());
1088        let warn = sample(12.0, 0.0, 2_000);
1089        let t0 = vec![
1090            target("firefox", "/app.slice/ff.scope", 4000),
1091            target("script", "/app.slice/sh.scope", 1000),
1092        ];
1093        assert!(freeze_targets(&e.tick(0, warn, &t0, &live_from(&t0))).is_empty());
1094        let t1 = vec![
1095            target("firefox", "/app.slice/ff.scope", 4000),
1096            target("script", "/app.slice/sh.scope", 1600),
1097        ];
1098        assert_eq!(
1099            freeze_targets(&e.tick(1_000, high(), &t1, &live_from(&t1))),
1100            vec!["/app.slice/sh.scope"]
1101        );
1102    }
1103
1104    #[test]
1105    fn largest_app_is_the_fallback_when_nothing_grows() {
1106        let mut e = PolicyEngine::new(cfg());
1107        let ts = vec![
1108            target("small", "/app.slice/s.scope", 300),
1109            target("big", "/app.slice/b.scope", 3000),
1110        ];
1111        e.tick(0, sample(12.0, 0.0, 2_000), &ts, &live_from(&ts));
1112        assert_eq!(
1113            freeze_targets(&e.tick(1_000, high(), &ts, &live_from(&ts))),
1114            vec!["/app.slice/b.scope"]
1115        );
1116    }
1117
1118    #[test]
1119    fn at_most_three_apps_are_held_at_once() {
1120        let mut e = PolicyEngine::new(cfg());
1121        let ts: Vec<Target> = (0..5u64)
1122            .map(|i| {
1123                target(
1124                    &format!("app{i}"),
1125                    &format!("/app.slice/a{i}.scope"),
1126                    1000 + i * 100,
1127                )
1128            })
1129            .collect();
1130        for step in 0..40u64 {
1131            e.tick(step * 5_000, high(), &ts, &live_from(&ts));
1132            assert!(
1133                e.interventions().len() <= MAX_HELD_APPS,
1134                "held {:?}",
1135                e.interventions()
1136            );
1137        }
1138    }
1139
1140    #[test]
1141    fn held_limit_counts_apps_not_cgroups() {
1142        let mut e = PolicyEngine::new(cfg());
1143        let ts = vec![
1144            target("a", "/app.slice/a1.scope", 2000),
1145            target("a", "/app.slice/a2.scope", 2000),
1146            target("b", "/app.slice/b.scope", 1500),
1147            target("c", "/app.slice/c.scope", 1200),
1148            target("d", "/app.slice/d.scope", 1100),
1149        ];
1150        for step in 0..40u64 {
1151            e.tick(step * 5_000, high(), &ts, &live_from(&ts));
1152        }
1153        let held: Vec<String> = e.interventions().into_iter().map(|(cg, _)| cg).collect();
1154        assert_eq!(
1155            held,
1156            vec![
1157                "/app.slice/a1.scope",
1158                "/app.slice/a2.scope",
1159                "/app.slice/b.scope",
1160                "/app.slice/c.scope"
1161            ],
1162            "4 cgroups across 3 apps are allowed, a 4th app is refused"
1163        );
1164    }
1165
1166    #[test]
1167    fn candidates_are_wanted_above_calm_or_when_scarce() {
1168        let e = PolicyEngine::new(cfg());
1169        assert!(!e.wants_candidates(calm()));
1170        assert!(e.wants_candidates(sample(12.0, 0.0, 8_000)));
1171        // Calm PSI but under 20% of RAM available: scan so growth stays warm.
1172        assert!(e.wants_candidates(sample(0.0, 0.0, 3_000)));
1173    }
1174
1175    /// Regression (final review I1): available memory drops below the floor
1176    /// in one step with no stall first, so the Critical tick is the first
1177    /// scan. An idle 6 GB app must not be frozen in place of a 3 GB app
1178    /// that is growing: the first tick defers, the next picks the grower.
1179    #[test]
1180    fn cold_start_defers_then_picks_grower_not_largest() {
1181        let mut e = PolicyEngine::new(cfg());
1182        for step in 0..5u64 {
1183            e.tick(step * 1_000, calm(), &[], &HashSet::new());
1184        }
1185        let crit = sample(0.0, 0.0, 300);
1186        let t0 = vec![
1187            target("chrome", "/app.slice/chrome.scope", 6000),
1188            target("hog", "/app.slice/hog.scope", 3000),
1189        ];
1190        let a0 = e.tick(5_000, crit, &t0, &live_from(&t0));
1191        assert_eq!(e.level, Level::Critical);
1192        assert!(
1193            freeze_targets(&a0).is_empty(),
1194            "no growth data yet, must defer: {a0:?}"
1195        );
1196        let t1 = vec![
1197            target("chrome", "/app.slice/chrome.scope", 6000),
1198            target("hog", "/app.slice/hog.scope", 3200),
1199        ];
1200        assert_eq!(
1201            freeze_targets(&e.tick(6_000, crit, &t1, &live_from(&t1))),
1202            vec!["/app.slice/hog.scope"]
1203        );
1204    }
1205
1206    /// With scarce-but-calm ticks feeding growth, the grower is picked on
1207    /// the very first Critical tick.
1208    #[test]
1209    fn scarce_calm_ticks_warm_growth_for_first_critical_tick() {
1210        let mut e = PolicyEngine::new(cfg());
1211        let scarce_calm = sample(0.0, 0.0, 3_000);
1212        assert!(e.wants_candidates(scarce_calm));
1213        let t0 = vec![
1214            target("chrome", "/app.slice/chrome.scope", 6000),
1215            target("hog", "/app.slice/hog.scope", 2000),
1216        ];
1217        assert!(freeze_targets(&e.tick(0, scarce_calm, &t0, &live_from(&t0))).is_empty());
1218        let t1 = vec![
1219            target("chrome", "/app.slice/chrome.scope", 6000),
1220            target("hog", "/app.slice/hog.scope", 3000),
1221        ];
1222        assert_eq!(
1223            freeze_targets(&e.tick(1_000, sample(0.0, 0.0, 300), &t1, &live_from(&t1))),
1224            vec!["/app.slice/hog.scope"]
1225        );
1226    }
1227
1228    /// A new cgroup on every tick never gets a measured growth rate. The
1229    /// deferral is bounded, so the guard still acts by the fourth tick and
1230    /// falls back to the largest eligible app.
1231    #[test]
1232    fn cold_start_deferral_is_bounded_when_cgroups_keep_changing() {
1233        let mut e = PolicyEngine::new(cfg());
1234        let crit = sample(0.0, 0.0, 300);
1235        for tick in 0..4u64 {
1236            let ts = vec![
1237                target(
1238                    &format!("big{tick}"),
1239                    &format!("/app.slice/b{tick}.scope"),
1240                    3000,
1241                ),
1242                target(
1243                    &format!("small{tick}"),
1244                    &format!("/app.slice/s{tick}.scope"),
1245                    1000,
1246                ),
1247            ];
1248            let frozen = freeze_targets(&e.tick(tick * 1_000, crit, &ts, &live_from(&ts)));
1249            if tick < u64::from(MAX_COLD_DEFER_TICKS) {
1250                assert!(frozen.is_empty(), "tick {tick} should defer: {frozen:?}");
1251            } else {
1252                assert_eq!(frozen, vec![format!("/app.slice/b{tick}.scope")]);
1253            }
1254        }
1255    }
1256
1257    /// Without any memory.current reading, growth can never be measured, so
1258    /// the guard does not wait and falls back to the largest app.
1259    #[test]
1260    fn no_current_bytes_does_not_defer() {
1261        let mut e = PolicyEngine::new(cfg());
1262        let mut big = target("big", "/app.slice/b.scope", 3000);
1263        big.current_bytes = None;
1264        let mut small = target("small", "/app.slice/s.scope", 1000);
1265        small.current_bytes = None;
1266        let ts = vec![big, small];
1267        assert_eq!(
1268            freeze_targets(&e.tick(0, high(), &ts, &live_from(&ts))),
1269            vec!["/app.slice/b.scope"]
1270        );
1271    }
1272}