car-inference 0.56.1

Local model inference for CAR — Candle backend with Qwen3 models
Documentation
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
//! Autonomous local-model maintenance — what the concierge does, unasked, to
//! keep a machine's local models right.
//!
//! Pure over its inputs, like [`crate::portfolio`]: the engine gathers the
//! portfolio, the disk numbers, the user's "keep" choices and the retirement
//! history, and this module decides. Nothing here reads a clock or the disk.
//!
//! Rules:
//! - [`UpdatePolicy::Off`] does nothing at all. `Notify` decides and
//!   suggests but never acts; `Auto` acts.
//! - At most one retirement per pass. Every pass re-plans, and execution
//!   refuses a plan that moved (`expect`), so one at a time is never stale.
//! - `never_runs_here` and `cannot_load` act whenever recommended: they do
//!   nothing for anyone.
//! - `superseded` acts after [`MaintenanceContext::superseded_unpressured_secs`]
//!   of silence, or sooner under disk pressure; `idle` acts only under disk
//!   pressure. Space is the reason to delete something someone might still
//!   reach for.
//! - Disk pressure has hysteresis: it starts below the low-water mark and
//!   holds until free space is back above the high-water mark, so a machine
//!   sitting at the line does not retire one model a week forever.
//! - A model the user said to keep (a `retire:<model id>` dismissal) is never
//!   acted on; nor, for anything short of "does nothing here", is one CAR
//!   retired before and the user then fetched back — that model is used, the
//!   evidence just did not show it, and retiring it again is a download loop.

use serde::Serialize;

use crate::portfolio::{Assessment, Portfolio, PortfolioAction, Verdict};
use crate::update_prefs::UpdatePolicy;

/// The models the user asked to keep at `now`: a `retire:<id>` dismissal with
/// a permanent reason keeps a model for good; `not_now` keeps it 30 days.
pub fn kept_models(
    nudge: &crate::nudge::NudgeState,
    now: u64,
) -> std::collections::BTreeSet<String> {
    nudge
        .concierge_dismissals
        .iter()
        .filter(|d| d.reason.is_permanent() || now.saturating_sub(d.timestamp) < 30 * 24 * 60 * 60)
        .filter_map(|d| d.key.strip_prefix("retire:").map(str::to_owned))
        .collect()
}

/// The dismissal key that tells the concierge to keep `model_id`.
pub fn keep_key(model_id: &str) -> String {
    format!("retire:{model_id}")
}

/// Free space on the volume(s) holding the weights.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
pub struct DiskSpace {
    pub free_bytes: u64,
    pub total_bytes: u64,
}

impl DiskSpace {
    fn below(&self, percent: u64, floor_bytes: u64) -> bool {
        let by_share =
            self.free_bytes.saturating_mul(100) < self.total_bytes.saturating_mul(percent);
        by_share || self.free_bytes < floor_bytes
    }
}

/// Low-water: pressure starts when free space is under 15% or 20 GB.
pub const PRESSURE_LOW_PERCENT: u64 = 15;
pub const PRESSURE_LOW_BYTES: u64 = 20 * 1_000_000_000;
/// High-water: pressure holds until free space is over 25% and 40 GB.
pub const PRESSURE_HIGH_PERCENT: u64 = 25;
pub const PRESSURE_HIGH_BYTES: u64 = 40 * 1_000_000_000;

/// Whether the disk is under pressure now, given whether it was last pass.
pub fn disk_pressure(disk: Option<DiskSpace>, was_pressured: bool) -> bool {
    let Some(disk) = disk else {
        // Unknown space is no reason to delete anything.
        return false;
    };
    if was_pressured {
        disk.below(PRESSURE_HIGH_PERCENT, PRESSURE_HIGH_BYTES)
    } else {
        disk.below(PRESSURE_LOW_PERCENT, PRESSURE_LOW_BYTES)
    }
}

#[derive(Debug, Clone, PartialEq, Eq)]
pub struct MaintenanceContext {
    pub policy: UpdatePolicy,
    pub pressured: bool,
    /// Models the user asked to keep.
    pub keep: std::collections::BTreeSet<String>,
    /// Models CAR retired that were fetched back afterwards.
    pub churned: std::collections::BTreeSet<String>,
    /// How long a superseded model must be silent to go with no pressure.
    pub superseded_unpressured_secs: u64,
}

impl MaintenanceContext {
    pub const DEFAULT_SUPERSEDED_UNPRESSURED_SECS: u64 = 30 * 24 * 60 * 60;
}

/// What a pass decided for one model.
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
pub struct Decision {
    pub model_id: String,
    pub verdict: Verdict,
    pub freed_bytes: u64,
    /// The plan digest to execute exactly what was assessed.
    pub digest: String,
    pub reason: String,
}

/// While the unchosen default `Auto` is still in its first `window` since
/// `since`, the end of that window: the pass suggests and does not act.
pub fn auto_grace_until(since: u64, now: u64, window: u64) -> Option<u64> {
    let until = since.saturating_add(window);
    (now < until).then_some(until)
}

/// The least RAM on which the concierge will move a machine to the
/// Local-focused memory policy: below it, 80% for models leaves too little
/// for everything else.
pub const AUTO_LOCAL_FOCUSED_MIN_RAM_MB: u64 = 32 * 1024;
/// Memory refusals, observed for local models that Local-focused would
/// admit, before the concierge raises the policy — on at least
/// [`AUTO_LOCAL_FOCUSED_MIN_DAYS`] distinct days, so one request retried
/// three times is one occasion, not three.
pub const AUTO_LOCAL_FOCUSED_MIN_REFUSALS: u64 = 3;
pub const AUTO_LOCAL_FOCUSED_MIN_DAYS: u64 = 2;
/// Live-memory refusals of models the old policy admitted, after the
/// concierge raised it, that make it undo the raise.
pub const AUTO_REVERT_LIVE_REFUSALS: u64 = 3;

/// A memory-policy change a pass made or would make.
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
pub struct PolicySizing {
    pub from: crate::resource_policy::ResourceProfile,
    pub to: crate::resource_policy::ResourceProfile,
    pub reason: String,
    pub applied: bool,
}

/// What the concierge should do to the memory policy.
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum PolicyMove {
    /// Everyday → Local-focused.
    Raise(String),
    /// Undo the concierge's own raise: Local-focused → Everyday.
    Revert(String),
}

/// The evidence a memory-policy decision reads.
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub struct PolicyEvidence {
    /// Refusals of local models too big for the current policy that
    /// Local-focused admits (the ceiling, not live memory, refused them).
    pub ceiling_refusals: u64,
    /// The distinct days those refusals fell on.
    pub ceiling_refusal_days: u64,
    /// Since the concierge raised the policy: refusals of models the old
    /// policy admitted — live memory ran short with the bigger allocation.
    pub live_refusals_since_raise: u64,
    /// Since the raise, for models only the raise admits: failures the
    /// model was blamed for (an in-process allocation that would not fit),
    /// and successes.
    pub admitted_failures_since_raise: u64,
    pub admitted_successes_since_raise: u64,
}

/// Decide a memory-policy move. Raises only a policy nobody chose, only
/// Everyday to Local-focused, only on a machine with room, only on ceiling
/// refusals seen on more than one day, and never after it once undid a
/// raise. Undoes only its own raise, when live memory then ran short.
/// Never touches a policy the user chose.
pub fn decide_resource_policy(
    current: &crate::resource_policy::ResourcePolicy,
    user_chose: bool,
    concierge: Option<&crate::resource_policy::ConciergeChoice>,
    total_ram_mb: u64,
    evidence: &PolicyEvidence,
) -> Option<PolicyMove> {
    use crate::resource_policy::ResourceProfile;
    if user_chose {
        return None;
    }
    let raised_by_concierge = concierge
        .is_some_and(|c| !c.reverted && c.policy.profile == ResourceProfile::LocalFocused)
        && current.profile == ResourceProfile::LocalFocused;
    if raised_by_concierge {
        if evidence.admitted_failures_since_raise >= AUTO_REVERT_LIVE_REFUSALS
            && evidence.admitted_successes_since_raise == 0
        {
            return Some(PolicyMove::Revert(format!(
                "the models only the raise admits failed {} times and never succeeded since",
                evidence.admitted_failures_since_raise
            )));
        }
        return (evidence.live_refusals_since_raise >= AUTO_REVERT_LIVE_REFUSALS).then(|| {
            PolicyMove::Revert(format!(
                "{} loads of models the Everyday policy admits were refused for live memory \
                 after the raise (the larger allocation may not be the cause; going back is the \
                 safe direction)",
                evidence.live_refusals_since_raise
            ))
        });
    }
    if current.profile != ResourceProfile::Everyday
        || concierge.is_some_and(|c| c.reverted)
        || total_ram_mb < AUTO_LOCAL_FOCUSED_MIN_RAM_MB
        || evidence.ceiling_refusals < AUTO_LOCAL_FOCUSED_MIN_REFUSALS
        || evidence.ceiling_refusal_days < AUTO_LOCAL_FOCUSED_MIN_DAYS
    {
        return None;
    }
    Some(PolicyMove::Raise(format!(
        "{} loads of local models too big for the Everyday policy, that Local-focused admits, \
         were refused on {} days, on a machine with {} GB",
        evidence.ceiling_refusals,
        evidence.ceiling_refusal_days,
        total_ram_mb / 1024
    )))
}

/// A recommended retirement a pass held back.
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
pub struct Held {
    pub model_id: String,
    pub why: String,
}

/// A retirement a maintenance pass carried out.
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
pub struct Retired {
    pub model_id: String,
    /// Rows that went with it (they shared its repo).
    pub also_retired: Vec<String>,
    pub freed_bytes: u64,
    /// How to get it back: it is re-fetched on next use anyway.
    pub restore: String,
}

/// A curated upgrade a maintenance pass applied.
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
pub struct Upgraded {
    pub from: String,
    pub to: String,
    /// Lanes switched from `from` to `to`, each watched by the canary that
    /// reverts a regression. Empty when `to` was only installed beside a
    /// `from` in use outside any lane default.
    pub lanes: Vec<String>,
}

/// An orphaned hub repo a maintenance pass deleted.
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
pub struct RetiredOrphan {
    pub repo: String,
    pub freed_bytes: u64,
}

/// What one maintenance pass saw, decided and did.
#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize)]
pub struct MaintenanceReport {
    pub policy: UpdatePolicy,
    pub dry_run: bool,
    pub disk: Option<DiskSpace>,
    pub pressured: bool,
    pub plan: MaintenancePlan,
    #[serde(skip_serializing_if = "Option::is_none")]
    pub retired: Option<Retired>,
    /// A hub repo CAR fetched whole that no model uses any more, retired
    /// under disk pressure when no model was.
    #[serde(skip_serializing_if = "Option::is_none")]
    pub orphan_retired: Option<RetiredOrphan>,
    /// Abandoned partial downloads deleted (no writer holds them, untouched
    /// for a day), in repos CAR owns.
    pub partials_discarded: Vec<std::path::PathBuf>,
    /// The portfolio the pass decided on, for a caller that also wants it
    /// (`concierge.ask`) without assessing everything twice.
    #[serde(skip)]
    pub portfolio: Option<crate::portfolio::Portfolio>,
    /// A curated upgrade the pass applied (`Auto` only, one per pass).
    #[serde(skip_serializing_if = "Option::is_none")]
    pub upgraded: Option<Upgraded>,
    /// Until when this machine runs as `Notify` although its policy is the
    /// unchosen default `Auto`: the first week, before CAR deletes anything
    /// unasked, so a user who wanted `notify` can still say so.
    #[serde(skip_serializing_if = "Option::is_none")]
    pub auto_grace_until: Option<u64>,
    /// The memory-policy change made (`Auto`) or suggested (`Notify`).
    #[serde(skip_serializing_if = "Option::is_none")]
    pub resource_policy: Option<PolicySizing>,
    pub errors: Vec<String>,
}

impl MaintenanceReport {
    /// Whether the pass changed anything on disk.
    pub fn acted(&self) -> bool {
        self.retired.is_some()
            || self.upgraded.is_some()
            || self.orphan_retired.is_some()
            || !self.partials_discarded.is_empty()
            || self.resource_policy.as_ref().is_some_and(|p| p.applied)
    }
}

#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize)]
pub struct MaintenancePlan {
    /// The one retirement to execute now (`Auto` only).
    #[serde(skip_serializing_if = "Option::is_none")]
    pub act: Option<Decision>,
    /// Retirements worth doing that this pass will not execute: everything
    /// under `Notify`, the rest of the queue under `Auto`.
    pub suggest: Vec<Decision>,
    /// Recommended retirements held back, and why.
    pub held: Vec<Held>,
}

fn days(secs: u64) -> u64 {
    secs / (24 * 60 * 60)
}

/// Decide this pass. Candidates are ordered "does nothing here" first, then
/// superseded, then idle by bytes freed, so under pressure the least useful
/// bytes go first.
pub fn decide(portfolio: &Portfolio, ctx: &MaintenanceContext) -> MaintenancePlan {
    let mut plan = MaintenancePlan::default();
    if ctx.policy == UpdatePolicy::Off {
        return plan;
    }
    let now = portfolio.policy.now;
    let rank = |a: &Assessment| match a.verdict {
        Verdict::NeverRunsHere | Verdict::CannotLoad => 0,
        Verdict::Superseded => 1,
        _ => 2,
    };
    let mut candidates: Vec<&Assessment> = portfolio
        .models
        .iter()
        .filter(|a| a.action == PortfolioAction::Retire && a.retire.is_some())
        .collect();
    candidates.sort_by(|a, b| {
        rank(a).cmp(&rank(b)).then_with(|| {
            let freed = |x: &Assessment| x.retire.as_ref().map_or(0, |r| r.freed_bytes);
            freed(b).cmp(&freed(a))
        })
    });
    for assessment in candidates {
        let id = &assessment.model_id;
        let does_nothing = matches!(
            assessment.verdict,
            Verdict::NeverRunsHere | Verdict::CannotLoad
        );
        if ctx.keep.contains(id) {
            plan.held.push(Held {
                model_id: id.clone(),
                why: "you asked to keep it".into(),
            });
            continue;
        }
        if !does_nothing && ctx.churned.contains(id) {
            plan.held.push(Held {
                model_id: id.clone(),
                why: "it was retired before and fetched back, so it is in use".into(),
            });
            continue;
        }
        // Silence as observed: from the last use, floored at when usage
        // became observable (a coverage restart means nothing before it was
        // seen). No observable window, no silence.
        let silent = portfolio
            .policy
            .tracking_since
            .map(|since| now.saturating_sub(assessment.last_used.unwrap_or(0).max(since)));
        let allowed = match assessment.verdict {
            Verdict::NeverRunsHere | Verdict::CannotLoad => true,
            Verdict::Superseded => {
                ctx.pressured || silent.is_some_and(|s| s >= ctx.superseded_unpressured_secs)
            }
            Verdict::Idle => ctx.pressured,
            _ => false,
        };
        if !allowed {
            plan.held.push(Held {
                model_id: id.clone(),
                why: match assessment.verdict {
                    Verdict::Superseded => format!(
                        "replaced, but space is not short and it was used within {} days",
                        days(ctx.superseded_unpressured_secs)
                    ),
                    _ => "unused, but space is not short".into(),
                },
            });
            continue;
        }
        let retire = assessment.retire.as_ref().expect("filtered above");
        // Busy now (`in_use`, `downloading`): suggested, never acted on, so
        // one resident model cannot stall everything queued behind it.
        let busy = !retire.refusals.is_empty();
        let decision = Decision {
            model_id: id.clone(),
            verdict: assessment.verdict,
            freed_bytes: retire.freed_bytes,
            digest: retire.digest.clone(),
            reason: assessment.evidence.join("; "),
        };
        if ctx.policy == UpdatePolicy::Auto && plan.act.is_none() && !busy {
            plan.act = Some(decision);
        } else {
            plan.suggest.push(decision);
        }
    }
    plan
}

#[cfg(test)]
mod tests {
    use super::*;
    use crate::portfolio::{PortfolioPolicy, RetireSummary};

    const DAY: u64 = 24 * 60 * 60;
    const NOW: u64 = 1_000 * DAY;
    const GB: u64 = 1_000_000_000;

    fn assessment(id: &str, verdict: Verdict, freed: u64, last_used: Option<u64>) -> Assessment {
        Assessment {
            model_id: id.into(),
            name: id.into(),
            verdict,
            action: PortfolioAction::Retire,
            evidence: vec![format!("{verdict:?}")],
            kept: false,
            last_used,
            retire: Some(RetireSummary {
                freed_bytes: freed,
                digest: format!("digest-{id}"),
                refusals: Vec::new(),
                also_retires: Vec::new(),
            }),
        }
    }

    fn portfolio(models: Vec<Assessment>) -> Portfolio {
        Portfolio {
            policy: PortfolioPolicy::new(NOW, Some(0)),
            models,
            reclaimable_bytes: 0,
            orphans: Vec::new(),
            recent_actions: Vec::new(),
            revoked_pins: Vec::new(),
            revoked_copies: Vec::new(),
        }
    }

    fn ctx(policy: UpdatePolicy, pressured: bool) -> MaintenanceContext {
        MaintenanceContext {
            policy,
            pressured,
            keep: Default::default(),
            churned: Default::default(),
            superseded_unpressured_secs: MaintenanceContext::DEFAULT_SUPERSEDED_UNPRESSURED_SECS,
        }
    }

    #[test]
    fn off_does_nothing_and_notify_never_acts() {
        let p = portfolio(vec![assessment("a", Verdict::NeverRunsHere, GB, None)]);
        assert_eq!(
            decide(&p, &ctx(UpdatePolicy::Off, true)),
            MaintenancePlan::default()
        );
        let notify = decide(&p, &ctx(UpdatePolicy::Notify, true));
        assert!(notify.act.is_none());
        assert_eq!(notify.suggest.len(), 1);
    }

    #[test]
    fn useless_models_go_first_and_idle_waits_for_pressure() {
        let p = portfolio(vec![
            assessment("idle", Verdict::Idle, 9 * GB, Some(NOW - 40 * DAY)),
            assessment("gguf", Verdict::NeverRunsHere, GB, None),
        ]);
        let calm = decide(&p, &ctx(UpdatePolicy::Auto, false));
        assert_eq!(calm.act.as_ref().map(|d| d.model_id.as_str()), Some("gguf"));
        assert!(calm.suggest.is_empty());
        assert!(calm.held.iter().any(|h| h.model_id == "idle"));

        let pressed = decide(&p, &ctx(UpdatePolicy::Auto, true));
        assert_eq!(
            pressed.act.as_ref().map(|d| d.model_id.as_str()),
            Some("gguf")
        );
        assert_eq!(pressed.suggest.len(), 1, "one per pass; the rest queue");
    }

    #[test]
    fn superseded_goes_after_a_month_or_under_pressure() {
        let recent = assessment("old", Verdict::Superseded, GB, Some(NOW - 10 * DAY));
        let p = portfolio(vec![recent.clone()]);
        assert!(decide(&p, &ctx(UpdatePolicy::Auto, false)).act.is_none());
        assert!(decide(&p, &ctx(UpdatePolicy::Auto, true)).act.is_some());
        let stale = assessment("old", Verdict::Superseded, GB, Some(NOW - 31 * DAY));
        assert!(
            decide(&portfolio(vec![stale]), &ctx(UpdatePolicy::Auto, false))
                .act
                .is_some()
        );
    }

    /// An old stamp does not make a month of silence when usage only became
    /// observable a week ago (a daemon that did not stamp ran in between).
    #[test]
    fn superseded_silence_counts_only_observed_time() {
        let mut p = portfolio(vec![assessment(
            "old",
            Verdict::Superseded,
            GB,
            Some(NOW - 40 * DAY),
        )]);
        p.policy.tracking_since = Some(NOW - 8 * DAY);
        assert!(decide(&p, &ctx(UpdatePolicy::Auto, false)).act.is_none());
        p.policy.tracking_since = Some(NOW - 35 * DAY);
        assert!(decide(&p, &ctx(UpdatePolicy::Auto, false)).act.is_some());
        // Never used: still a full month of observed silence first.
        let mut never = portfolio(vec![assessment("old", Verdict::Superseded, GB, None)]);
        never.policy.tracking_since = Some(NOW - 10 * DAY);
        assert!(decide(&never, &ctx(UpdatePolicy::Auto, false))
            .act
            .is_none());
    }

    #[test]
    fn a_busy_candidate_does_not_stall_the_queue() {
        let mut busy = assessment("busy", Verdict::NeverRunsHere, 9 * GB, None);
        busy.retire.as_mut().unwrap().refusals = vec!["in_use".into()];
        let p = portfolio(vec![
            busy,
            assessment("next", Verdict::CannotLoad, GB, None),
        ]);
        let plan = decide(&p, &ctx(UpdatePolicy::Auto, false));
        assert_eq!(plan.act.as_ref().map(|d| d.model_id.as_str()), Some("next"));
        assert!(plan.suggest.iter().any(|d| d.model_id == "busy"));
    }

    #[test]
    fn keep_and_churn_hold_a_model_back() {
        let p = portfolio(vec![
            assessment("kept", Verdict::Idle, GB, None),
            assessment("loop", Verdict::Idle, GB, None),
            assessment("dead", Verdict::CannotLoad, GB, None),
        ]);
        let mut c = ctx(UpdatePolicy::Auto, true);
        c.keep.insert("kept".into());
        c.churned.insert("loop".into());
        c.churned.insert("dead".into());
        let plan = decide(&p, &c);
        // Churn does not save a model that cannot run at all.
        assert_eq!(plan.act.as_ref().map(|d| d.model_id.as_str()), Some("dead"));
        assert!(plan.suggest.is_empty(), "{plan:?}");
        assert_eq!(plan.held.len(), 2);
    }

    #[test]
    fn the_memory_policy_rises_only_when_unchosen_roomy_and_refused() {
        use crate::resource_policy::{ConciergeChoice, ResourcePolicy};
        let everyday = ResourcePolicy::everyday();
        let ram = 64 * 1024;
        let seen = PolicyEvidence {
            ceiling_refusals: 3,
            ceiling_refusal_days: 2,
            ..Default::default()
        };
        let raise = |chose, ram, ev: &PolicyEvidence| {
            decide_resource_policy(&everyday, chose, None, ram, ev)
        };
        assert!(matches!(
            raise(false, ram, &seen),
            Some(PolicyMove::Raise(_))
        ));
        assert!(raise(true, ram, &seen).is_none(), "user chose");
        assert!(raise(false, 16 * 1024, &seen).is_none(), "small machine");
        let one_day = PolicyEvidence {
            ceiling_refusal_days: 1,
            ..seen.clone()
        };
        assert!(
            raise(false, ram, &one_day).is_none(),
            "one occasion retried"
        );
        let reverted = ConciergeChoice {
            policy: everyday.clone(),
            at: 1,
            reverted: true,
        };
        assert!(
            decide_resource_policy(&everyday, false, Some(&reverted), ram, &seen).is_none(),
            "never raises again after undoing a raise"
        );
        // Its own raise, then live memory ran short: undone. A user's
        // Local-focused is never touched.
        let local = ResourcePolicy::local_focused();
        let raised = ConciergeChoice {
            policy: local.clone(),
            at: 1,
            reverted: false,
        };
        let pressed = PolicyEvidence {
            live_refusals_since_raise: 3,
            ..Default::default()
        };
        assert!(matches!(
            decide_resource_policy(&local, false, Some(&raised), ram, &pressed),
            Some(PolicyMove::Revert(_))
        ));
        assert!(decide_resource_policy(&local, true, None, ram, &pressed).is_none());
        let failing = PolicyEvidence {
            admitted_failures_since_raise: 3,
            ..Default::default()
        };
        assert!(matches!(
            decide_resource_policy(&local, false, Some(&raised), ram, &failing),
            Some(PolicyMove::Revert(_))
        ));
        let mixed = PolicyEvidence {
            admitted_failures_since_raise: 3,
            admitted_successes_since_raise: 1,
            ..Default::default()
        };
        assert!(decide_resource_policy(&local, false, Some(&raised), ram, &mixed).is_none());
        assert!(decide_resource_policy(&local, false, Some(&raised), ram, &seen).is_none());
    }

    #[test]
    fn the_unchosen_default_waits_a_week_before_acting() {
        assert_eq!(auto_grace_until(NOW, NOW, 7 * DAY), Some(NOW + 7 * DAY));
        assert_eq!(
            auto_grace_until(NOW - 6 * DAY, NOW, 7 * DAY),
            Some(NOW + DAY)
        );
        assert_eq!(auto_grace_until(NOW - 7 * DAY, NOW, 7 * DAY), None);
    }

    #[test]
    fn pressure_has_hysteresis() {
        let disk = |free_gb: u64| {
            Some(DiskSpace {
                free_bytes: free_gb * GB,
                total_bytes: 1_000 * GB,
            })
        };
        assert!(!disk_pressure(disk(200), false));
        assert!(disk_pressure(disk(100), false), "under 15%");
        assert!(disk_pressure(disk(200), true), "still under 25%");
        assert!(!disk_pressure(disk(300), true), "back over the high mark");
        assert!(!disk_pressure(None, true), "unknown space deletes nothing");
        // A small disk: the absolute floor binds before the share does.
        let small = Some(DiskSpace {
            free_bytes: 19 * GB,
            total_bytes: 100 * GB,
        });
        assert!(disk_pressure(small, false));
    }
}