aft/
memory.rs

1use std::collections::BTreeMap;
2use std::path::Path;
3
4use serde::Serialize;
5use serde_json::Value;
6
7/// A cold-path estimate of memory AFT can attribute without allocator hooks.
8///
9/// `estimated_bytes` is `None` when a subsystem is busy or its resident bytes
10/// are not cheaply observable. Counts remain available in those cases so the
11/// status response never substitutes a fabricated byte estimate.
12#[derive(Debug, Clone, Serialize, PartialEq, Eq)]
13pub struct MemoryEstimate {
14    pub status: &'static str,
15    pub bytes_status: &'static str,
16    pub estimated_bytes: Option<u64>,
17    #[serde(skip_serializing_if = "Vec::is_empty")]
18    pub not_estimated: Vec<String>,
19    #[serde(flatten)]
20    pub counts: BTreeMap<String, u64>,
21}
22
23impl MemoryEstimate {
24    pub fn estimated(bytes: u64) -> Self {
25        Self {
26            status: "ready",
27            bytes_status: "estimated",
28            estimated_bytes: Some(bytes),
29            not_estimated: Vec::new(),
30            counts: BTreeMap::new(),
31        }
32    }
33
34    pub fn partial(bytes: u64) -> Self {
35        Self {
36            status: "ready",
37            bytes_status: "partial",
38            estimated_bytes: Some(bytes),
39            not_estimated: Vec::new(),
40            counts: BTreeMap::new(),
41        }
42    }
43
44    pub fn not_estimated() -> Self {
45        Self {
46            status: "ready",
47            bytes_status: "not_estimated",
48            estimated_bytes: None,
49            not_estimated: Vec::new(),
50            counts: BTreeMap::new(),
51        }
52    }
53
54    pub fn busy() -> Self {
55        Self {
56            status: "busy",
57            bytes_status: "not_estimated",
58            estimated_bytes: None,
59            not_estimated: Vec::new(),
60            counts: BTreeMap::new(),
61        }
62    }
63
64    pub fn count(mut self, name: impl Into<String>, value: usize) -> Self {
65        self.counts.insert(name.into(), usize_to_u64(value));
66        self
67    }
68
69    pub fn count_u64(mut self, name: impl Into<String>, value: u64) -> Self {
70        self.counts.insert(name.into(), value);
71        self
72    }
73
74    pub fn gap(mut self, name: impl Into<String>) -> Self {
75        self.not_estimated.push(name.into());
76        self
77    }
78}
79
80#[derive(Debug, Clone, Serialize, PartialEq, Eq)]
81pub struct RootMemorySnapshot {
82    pub status: &'static str,
83    pub attributed_bytes: u64,
84    pub semantic: MemoryEstimate,
85    pub trigram: MemoryEstimate,
86    pub symbols: MemoryEstimate,
87    pub callgraph: MemoryEstimate,
88    pub callgraph_projection: MemoryEstimate,
89    pub inspect: MemoryEstimate,
90    pub bash: MemoryEstimate,
91    pub lsp: MemoryEstimate,
92    pub parser_pool: MemoryEstimate,
93}
94
95impl RootMemorySnapshot {
96    pub fn new(
97        semantic: MemoryEstimate,
98        trigram: MemoryEstimate,
99        symbols: MemoryEstimate,
100        callgraph: MemoryEstimate,
101        callgraph_projection: MemoryEstimate,
102        inspect: MemoryEstimate,
103        bash: MemoryEstimate,
104        lsp: MemoryEstimate,
105        parser_pool: MemoryEstimate,
106    ) -> Self {
107        let estimates = [
108            &semantic,
109            &trigram,
110            &symbols,
111            &callgraph,
112            &callgraph_projection,
113            &inspect,
114            &bash,
115            &lsp,
116            &parser_pool,
117        ];
118        let attributed_bytes = estimates
119            .iter()
120            .filter_map(|estimate| estimate.estimated_bytes)
121            .fold(0u64, u64::saturating_add);
122        let status = if estimates.iter().any(|estimate| estimate.status == "busy") {
123            "busy"
124        } else {
125            "ready"
126        };
127        Self {
128            status,
129            attributed_bytes,
130            semantic,
131            trigram,
132            symbols,
133            callgraph,
134            callgraph_projection,
135            inspect,
136            bash,
137            lsp,
138            parser_pool,
139        }
140    }
141
142    pub fn busy_subsystem_count(&self) -> usize {
143        self.estimates()
144            .iter()
145            .filter(|estimate| estimate.status == "busy")
146            .count()
147    }
148
149    pub fn not_estimated_subsystem_count(&self) -> usize {
150        self.estimates()
151            .iter()
152            .filter(|estimate| estimate.estimated_bytes.is_none())
153            .count()
154    }
155
156    fn estimates(&self) -> [&MemoryEstimate; 9] {
157        [
158            &self.semantic,
159            &self.trigram,
160            &self.symbols,
161            &self.callgraph,
162            &self.callgraph_projection,
163            &self.inspect,
164            &self.bash,
165            &self.lsp,
166            &self.parser_pool,
167        ]
168    }
169}
170
171#[derive(Debug, Clone, Serialize, PartialEq, Eq)]
172pub struct SqliteMemorySnapshot {
173    pub status: &'static str,
174    pub memory_used_bytes: u64,
175    pub memory_highwater_bytes: u64,
176}
177
178impl SqliteMemorySnapshot {
179    fn measure() -> Self {
180        // SQLite's allocator counters are process-wide and internally synchronized.
181        // They intentionally replace per-connection guesses in root estimates.
182        let memory_used = unsafe { rusqlite::ffi::sqlite3_memory_used() };
183        let memory_highwater = unsafe { rusqlite::ffi::sqlite3_memory_highwater(0) };
184        Self {
185            status: "measured",
186            memory_used_bytes: nonnegative_i64_to_u64(memory_used),
187            memory_highwater_bytes: nonnegative_i64_to_u64(memory_highwater),
188        }
189    }
190}
191
192#[derive(Debug, Clone, Serialize, PartialEq, Eq)]
193pub struct AllocatorMemorySnapshot {
194    pub status: &'static str,
195    pub bytes_in_use: Option<u64>,
196    pub size_allocated: Option<u64>,
197    pub retained_slack_bytes: Option<u64>,
198    #[serde(skip_serializing_if = "Option::is_none")]
199    pub not_estimated: Option<&'static str>,
200}
201
202impl AllocatorMemorySnapshot {
203    #[cfg(any(target_os = "macos", all(target_os = "linux", target_env = "gnu")))]
204    fn measured(bytes_in_use: u64, size_allocated: u64) -> Self {
205        Self {
206            status: "measured",
207            bytes_in_use: Some(bytes_in_use),
208            size_allocated: Some(size_allocated),
209            retained_slack_bytes: Some(size_allocated.saturating_sub(bytes_in_use)),
210            not_estimated: None,
211        }
212    }
213
214    // Not cfg-gated to the fallback platforms: linux-gnu also uses this at
215    // RUNTIME when the host glibc predates mallinfo2 (< 2.33), which only
216    // manifests on release binaries built against an old glibc floor.
217    #[cfg_attr(target_os = "macos", allow(dead_code))]
218    fn not_estimated(reason: &'static str) -> Self {
219        Self {
220            status: "not_estimated_on_this_platform",
221            bytes_in_use: None,
222            size_allocated: None,
223            retained_slack_bytes: None,
224            not_estimated: Some(reason),
225        }
226    }
227}
228
229#[derive(Debug, Clone, Serialize, PartialEq, Eq)]
230pub struct ProcessMemorySnapshot {
231    pub rss_status: &'static str,
232    pub rss_bytes: Option<u64>,
233    /// Kernel physical footprint (macOS `phys_footprint`): dirty + compressed +
234    /// IOKit pages, excluding clean/reclaimable ones. This is the number
235    /// Activity Monitor's "Real Memory" and the OOM killer use. RSS counts
236    /// MADV_FREE pages the allocator has already surrendered (the kernel
237    /// reclaims them lazily), so RSS can read gigabytes above what the process
238    /// actually holds — observed 5.1 GB RSS over a 610 MB footprint. None on
239    /// non-macOS platforms (Linux RSS does not have this skew; MADV_FREE'd
240    /// pages leave Linux RSS on reclaim, not on advice).
241    #[serde(skip_serializing_if = "Option::is_none")]
242    pub phys_footprint_bytes: Option<u64>,
243    #[serde(skip_serializing_if = "Option::is_none")]
244    pub rss_not_estimated: Option<&'static str>,
245    pub sqlite: SqliteMemorySnapshot,
246    /// Allocator bytes overlap the attributed subsystem totals and are an
247    /// allocation envelope, not another amount to subtract from RSS.
248    pub allocator: AllocatorMemorySnapshot,
249    pub total_attributed_bytes: u64,
250    pub unattributed_bytes: Option<i64>,
251    pub root_count: usize,
252    pub busy_subsystems: usize,
253    pub not_estimated_subsystems: usize,
254}
255
256#[derive(Debug, Clone, PartialEq, Eq)]
257pub struct AllocatorPressureRelief {
258    pub bytes_released: u64,
259    pub rss_before_bytes: Option<u64>,
260    pub rss_after_bytes: Option<u64>,
261    pub allocator_before: AllocatorMemorySnapshot,
262    pub allocator_after: AllocatorMemorySnapshot,
263}
264
265impl ProcessMemorySnapshot {
266    pub fn from_roots(
267        roots: &BTreeMap<String, RootMemorySnapshot>,
268        shared_semantic_bases: &MemoryEstimate,
269    ) -> Self {
270        let sqlite = SqliteMemorySnapshot::measure();
271        let allocator = allocator_memory_snapshot();
272        let total_attributed_bytes = roots
273            .values()
274            .map(|root| root.attributed_bytes)
275            .fold(0u64, u64::saturating_add)
276            .saturating_add(shared_semantic_bases.estimated_bytes.unwrap_or(0))
277            .saturating_add(sqlite.memory_used_bytes);
278        let busy_subsystems = roots
279            .values()
280            .map(RootMemorySnapshot::busy_subsystem_count)
281            .sum();
282        let not_estimated_subsystems = roots
283            .values()
284            .map(RootMemorySnapshot::not_estimated_subsystem_count)
285            .sum();
286        let rss_bytes = process_rss_bytes();
287        let phys_footprint_bytes = process_phys_footprint_bytes();
288        // Attribute against the footprint when available: it excludes
289        // already-surrendered pages, so the residual actually means
290        // "held memory we cannot explain" instead of allocator noise.
291        let unattributed_basis = phys_footprint_bytes.or(rss_bytes);
292        let unattributed_bytes =
293            unattributed_basis.map(|held| signed_difference(held, total_attributed_bytes));
294        Self {
295            rss_status: if rss_bytes.is_some() {
296                "estimated"
297            } else {
298                "not_estimated_on_this_platform"
299            },
300            rss_bytes,
301            phys_footprint_bytes,
302            rss_not_estimated: rss_bytes
303                .is_none()
304                .then_some("platform_process_rss_unavailable"),
305            sqlite,
306            allocator,
307            total_attributed_bytes,
308            unattributed_bytes,
309            root_count: roots.len(),
310            busy_subsystems,
311            not_estimated_subsystems,
312        }
313    }
314}
315
316/// Cap on per-root detail entries in serialized snapshots. Process totals
317/// always cover every root; only the per-root breakdown is capped so a
318/// many-root daemon process cannot balloon the status payload past
319/// downstream consumers' size limits (the daemon metrics cache truncates
320/// around 27 KB, and JSON keys serialize alphabetically, so an oversized
321/// `memory.roots` map pushes later sections past the cut).
322pub const MEMORY_SNAPSHOT_ROOT_DETAIL_CAP: usize = 8;
323
324#[derive(Debug, Clone, Serialize, PartialEq, Eq)]
325pub struct MemorySnapshot {
326    pub roots_status: &'static str,
327    /// Top roots by attributed bytes, capped at
328    /// [`MEMORY_SNAPSHOT_ROOT_DETAIL_CAP`]; the remainder is summarized by
329    /// `roots_omitted` / `roots_omitted_bytes`.
330    pub roots: BTreeMap<String, RootMemorySnapshot>,
331    /// Total roots attributed (including omitted ones).
332    pub roots_total: usize,
333    /// Roots summarized out of the detail map.
334    pub roots_omitted: usize,
335    /// Attributed bytes carried by the omitted roots (already included in
336    /// `process.total_attributed_bytes`).
337    pub roots_omitted_bytes: u64,
338    /// Immutable borrowed semantic snapshots, attributed once process-wide.
339    pub shared_semantic_bases: MemoryEstimate,
340    pub process: ProcessMemorySnapshot,
341}
342
343impl MemorySnapshot {
344    pub fn new(roots_status: &'static str, roots: BTreeMap<String, RootMemorySnapshot>) -> Self {
345        let shared_semantic_bases = crate::semantic_index::shared_semantic_bases_memory();
346        // Totals cover EVERY root before the detail map is capped.
347        let process = ProcessMemorySnapshot::from_roots(&roots, &shared_semantic_bases);
348        let roots_total = roots.len();
349        let (roots, roots_omitted, roots_omitted_bytes) =
350            cap_root_detail(roots, MEMORY_SNAPSHOT_ROOT_DETAIL_CAP);
351        Self {
352            roots_status,
353            roots,
354            roots_total,
355            roots_omitted,
356            roots_omitted_bytes,
357            shared_semantic_bases,
358            process,
359        }
360    }
361}
362
363/// Keep the `cap` roots with the highest attributed bytes; report the rest
364/// as an omitted-count + omitted-bytes rollup.
365fn cap_root_detail(
366    roots: BTreeMap<String, RootMemorySnapshot>,
367    cap: usize,
368) -> (BTreeMap<String, RootMemorySnapshot>, usize, u64) {
369    if roots.len() <= cap {
370        return (roots, 0, 0);
371    }
372    let mut entries: Vec<(String, RootMemorySnapshot)> = roots.into_iter().collect();
373    // Sort by attributed bytes descending; ties keep path order for determinism.
374    entries.sort_by(|a, b| {
375        b.1.attributed_bytes
376            .cmp(&a.1.attributed_bytes)
377            .then_with(|| a.0.cmp(&b.0))
378    });
379    let omitted: Vec<(String, RootMemorySnapshot)> = entries.split_off(cap);
380    let omitted_bytes = omitted
381        .iter()
382        .map(|(_, snapshot)| snapshot.attributed_bytes)
383        .fold(0u64, u64::saturating_add);
384    (entries.into_iter().collect(), omitted.len(), omitted_bytes)
385}
386
387#[cfg(test)]
388mod snapshot_cap_tests {
389    use super::*;
390
391    fn root_with_bytes(bytes: u64) -> RootMemorySnapshot {
392        let estimate = MemoryEstimate::estimated;
393        RootMemorySnapshot::new(
394            estimate(bytes),
395            estimate(0),
396            estimate(0),
397            estimate(0),
398            estimate(0),
399            estimate(0),
400            estimate(0),
401            estimate(0),
402            estimate(0),
403        )
404    }
405
406    #[test]
407    fn detail_map_capped_but_totals_cover_all_roots() {
408        let mut roots = BTreeMap::new();
409        for i in 0..(MEMORY_SNAPSHOT_ROOT_DETAIL_CAP + 4) {
410            // Distinct sizes so the kept set is deterministic: later roots larger.
411            roots.insert(
412                format!("/root/{i:02}"),
413                root_with_bytes((i as u64 + 1) * 1000),
414            );
415        }
416        let snapshot = MemorySnapshot::new("ready", roots);
417
418        assert_eq!(snapshot.roots.len(), MEMORY_SNAPSHOT_ROOT_DETAIL_CAP);
419        assert_eq!(snapshot.roots_total, MEMORY_SNAPSHOT_ROOT_DETAIL_CAP + 4);
420        assert_eq!(snapshot.roots_omitted, 4);
421        // The four smallest (1000..=4000) are the omitted ones.
422        assert_eq!(snapshot.roots_omitted_bytes, 1000 + 2000 + 3000 + 4000);
423        // Largest roots are the ones kept.
424        assert!(snapshot
425            .roots
426            .values()
427            .all(|root| root.attributed_bytes > 4000));
428        // Process totals include omitted roots' bytes.
429        let expected_total: u64 = (1..=(MEMORY_SNAPSHOT_ROOT_DETAIL_CAP as u64 + 4))
430            .map(|i| i * 1000)
431            .sum();
432        assert!(snapshot.process.total_attributed_bytes >= expected_total);
433    }
434
435    #[test]
436    fn under_cap_keeps_everything_with_zero_omitted() {
437        let mut roots = BTreeMap::new();
438        roots.insert("/a".to_string(), root_with_bytes(10));
439        roots.insert("/b".to_string(), root_with_bytes(20));
440        let snapshot = MemorySnapshot::new("ready", roots);
441        assert_eq!(snapshot.roots.len(), 2);
442        assert_eq!(snapshot.roots_total, 2);
443        assert_eq!(snapshot.roots_omitted, 0);
444        assert_eq!(snapshot.roots_omitted_bytes, 0);
445    }
446}
447
448pub fn path_bytes(path: &Path) -> u64 {
449    #[cfg(unix)]
450    {
451        use std::os::unix::ffi::OsStrExt;
452        usize_to_u64(path.as_os_str().as_bytes().len())
453    }
454    #[cfg(windows)]
455    {
456        use std::os::windows::ffi::OsStrExt;
457        usize_to_u64(path.as_os_str().encode_wide().count())
458            .saturating_mul(std::mem::size_of::<u16>() as u64)
459    }
460    #[cfg(not(any(unix, windows)))]
461    {
462        usize_to_u64(path.to_string_lossy().len())
463    }
464}
465
466pub fn usize_to_u64(value: usize) -> u64 {
467    u64::try_from(value).unwrap_or(u64::MAX)
468}
469
470pub fn estimated_json_bytes(value: &Value) -> u64 {
471    match value {
472        Value::Null => 0,
473        Value::Bool(_) => std::mem::size_of::<bool>() as u64,
474        Value::Number(_) => std::mem::size_of::<serde_json::Number>() as u64,
475        Value::String(value) => usize_to_u64(value.len()),
476        Value::Array(values) => values
477            .iter()
478            .map(estimated_json_bytes)
479            .fold(0u64, u64::saturating_add),
480        Value::Object(values) => values.iter().fold(0u64, |bytes, (key, value)| {
481            bytes
482                .saturating_add(usize_to_u64(key.len()))
483                .saturating_add(estimated_json_bytes(value))
484        }),
485    }
486}
487
488fn signed_difference(lhs: u64, rhs: u64) -> i64 {
489    let difference = i128::from(lhs) - i128::from(rhs);
490    difference.clamp(i128::from(i64::MIN), i128::from(i64::MAX)) as i64
491}
492
493fn nonnegative_i64_to_u64(value: i64) -> u64 {
494    u64::try_from(value).unwrap_or(0)
495}
496
497#[cfg(target_os = "macos")]
498fn allocator_memory_snapshot() -> AllocatorMemorySnapshot {
499    let mut statistics = std::mem::MaybeUninit::<libc::malloc_statistics_t>::zeroed();
500    unsafe {
501        libc::malloc_zone_statistics(libc::malloc_default_zone(), statistics.as_mut_ptr());
502    }
503    let statistics = unsafe { statistics.assume_init() };
504    AllocatorMemorySnapshot::measured(
505        usize_to_u64(statistics.size_in_use),
506        usize_to_u64(statistics.size_allocated),
507    )
508}
509
510#[cfg(all(target_os = "linux", target_env = "gnu"))]
511fn allocator_memory_snapshot() -> AllocatorMemorySnapshot {
512    // mallinfo2 exists only in glibc >= 2.33. Release Linux binaries link
513    // against an older glibc floor (cross gnu images, kept old so dlopen and
514    // wide distro compatibility hold), so a link-time reference to the symbol
515    // fails the release build even though native CI (glibc 2.35) links fine.
516    // Resolve it at runtime instead and report honestly when it is absent.
517    use std::sync::OnceLock;
518    type Mallinfo2Fn = unsafe extern "C" fn() -> libc::mallinfo2;
519    static MALLINFO2: OnceLock<Option<Mallinfo2Fn>> = OnceLock::new();
520    let resolved = MALLINFO2.get_or_init(|| {
521        let symbol = unsafe { libc::dlsym(libc::RTLD_DEFAULT, c"mallinfo2".as_ptr()) };
522        if symbol.is_null() {
523            None
524        } else {
525            // SAFETY: glibc declares mallinfo2 as `struct mallinfo2 (*)(void)`;
526            // the signature matches Mallinfo2Fn exactly.
527            Some(unsafe { std::mem::transmute::<*mut libc::c_void, Mallinfo2Fn>(symbol) })
528        }
529    });
530    let Some(mallinfo2) = resolved else {
531        return AllocatorMemorySnapshot::not_estimated("mallinfo2_requires_glibc_2_33");
532    };
533    let statistics = unsafe { mallinfo2() };
534    let mapped_bytes = statistics.hblkhd as u64;
535    let bytes_in_use = (statistics.uordblks as u64).saturating_add(mapped_bytes);
536    let size_allocated = (statistics.arena as u64).saturating_add(mapped_bytes);
537    AllocatorMemorySnapshot::measured(bytes_in_use, size_allocated)
538}
539
540#[cfg(all(target_os = "linux", target_env = "gnu"))]
541type MallocTrimFn = unsafe extern "C" fn(libc::size_t) -> libc::c_int;
542
543/// Resolve glibc's optional trimming primitive without creating a link-time
544/// dependency on a symbol that musl and alternate allocators do not provide.
545#[cfg(all(target_os = "linux", target_env = "gnu"))]
546fn resolved_malloc_trim() -> Option<MallocTrimFn> {
547    use std::sync::OnceLock;
548    static MALLOC_TRIM: OnceLock<Option<MallocTrimFn>> = OnceLock::new();
549    MALLOC_TRIM
550        .get_or_init(|| {
551            let symbol = unsafe { libc::dlsym(libc::RTLD_DEFAULT, c"malloc_trim".as_ptr()) };
552            if symbol.is_null() {
553                None
554            } else {
555                // SAFETY: glibc declares malloc_trim as `int (size_t)`;
556                // the signature matches MallocTrimFn exactly.
557                Some(unsafe { std::mem::transmute::<*mut libc::c_void, MallocTrimFn>(symbol) })
558            }
559        })
560        .as_ref()
561        .copied()
562}
563
564#[cfg(not(any(target_os = "macos", all(target_os = "linux", target_env = "gnu"))))]
565fn allocator_memory_snapshot() -> AllocatorMemorySnapshot {
566    AllocatorMemorySnapshot::not_estimated("platform_allocator_statistics_unavailable")
567}
568
569#[cfg(target_os = "macos")]
570unsafe extern "C" {
571    fn malloc_zone_pressure_relief(zone: *mut libc::malloc_zone_t, goal: usize) -> usize;
572}
573
574/// Allocator slack (mapped-but-unused arena bytes) above which opportunistic
575/// pressure relief is worth the zone-lock contention it briefly causes.
576pub const ALLOCATOR_SLACK_RELIEF_THRESHOLD_BYTES: u64 = 1024 * 1024 * 1024;
577
578/// Minimum spacing between opportunistic relief passes so a workload that
579/// legitimately cycles through large allocations does not thrash the allocator.
580pub const ALLOCATOR_SLACK_RELIEF_MIN_INTERVAL: std::time::Duration =
581    std::time::Duration::from_secs(300);
582
583/// Decide whether an opportunistic allocator relief pass is due.
584///
585/// Pure so the policy is unit-testable: fires only when the allocator reports
586/// at least the threshold of retained slack AND the previous pass is old
587/// enough. Callers own actually measuring the snapshot and running the pass.
588pub fn allocator_slack_relief_due(
589    retained_slack_bytes: Option<u64>,
590    last_relief: Option<std::time::Instant>,
591    now: std::time::Instant,
592) -> bool {
593    let Some(slack) = retained_slack_bytes else {
594        return false;
595    };
596    if slack < ALLOCATOR_SLACK_RELIEF_THRESHOLD_BYTES {
597        return false;
598    }
599    match last_relief {
600        None => true,
601        Some(at) => now.duration_since(at) >= ALLOCATOR_SLACK_RELIEF_MIN_INTERVAL,
602    }
603}
604
605/// Opportunistically return unused allocator pages when slack is large, even
606/// while sessions are active. The whole-process idle sweep only fires when
607/// every root has been quiet, so one long-lived chatty session used to block
608/// reclamation for the process lifetime (observed: 5.1 GB RSS over ~600 MB of
609/// live data). Runs the relief on a detached thread because allocator trimming
610/// walks allocator state under its lock and must not stall the dispatch loop or
611/// health probes.
612///
613/// Returns true when a pass was spawned (caller records the timestamp).
614#[cfg(any(target_os = "macos", target_os = "linux"))]
615pub fn spawn_allocator_slack_relief_if_due(
616    last_relief: Option<std::time::Instant>,
617    now: std::time::Instant,
618) -> bool {
619    let slack = allocator_memory_snapshot().retained_slack_bytes;
620    if !allocator_slack_relief_due(slack, last_relief, now) {
621        return false;
622    }
623    std::thread::Builder::new()
624        .name("aft-mem-relief".to_string())
625        .spawn(|| {
626            let relief = relieve_allocator_pressure();
627            log::info!(
628                "allocator slack relief: released={} allocator_slack_bytes_before={:?} allocator_slack_bytes_after={:?} rss_bytes_before={:?} rss_bytes_after={:?}",
629                relief.bytes_released,
630                relief.allocator_before.retained_slack_bytes,
631                relief.allocator_after.retained_slack_bytes,
632                relief.rss_before_bytes,
633                relief.rss_after_bytes,
634            );
635        })
636        .is_ok()
637}
638
639/// Ask the platform allocator to return unused pages after a process-wide idle
640/// gate. Callers own that gate because allocator pressure relief can add
641/// latency. Linux invokes glibc's optional `malloc_trim(0)` when the symbol is
642/// available; non-glibc allocators intentionally remain a no-op.
643#[cfg(target_os = "macos")]
644pub fn relieve_allocator_pressure() -> AllocatorPressureRelief {
645    let rss_before_bytes = process_rss_bytes();
646    let allocator_before = allocator_memory_snapshot();
647    let bytes_released = unsafe { malloc_zone_pressure_relief(std::ptr::null_mut(), 0) };
648    let allocator_after = allocator_memory_snapshot();
649    let rss_after_bytes = process_rss_bytes();
650    AllocatorPressureRelief {
651        bytes_released: usize_to_u64(bytes_released),
652        rss_before_bytes,
653        rss_after_bytes,
654        allocator_before,
655        allocator_after,
656    }
657}
658
659#[cfg(target_os = "linux")]
660pub fn relieve_allocator_pressure() -> AllocatorPressureRelief {
661    let rss_before_bytes = process_rss_bytes();
662    let allocator_before = allocator_memory_snapshot();
663    #[cfg(target_env = "gnu")]
664    if let Some(malloc_trim) = resolved_malloc_trim() {
665        // SAFETY: resolved_malloc_trim verifies the symbol and its C ABI
666        // signature before returning the function pointer.
667        unsafe { malloc_trim(0) };
668    }
669    let allocator_after = allocator_memory_snapshot();
670    let rss_after_bytes = process_rss_bytes();
671    let bytes_released = allocator_before
672        .size_allocated
673        .zip(allocator_after.size_allocated)
674        .map(|(before, after)| before.saturating_sub(after))
675        .unwrap_or(0);
676    AllocatorPressureRelief {
677        bytes_released,
678        rss_before_bytes,
679        rss_after_bytes,
680        allocator_before,
681        allocator_after,
682    }
683}
684
685#[cfg(target_os = "macos")]
686fn process_rss_bytes() -> Option<u64> {
687    let mut info = std::mem::MaybeUninit::<libc::proc_taskinfo>::zeroed();
688    let size = std::mem::size_of::<libc::proc_taskinfo>();
689    let written = unsafe {
690        libc::proc_pidinfo(
691            libc::getpid(),
692            libc::PROC_PIDTASKINFO,
693            0,
694            info.as_mut_ptr().cast(),
695            i32::try_from(size).ok()?,
696        )
697    };
698    if written != i32::try_from(size).ok()? {
699        return None;
700    }
701    Some(unsafe { info.assume_init() }.pti_resident_size)
702}
703
704/// Kernel physical footprint via `proc_pid_rusage` (`ri_phys_footprint`).
705/// See `phys_footprint_bytes` for why this, not RSS, is the headline number.
706#[cfg(target_os = "macos")]
707fn process_phys_footprint_bytes() -> Option<u64> {
708    let mut usage = std::mem::MaybeUninit::<libc::rusage_info_v4>::zeroed();
709    let rc = unsafe {
710        libc::proc_pid_rusage(
711            libc::getpid(),
712            libc::RUSAGE_INFO_V4,
713            usage.as_mut_ptr().cast(),
714        )
715    };
716    if rc != 0 {
717        return None;
718    }
719    Some(unsafe { usage.assume_init() }.ri_phys_footprint)
720}
721
722#[cfg(not(target_os = "macos"))]
723fn process_phys_footprint_bytes() -> Option<u64> {
724    None
725}
726
727#[cfg(target_os = "linux")]
728fn process_rss_bytes() -> Option<u64> {
729    let statm = std::fs::read_to_string("/proc/self/statm").ok()?;
730    let resident_pages = statm.split_whitespace().nth(1)?.parse::<u64>().ok()?;
731    let page_size = unsafe { libc::sysconf(libc::_SC_PAGESIZE) };
732    if page_size <= 0 {
733        return None;
734    }
735    resident_pages.checked_mul(page_size as u64)
736}
737
738#[cfg(not(any(target_os = "macos", target_os = "linux")))]
739fn process_rss_bytes() -> Option<u64> {
740    None
741}
742
743#[cfg(test)]
744mod tests {
745    use super::*;
746
747    #[test]
748    fn process_snapshot_preserves_negative_residuals() {
749        assert_eq!(signed_difference(5, 8), -3);
750    }
751
752    #[test]
753    fn slack_relief_fires_on_large_slack_and_respects_spacing() {
754        use std::time::{Duration, Instant};
755        let now = Instant::now();
756        let big = Some(ALLOCATOR_SLACK_RELIEF_THRESHOLD_BYTES);
757        // Unknown slack (allocator stats unavailable) never fires.
758        assert!(!allocator_slack_relief_due(None, None, now));
759        // Below threshold never fires.
760        assert!(!allocator_slack_relief_due(
761            Some(ALLOCATOR_SLACK_RELIEF_THRESHOLD_BYTES - 1),
762            None,
763            now
764        ));
765        // At threshold with no prior pass fires.
766        assert!(allocator_slack_relief_due(big, None, now));
767        // A recent pass suppresses the next one...
768        let recent = now - Duration::from_secs(10);
769        assert!(!allocator_slack_relief_due(big, Some(recent), now));
770        // ...until the minimum spacing has elapsed.
771        let stale = now - ALLOCATOR_SLACK_RELIEF_MIN_INTERVAL;
772        assert!(allocator_slack_relief_due(big, Some(stale), now));
773    }
774
775    #[test]
776    fn json_estimator_scales_with_payload_content() {
777        let empty = estimated_json_bytes(&serde_json::json!({}));
778        let populated = estimated_json_bytes(&serde_json::json!({"message": "hello"}));
779        assert_eq!(empty, 0);
780        assert!(populated >= 12);
781    }
782
783    #[test]
784    fn process_snapshot_exposes_sqlite_and_allocator_sections() {
785        let shared = MemoryEstimate::estimated(7);
786        let snapshot = ProcessMemorySnapshot::from_roots(&BTreeMap::new(), &shared);
787        assert_eq!(snapshot.sqlite.status, "measured");
788        assert!(snapshot.sqlite.memory_highwater_bytes >= snapshot.sqlite.memory_used_bytes);
789        assert_eq!(
790            snapshot.total_attributed_bytes,
791            snapshot.sqlite.memory_used_bytes.saturating_add(7)
792        );
793
794        let serialized = serde_json::to_value(&snapshot).expect("serialize process memory");
795        assert!(serialized["sqlite"]["memory_used_bytes"].is_u64());
796        assert!(serialized["allocator"].get("bytes_in_use").is_some());
797        assert!(serialized["allocator"].get("size_allocated").is_some());
798        assert!(serialized["allocator"]
799            .get("retained_slack_bytes")
800            .is_some());
801    }
802
803    #[cfg(any(target_os = "macos", all(target_os = "linux", target_env = "gnu")))]
804    #[test]
805    fn allocator_snapshot_reports_measured_slack() {
806        let allocator = allocator_memory_snapshot();
807        if allocator.status == "measured" {
808            let in_use = allocator.bytes_in_use.expect("allocator bytes in use");
809            let allocated = allocator.size_allocated.expect("allocator size allocated");
810            assert_eq!(
811                allocator.retained_slack_bytes,
812                Some(allocated.saturating_sub(in_use))
813            );
814        } else {
815            assert_eq!(allocator.status, "not_estimated_on_this_platform");
816            assert_eq!(allocator.bytes_in_use, None);
817            assert_eq!(allocator.size_allocated, None);
818            assert_eq!(allocator.retained_slack_bytes, None);
819            assert_eq!(
820                allocator.not_estimated,
821                Some("mallinfo2_requires_glibc_2_33")
822            );
823        }
824    }
825
826    #[cfg(target_os = "linux")]
827    #[test]
828    fn linux_allocator_pressure_relief_smoke() {
829        let mut allocation = vec![0u8; 32 * 1024 * 1024];
830        for byte in allocation.iter_mut().step_by(4096) {
831            *byte = 1;
832        }
833        std::hint::black_box(&allocation);
834        drop(allocation);
835
836        let relief = relieve_allocator_pressure();
837        std::hint::black_box(relief);
838
839        #[cfg(target_env = "gnu")]
840        assert!(
841            resolved_malloc_trim().is_some(),
842            "glibc malloc_trim must be available for the Linux relief path"
843        );
844    }
845
846    #[cfg(not(any(target_os = "macos", all(target_os = "linux", target_env = "gnu"))))]
847    #[test]
848    fn allocator_snapshot_is_honest_when_platform_counters_are_unavailable() {
849        let allocator = allocator_memory_snapshot();
850        assert_eq!(allocator.status, "not_estimated_on_this_platform");
851        assert_eq!(allocator.bytes_in_use, None);
852        assert_eq!(allocator.size_allocated, None);
853        assert_eq!(allocator.retained_slack_bytes, None);
854        assert_eq!(
855            allocator.not_estimated,
856            Some("platform_allocator_statistics_unavailable")
857        );
858    }
859
860    #[cfg(target_os = "macos")]
861    #[test]
862    #[ignore = "bounded live RSS experiment; run explicitly after allocator changes"]
863    fn allocator_pressure_relief_warm_then_idle_measurement() {
864        let warm_pages = (0..16 * 1024)
865            .map(|seed| {
866                let mut page = Box::new([0u8; 4096]);
867                page[0] = seed as u8;
868                page
869            })
870            .collect::<Vec<_>>();
871        std::hint::black_box(&warm_pages);
872        drop(warm_pages);
873
874        let relief = relieve_allocator_pressure();
875        let sqlite = SqliteMemorySnapshot::measure();
876        eprintln!(
877            "warm-then-idle pressure relief: rss_before={:?} rss_after={:?} allocator_in_use_before={:?} allocator_in_use_after={:?} allocator_allocated_before={:?} allocator_allocated_after={:?} allocator_slack_before={:?} allocator_slack_after={:?} allocator_reported_released={} sqlite_used={} sqlite_highwater={}",
878            relief.rss_before_bytes,
879            relief.rss_after_bytes,
880            relief.allocator_before.bytes_in_use,
881            relief.allocator_after.bytes_in_use,
882            relief.allocator_before.size_allocated,
883            relief.allocator_after.size_allocated,
884            relief.allocator_before.retained_slack_bytes,
885            relief.allocator_after.retained_slack_bytes,
886            relief.bytes_released,
887            sqlite.memory_used_bytes,
888            sqlite.memory_highwater_bytes,
889        );
890        assert_eq!(relief.allocator_before.status, "measured");
891        assert_eq!(relief.allocator_after.status, "measured");
892    }
893}
aft/memory.rs

aft/
memory.rs
