Skip to main content

second_brain_api/
footprint.rs

1use std::io;
2use std::path::{Path, PathBuf};
3use std::sync::Arc;
4use std::time::Duration;
5
6use anyhow::{Context, Result};
7use chrono::{DateTime, Utc};
8
9use crate::AppState;
10use crate::health::{Health, MaintenanceState};
11use crate::heap::heap_report;
12use crate::value_metrics::{ResidentSize, StoreSize};
13
14pub const INTERVAL: Duration = Duration::from_secs(5 * 60);
15
16pub struct StorePaths {
17    pub db: PathBuf,
18    pub indexes: PathBuf,
19    pub archive: PathBuf,
20    pub raw_index: PathBuf,
21}
22
23pub struct FootprintCheck {
24    pub paths: StorePaths,
25    pub state: Arc<AppState>,
26    pub health: Arc<Health>,
27}
28
29impl FootprintCheck {
30    pub fn spawn(self) -> tokio::task::JoinHandle<()> {
31        let check = Arc::new(self);
32        tokio::spawn(async move {
33            let mut interval = tokio::time::interval(INTERVAL);
34            loop {
35                interval.tick().await;
36                let pass = check.clone();
37                if let Err(e) = tokio::task::spawn_blocking(move || pass.run_once(Utc::now())).await
38                {
39                    tracing::error!("store and memory footprint check panicked: {e}");
40                }
41            }
42        })
43    }
44
45    pub fn run_once(&self, now: DateTime<Utc>) {
46        self.measure_store(now);
47        let heap = heap_report(&self.state);
48        let resident = (heap.stats.rss_anon_bytes > 0).then(|| ResidentSize {
49            checked_at: Some(now),
50            rss_anon_bytes: heap.stats.rss_anon_bytes,
51            components: heap.components,
52            raw_index_bytes: self.raw_index_bytes(),
53            unattributed_bytes: heap.unattributed_bytes,
54            limit_bytes: memory_limit(Path::new("/")),
55        });
56        self.health.resident_measured(resident, now);
57    }
58}
59
60impl FootprintCheck {
61    /// The compaction's index merge and VACUUM briefly double part of the
62    /// store on disk, so a pass that overlaps either keeps the last reading.
63    fn measure_store(&self, now: DateTime<Utc>) {
64        if self.compacting() {
65            return;
66        }
67        let store = match store_size(&self.paths, now) {
68            Ok(size) => Some(size),
69            Err(e) => {
70                tracing::warn!("cannot measure the store on disk: {e:#}");
71                None
72            }
73        };
74        if !self.compacting() {
75            self.health.store_measured(store, now);
76        }
77    }
78
79    fn compacting(&self) -> bool {
80        matches!(
81            self.health.report().maintenance.state,
82            MaintenanceState::MergingIndex | MaintenanceState::Vacuuming
83        )
84    }
85
86    /// Kept apart from the heap components and their unattributed rest
87    /// because it is an upper bound that counts mmapped, file-backed pages.
88    fn raw_index_bytes(&self) -> u64 {
89        let Some(raw) = &self.state.raw_index else {
90            return 0;
91        };
92        match raw.read() {
93            Ok(raw) => raw.resident_bytes(),
94            Err(poisoned) => poisoned.into_inner().resident_bytes(),
95        }
96    }
97}
98
99pub fn store_size(paths: &StorePaths, now: DateTime<Utc>) -> Result<StoreSize> {
100    let mut db_bytes = 0;
101    let mut wal_bytes = 0;
102    for suffix in ["", "-wal", "-shm"] {
103        let mut file = paths.db.clone().into_os_string();
104        file.push(suffix);
105        let file = PathBuf::from(file);
106        let bytes = tree_bytes(&file).with_context(|| format!("reading {}", file.display()))?;
107        db_bytes += bytes;
108        if !suffix.is_empty() {
109            wal_bytes += bytes;
110        }
111    }
112    let dir_bytes =
113        |dir: &Path| tree_bytes(dir).with_context(|| format!("reading {}", dir.display()));
114    let index_bytes = dir_bytes(&paths.indexes)?;
115    let archive_bytes = dir_bytes(&paths.archive)?;
116    let raw_index_bytes = dir_bytes(&paths.raw_index)?;
117    let db_dir = match paths.db.parent() {
118        Some(dir) if !dir.as_os_str().is_empty() => dir,
119        _ => Path::new("."),
120    };
121    Ok(StoreSize {
122        checked_at: Some(now),
123        db_bytes,
124        wal_bytes,
125        index_bytes,
126        archive_bytes,
127        raw_index_bytes,
128        total_bytes: db_bytes + index_bytes + archive_bytes + raw_index_bytes,
129        free_bytes: fs4::available_space(db_dir).ok(),
130    })
131}
132
133/// Bytes of the regular files at or under `path`, without following
134/// symlinks. A missing `path` is 0 and any other error reading `path` itself
135/// fails. Entries below it that vanish or cannot be read count as 0, because
136/// index merges and archive rotation delete files during the walk.
137fn tree_bytes(path: &Path) -> io::Result<u64> {
138    let meta = match std::fs::symlink_metadata(path) {
139        Ok(meta) => meta,
140        Err(e) if e.kind() == io::ErrorKind::NotFound => return Ok(0),
141        Err(e) => return Err(e),
142    };
143    if !meta.is_dir() {
144        return Ok(if meta.is_file() { meta.len() } else { 0 });
145    }
146    let mut total = 0;
147    let entries = match std::fs::read_dir(path) {
148        Ok(entries) => entries,
149        Err(e) if e.kind() == io::ErrorKind::NotFound => return Ok(0),
150        Err(e) => return Err(e),
151    };
152    let mut pending: Vec<PathBuf> = entries.flatten().map(|entry| entry.path()).collect();
153    while let Some(path) = pending.pop() {
154        let Ok(meta) = std::fs::symlink_metadata(&path) else {
155            continue;
156        };
157        if meta.is_file() {
158            total += meta.len();
159        } else if meta.is_dir()
160            && let Ok(entries) = std::fs::read_dir(&path)
161        {
162            pending.extend(entries.flatten().map(|entry| entry.path()));
163        }
164    }
165    Ok(total)
166}
167
168/// The memory this process may use, read under the filesystem `root`: the
169/// lowest cgroup v2 `memory.max` on the way from the process's cgroup up to
170/// the hierarchy's root, capped by `MemTotal`, since a cgroup limit above
171/// physical memory never binds.
172pub fn memory_limit(root: &Path) -> Option<u64> {
173    let physical = std::fs::read_to_string(root.join("proc/meminfo"))
174        .ok()
175        .and_then(|meminfo| mem_total(&meminfo));
176    let cgroup = std::fs::read_to_string(root.join("proc/self/cgroup"))
177        .ok()
178        .and_then(|cgroup| cgroup_v2_path(&cgroup).map(str::to_string))
179        .and_then(|path| cgroup_memory_max(&root.join("sys/fs/cgroup"), &path));
180    match (cgroup, physical) {
181        (Some(c), Some(p)) => Some(c.min(p)),
182        (c, p) => c.or(p),
183    }
184}
185
186fn cgroup_v2_path(cgroup: &str) -> Option<&str> {
187    cgroup.lines().find_map(|line| line.strip_prefix("0::"))
188}
189
190fn cgroup_memory_max(hierarchy: &Path, path: &str) -> Option<u64> {
191    let mut dir = hierarchy.to_path_buf();
192    let mut dirs = vec![dir.clone()];
193    for part in path.split('/').filter(|p| !p.is_empty()) {
194        dir.push(part);
195        dirs.push(dir.clone());
196    }
197    dirs.iter()
198        .filter_map(|dir| std::fs::read_to_string(dir.join("memory.max")).ok())
199        .filter_map(|max| memory_max(&max))
200        .min()
201}
202
203fn memory_max(file: &str) -> Option<u64> {
204    file.trim().parse().ok()
205}
206
207fn mem_total(meminfo: &str) -> Option<u64> {
208    let line = meminfo.lines().find_map(|l| l.strip_prefix("MemTotal:"))?;
209    let kb: u64 = line.trim().strip_suffix("kB")?.trim().parse().ok()?;
210    kb.checked_mul(1024)
211}