Skip to main content

second_brain_api/
footprint.rs

1use std::io;
2use std::path::{Path, PathBuf};
3use std::sync::Arc;
4use std::time::Duration;
5
6use anyhow::{Context, Result};
7use chrono::{DateTime, Utc};
8
9use crate::AppState;
10use crate::health::{Health, MaintenanceState};
11use crate::heap::heap_report;
12use crate::value_metrics::{ResidentSize, StoreSize};
13
14pub const INTERVAL: Duration = Duration::from_secs(5 * 60);
15
16pub struct StorePaths {
17    pub db: PathBuf,
18    pub indexes: PathBuf,
19    pub archive: PathBuf,
20    pub raw_index: PathBuf,
21}
22
23pub struct FootprintCheck {
24    pub paths: StorePaths,
25    pub state: Arc<AppState>,
26    pub health: Arc<Health>,
27}
28
29impl FootprintCheck {
30    pub fn spawn(self) -> tokio::task::JoinHandle<()> {
31        let check = Arc::new(self);
32        tokio::spawn(async move {
33            let mut interval = tokio::time::interval(INTERVAL);
34            loop {
35                interval.tick().await;
36                let pass = check.clone();
37                if let Err(e) = tokio::task::spawn_blocking(move || pass.run_once(Utc::now())).await
38                {
39                    tracing::error!("store and memory footprint check panicked: {e}");
40                }
41            }
42        })
43    }
44
45    pub fn run_once(&self, now: DateTime<Utc>) {
46        self.measure_store(now);
47        let heap = heap_report(&self.state);
48        let resident = (heap.stats.rss_anon_bytes > 0).then(|| ResidentSize {
49            checked_at: Some(now),
50            rss_anon_bytes: heap.stats.rss_anon_bytes,
51            components: heap.components,
52            raw_index_bytes: self.raw_index_bytes(),
53            unattributed_bytes: heap.unattributed_bytes,
54            limit_bytes: memory_limit(Path::new("/")),
55        });
56        self.health.resident_measured(resident, now);
57    }
58}
59
60impl FootprintCheck {
61    /// The compaction's index merge and VACUUM briefly double part of the
62    /// store on disk, so a pass that overlaps either keeps the last reading.
63    fn measure_store(&self, now: DateTime<Utc>) {
64        if self.compacting() {
65            return;
66        }
67        let store = match store_size(&self.paths, now) {
68            Ok(size) => Some(size),
69            Err(e) => {
70                tracing::warn!("cannot measure the store on disk: {e:#}");
71                None
72            }
73        };
74        if !self.compacting() {
75            self.health.store_measured(store, now);
76        }
77    }
78
79    fn compacting(&self) -> bool {
80        matches!(
81            self.health.report().maintenance.state,
82            MaintenanceState::MergingIndex | MaintenanceState::Vacuuming
83        )
84    }
85
86    /// Kept apart from the heap components and their unattributed rest
87    /// because it is an upper bound that counts mmapped, file-backed pages.
88    fn raw_index_bytes(&self) -> u64 {
89        let Some(raw) = &self.state.raw_index else {
90            return 0;
91        };
92        match raw.read() {
93            Ok(raw) => raw.resident_bytes(),
94            Err(poisoned) => poisoned.into_inner().resident_bytes(),
95        }
96    }
97}
98
99pub fn store_size(paths: &StorePaths, now: DateTime<Utc>) -> Result<StoreSize> {
100    let mut db_bytes = 0;
101    let mut wal_bytes = 0;
102    for suffix in ["", "-wal", "-shm"] {
103        let mut file = paths.db.clone().into_os_string();
104        file.push(suffix);
105        let file = PathBuf::from(file);
106        let bytes = tree_bytes(&file).with_context(|| format!("reading {}", file.display()))?;
107        db_bytes += bytes;
108        if !suffix.is_empty() {
109            wal_bytes += bytes;
110        }
111    }
112    let dir_bytes =
113        |dir: &Path| tree_bytes(dir).with_context(|| format!("reading {}", dir.display()));
114    let index_bytes = dir_bytes(&paths.indexes)?;
115    let archive_bytes = dir_bytes(&paths.archive)?;
116    let raw_index_bytes = dir_bytes(&paths.raw_index)?;
117    let cold_bytes = dir_bytes(&second_brain_core::cold::dir_for_db(&paths.db))?;
118    let db_dir = match paths.db.parent() {
119        Some(dir) if !dir.as_os_str().is_empty() => dir,
120        _ => Path::new("."),
121    };
122    Ok(StoreSize {
123        checked_at: Some(now),
124        db_bytes,
125        wal_bytes,
126        index_bytes,
127        archive_bytes,
128        raw_index_bytes,
129        cold_bytes,
130        total_bytes: db_bytes + index_bytes + archive_bytes + raw_index_bytes + cold_bytes,
131        free_bytes: fs4::available_space(db_dir).ok(),
132    })
133}
134
135/// Bytes of the regular files at or under `path`, without following
136/// symlinks. A missing `path` is 0 and any other error reading `path` itself
137/// fails. Entries below it that vanish or cannot be read count as 0, because
138/// index merges and archive rotation delete files during the walk.
139fn tree_bytes(path: &Path) -> io::Result<u64> {
140    let meta = match std::fs::symlink_metadata(path) {
141        Ok(meta) => meta,
142        Err(e) if e.kind() == io::ErrorKind::NotFound => return Ok(0),
143        Err(e) => return Err(e),
144    };
145    if !meta.is_dir() {
146        return Ok(if meta.is_file() { meta.len() } else { 0 });
147    }
148    let mut total = 0;
149    let entries = match std::fs::read_dir(path) {
150        Ok(entries) => entries,
151        Err(e) if e.kind() == io::ErrorKind::NotFound => return Ok(0),
152        Err(e) => return Err(e),
153    };
154    let mut pending: Vec<PathBuf> = entries.flatten().map(|entry| entry.path()).collect();
155    while let Some(path) = pending.pop() {
156        let Ok(meta) = std::fs::symlink_metadata(&path) else {
157            continue;
158        };
159        if meta.is_file() {
160            total += meta.len();
161        } else if meta.is_dir()
162            && let Ok(entries) = std::fs::read_dir(&path)
163        {
164            pending.extend(entries.flatten().map(|entry| entry.path()));
165        }
166    }
167    Ok(total)
168}
169
170/// The memory this process may use, read under the filesystem `root`: the
171/// lowest cgroup v2 `memory.max` on the way from the process's cgroup up to
172/// the hierarchy's root, capped by `MemTotal`, since a cgroup limit above
173/// physical memory never binds.
174pub fn memory_limit(root: &Path) -> Option<u64> {
175    let physical = std::fs::read_to_string(root.join("proc/meminfo"))
176        .ok()
177        .and_then(|meminfo| mem_total(&meminfo));
178    let cgroup = std::fs::read_to_string(root.join("proc/self/cgroup"))
179        .ok()
180        .and_then(|cgroup| cgroup_v2_path(&cgroup).map(str::to_string))
181        .and_then(|path| cgroup_memory_max(&root.join("sys/fs/cgroup"), &path));
182    match (cgroup, physical) {
183        (Some(c), Some(p)) => Some(c.min(p)),
184        (c, p) => c.or(p),
185    }
186}
187
188fn cgroup_v2_path(cgroup: &str) -> Option<&str> {
189    cgroup.lines().find_map(|line| line.strip_prefix("0::"))
190}
191
192fn cgroup_memory_max(hierarchy: &Path, path: &str) -> Option<u64> {
193    let mut dir = hierarchy.to_path_buf();
194    let mut dirs = vec![dir.clone()];
195    for part in path.split('/').filter(|p| !p.is_empty()) {
196        dir.push(part);
197        dirs.push(dir.clone());
198    }
199    dirs.iter()
200        .filter_map(|dir| std::fs::read_to_string(dir.join("memory.max")).ok())
201        .filter_map(|max| memory_max(&max))
202        .min()
203}
204
205fn memory_max(file: &str) -> Option<u64> {
206    file.trim().parse().ok()
207}
208
209fn mem_total(meminfo: &str) -> Option<u64> {
210    let line = meminfo.lines().find_map(|l| l.strip_prefix("MemTotal:"))?;
211    let kb: u64 = line.trim().strip_suffix("kB")?.trim().parse().ok()?;
212    kb.checked_mul(1024)
213}