snarkos_cli/helpers/
fd_check.rs

1// Copyright (c) 2019-2026 Provable Inc.
2// This file is part of the snarkOS library.
3
4// Licensed under the Apache License, Version 2.0 (the "License");
5// you may not use this file except in compliance with the License.
6// You may obtain a copy of the License at:
7
8// http://www.apache.org/licenses/LICENSE-2.0
9
10// Unless required by applicable law or agreed to in writing, software
11// distributed under the License is distributed on an "AS IS" BASIS,
12// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13// See the License for the specific language governing permissions and
14// limitations under the License.
15
16use std::io;
17
18use tokio::{
19    task,
20    time::{Duration, MissedTickBehavior, interval},
21};
22use tracing::*;
23
24/// Node-scale fd use.
25#[derive(Debug, Clone, Copy)]
26pub struct FdUsage {
27    /// File descriptors currently open.
28    pub open: u64,
29    /// Current soft limit (RLIMIT_NOFILE). `None` == unlimited.
30    pub soft_limit: Option<u64>,
31}
32
33impl FdUsage {
34    /// Fraction of the soft limit in use (0.0..=1.0). 0.0 when unlimited.
35    pub fn ratio(&self) -> f64 {
36        match self.soft_limit {
37            Some(limit) if limit > 0 => self.open as f64 / limit as f64,
38            _ => 0.0,
39        }
40    }
41
42    /// True once usage reaches `threshold` of the soft limit (e.g. 0.8 == 80%).
43    pub fn approaching_limit(&self, threshold: f64) -> bool {
44        self.soft_limit.is_some() && self.ratio() >= threshold
45    }
46}
47
48/// Probe the live system: current soft limit + count of open descriptors.
49pub fn fd_usage() -> io::Result<FdUsage> {
50    let soft_limit = soft_nofile_limit()?;
51    let open = count_open_fds(soft_limit)?;
52    Ok(FdUsage { open, soft_limit })
53}
54
55fn soft_nofile_limit() -> io::Result<Option<u64>> {
56    let (soft, _hard) = rlimit::Resource::NOFILE.get()?;
57    Ok(if soft == rlimit::INFINITY { None } else { Some(soft) })
58}
59
60#[cfg(target_os = "linux")]
61fn count_open_fds(_limit: Option<u64>) -> io::Result<u64> {
62    // Each open descriptor is an entry in /proc/self/fd. The directory
63    // handle itself holds one fd while we iterate, so subtract it back out.
64    let mut n: u64 = 0;
65    for entry in std::fs::read_dir("/proc/self/fd")? {
66        entry?;
67        n += 1;
68    }
69    Ok(n.saturating_sub(1))
70}
71
72#[cfg(all(unix, not(target_os = "linux")))]
73fn count_open_fds(_limit: Option<u64>) -> io::Result<u64> {
74    // macOS and most BSDs expose open fds via /dev/fd (same idea as Linux's /proc/self/fd).
75    // The directory handle itself holds one fd while we iterate, so subtract it back out.
76    let mut n: u64 = 0;
77    for entry in std::fs::read_dir("/dev/fd")? {
78        entry?;
79        n += 1;
80    }
81    Ok(n.saturating_sub(1))
82}
83
84/// System-wide (whole machine) fd use.
85#[derive(Debug, Clone, Copy)]
86pub struct SystemFd {
87    pub allocated: u64,
88    pub max: u64,
89}
90
91impl SystemFd {
92    pub fn ratio(&self) -> f64 {
93        if self.max > 0 { self.allocated as f64 / self.max as f64 } else { 0.0 }
94    }
95}
96
97#[cfg(target_os = "linux")]
98pub fn system_fd_usage() -> std::io::Result<SystemFd> {
99    // /proc/sys/fs/file-nr => "<allocated>\t<free, always 0>\t<max>"
100    let s = std::fs::read_to_string("/proc/sys/fs/file-nr")?;
101    let mut f = s.split_whitespace();
102    let bad = || std::io::Error::new(std::io::ErrorKind::InvalidData, "unexpected file-nr format");
103    let allocated = f.next().and_then(|v| v.parse().ok()).ok_or_else(bad)?;
104    let _free = f.next(); // always 0 on modern kernels
105    let max = f.next().and_then(|v| v.parse().ok()).ok_or_else(bad)?;
106    Ok(SystemFd { allocated, max })
107}
108
109#[cfg(all(unix, not(target_os = "linux")))]
110pub fn system_fd_usage() -> std::io::Result<SystemFd> {
111    // OID names differ by flavor; values are plain integers.
112    #[cfg(target_os = "freebsd")]
113    let (cur_oid, max_oid) = ("kern.openfiles", "kern.maxfiles");
114    #[cfg(target_os = "macos")]
115    let (cur_oid, max_oid) = ("kern.num_files", "kern.maxfiles");
116    #[cfg(any(target_os = "openbsd", target_os = "netbsd"))]
117    let (cur_oid, max_oid) = ("kern.nfiles", "kern.maxfiles");
118    #[cfg(not(any(target_os = "freebsd", target_os = "macos", target_os = "openbsd", target_os = "netbsd")))]
119    return Err(std::io::Error::new(std::io::ErrorKind::Unsupported, "system fd probe unsupported on this OS"));
120
121    fn read(oid: &str) -> std::io::Result<u64> {
122        let out = std::process::Command::new("sysctl").arg("-n").arg(oid).output()?;
123        if !out.status.success() {
124            return Err(std::io::Error::new(std::io::ErrorKind::NotFound, format!("sysctl {oid} unavailable")));
125        }
126        String::from_utf8_lossy(&out.stdout)
127            .trim()
128            .parse()
129            .map_err(|_| std::io::Error::new(std::io::ErrorKind::InvalidData, format!("bad value for {oid}")))
130    }
131
132    Ok(SystemFd { allocated: read(cur_oid)?, max: read(max_oid)? })
133}
134
135pub fn spawn_fd_monitor() -> task::JoinHandle<()> {
136    tokio::spawn(async move {
137        let mut tick = interval(Duration::from_secs(30));
138        tick.set_missed_tick_behavior(MissedTickBehavior::Skip);
139
140        loop {
141            tick.tick().await;
142
143            // (1) the node's own fds
144            match fd_usage() {
145                Ok(u) => {
146                    if let Some(limit) = u.soft_limit {
147                        let (pct, left) = (u.ratio() * 100.0, limit.saturating_sub(u.open));
148                        if u.ratio() >= 0.95 {
149                            error!(
150                                scope = "process",
151                                open = u.open,
152                                limit,
153                                left,
154                                pct = format!("{pct:.1}%"),
155                                "node fd usage critical"
156                            );
157                        } else if u.ratio() >= 0.80 {
158                            warn!(
159                                scope = "process",
160                                open = u.open,
161                                limit,
162                                left,
163                                pct = format!("{pct:.1}%"),
164                                "node fd usage elevated"
165                            );
166                        }
167                    }
168                }
169                Err(e) => error!(error = %e, "process fd probe failed"),
170            }
171
172            // (2) whole-machine fds are allowed 5 percentage points more leeway.
173            match system_fd_usage() {
174                Ok(s) => {
175                    let (pct, left) = (s.ratio() * 100.0, s.max.saturating_sub(s.allocated));
176                    if s.ratio() >= 0.90 {
177                        error!(
178                            scope = "system",
179                            allocated = s.allocated,
180                            max = s.max,
181                            left,
182                            pct = format!("{pct:.1}%"),
183                            "system-wide fd usage critical"
184                        );
185                    } else if s.ratio() >= 0.75 {
186                        warn!(
187                            scope = "system",
188                            allocated = s.allocated,
189                            max = s.max,
190                            left,
191                            pct = format!("{pct:.1}%"),
192                            "system-wide fd usage elevated"
193                        );
194                    }
195                }
196                Err(e) => error!(error = %e, "system fd probe failed"),
197            }
198        }
199    })
200}
snarkos_cli/helpers/fd_check.rs

snarkos_cli/helpers/
fd_check.rs
