rc_core/admin/
diagnostics.rs

1//! Bounded read-only diagnostic snapshot contracts.
2
3use std::collections::BTreeMap;
4use std::time::Duration;
5
6use async_trait::async_trait;
7use serde::{Deserialize, Deserializer, Serialize};
8use serde_json::Value;
9
10use crate::{Error, Result};
11
12use super::RuntimeCapabilityStatus;
13
14/// Maximum encoded size accepted for one diagnostic response.
15pub const MAX_DIAGNOSTIC_RESPONSE_BYTES: usize = 8 * 1024 * 1024;
16
17/// Detailed authenticated RustFS health snapshot.
18#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
19pub struct DetailedHealthSnapshot {
20    pub version: String,
21    pub deployment_id: Option<String>,
22    pub region: Option<String>,
23    pub timestamp: Option<String>,
24    pub cpu: HealthCpuSnapshot,
25    pub memory: HealthMemorySnapshot,
26    pub os: HealthOsSnapshot,
27    pub process: HealthProcessSnapshot,
28    pub drives: Vec<HealthDriveSnapshot>,
29    pub unsupported_probes: Vec<String>,
30    #[serde(flatten, default)]
31    pub extra: BTreeMap<String, Value>,
32}
33
34#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
35pub struct HealthCpuSnapshot {
36    pub logical_cores: usize,
37    pub brand: String,
38    pub frequency_mhz: u64,
39    pub usage_percent: f64,
40    #[serde(flatten, default)]
41    pub extra: BTreeMap<String, Value>,
42}
43
44#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
45pub struct HealthMemorySnapshot {
46    pub total_bytes: u64,
47    pub used_bytes: u64,
48    pub available_bytes: u64,
49    pub total_swap_bytes: u64,
50    pub used_swap_bytes: u64,
51    #[serde(flatten, default)]
52    pub extra: BTreeMap<String, Value>,
53}
54
55#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
56pub struct HealthOsSnapshot {
57    pub os: String,
58    pub kernel_version: Option<String>,
59    pub os_version: Option<String>,
60    pub hostname: Option<String>,
61    pub arch: String,
62    pub uptime_secs: u64,
63    #[serde(flatten, default)]
64    pub extra: BTreeMap<String, Value>,
65}
66
67#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
68pub struct HealthProcessSnapshot {
69    pub pid: u32,
70    pub cpu_usage_percent: f64,
71    pub memory_bytes: u64,
72    #[serde(flatten, default)]
73    pub extra: BTreeMap<String, Value>,
74}
75
76#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
77pub struct HealthDriveSnapshot {
78    pub endpoint: String,
79    pub drive_path: String,
80    pub state: String,
81    pub total_space: u64,
82    pub used_space: u64,
83    pub available_space: u64,
84    pub read_throughput: f64,
85    pub write_throughput: f64,
86    pub read_latency: f64,
87    pub write_latency: f64,
88    #[serde(flatten, default)]
89    pub extra: BTreeMap<String, Value>,
90}
91
92/// Envelope returned by `/v4/cluster/snapshot`.
93#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
94pub struct ClusterSnapshotDocument {
95    #[serde(deserialize_with = "deserialize_required_nullable")]
96    pub snapshot: Option<DiagnosticClusterSnapshot>,
97    #[serde(flatten, default)]
98    pub extra: BTreeMap<String, Value>,
99}
100
101fn deserialize_required_nullable<'de, D, T>(
102    deserializer: D,
103) -> std::result::Result<Option<T>, D::Error>
104where
105    D: Deserializer<'de>,
106    T: Deserialize<'de>,
107{
108    Option::deserialize(deserializer)
109}
110
111/// Full read-only cluster snapshot with unstable subtrees preserved verbatim.
112#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
113pub struct DiagnosticClusterSnapshot {
114    pub summary: DiagnosticClusterSummary,
115    pub runtime_capabilities_path: String,
116    pub extensions_catalog_path: String,
117    #[serde(default)]
118    pub components: Option<ClusterComponentSnapshots>,
119    #[serde(default)]
120    pub topology: Value,
121    #[serde(default)]
122    pub membership: Value,
123    #[serde(default)]
124    pub pool_state: Value,
125    #[serde(default)]
126    pub local_storage: Value,
127    #[serde(default)]
128    pub peer_health: Value,
129    #[serde(default)]
130    pub rpc_boundary: Value,
131    #[serde(default)]
132    pub observability: Value,
133    #[serde(default)]
134    pub workload_admission: Value,
135    #[serde(default)]
136    pub runtime_status: Value,
137    #[serde(default)]
138    pub actionable_pressure: bool,
139    #[serde(flatten, default)]
140    pub extra: BTreeMap<String, Value>,
141}
142
143#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
144pub struct DiagnosticClusterSummary {
145    pub runtime: RuntimeCapabilityStatus,
146    pub topology: RuntimeCapabilityStatus,
147    pub membership: RuntimeCapabilityStatus,
148    #[serde(default)]
149    pub storage: Option<RuntimeCapabilityStatus>,
150    pub peer_health: RuntimeCapabilityStatus,
151    #[serde(default)]
152    pub listing: Option<RuntimeCapabilityStatus>,
153    #[serde(default)]
154    pub usage: Option<RuntimeCapabilityStatus>,
155    pub rpc_boundary: RuntimeCapabilityStatus,
156    pub observability: RuntimeCapabilityStatus,
157    pub workload_admission: RuntimeCapabilityStatus,
158    pub actionable_pressure: RuntimeCapabilityStatus,
159    #[serde(flatten, default)]
160    pub extra: BTreeMap<String, Value>,
161}
162
163#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
164pub struct ClusterComponentSnapshots {
165    #[serde(default)]
166    pub storage: Option<ClusterComponentSnapshot>,
167    #[serde(default)]
168    pub peer_health: Option<ClusterComponentSnapshot>,
169    #[serde(default)]
170    pub listing: Option<ClusterListingSnapshot>,
171    #[serde(default)]
172    pub usage: Option<ClusterUsageSnapshot>,
173    #[serde(default)]
174    pub workload_admission: Option<ClusterComponentSnapshot>,
175    #[serde(flatten, default)]
176    pub extra: BTreeMap<String, Value>,
177}
178
179#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
180pub struct ClusterComponentSnapshot {
181    #[serde(default)]
182    pub source: String,
183    #[serde(default)]
184    pub condition: String,
185    #[serde(default)]
186    pub status: Option<RuntimeCapabilityStatus>,
187    #[serde(flatten, default)]
188    pub extra: BTreeMap<String, Value>,
189}
190
191#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
192pub struct ClusterListingSnapshot {
193    #[serde(default)]
194    pub source: String,
195    #[serde(default)]
196    pub condition: String,
197    #[serde(default)]
198    pub status: Option<RuntimeCapabilityStatus>,
199    #[serde(default)]
200    pub internode_stall_timeouts_total: u64,
201    #[serde(default)]
202    pub hint: String,
203    #[serde(flatten, default)]
204    pub extra: BTreeMap<String, Value>,
205}
206
207#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
208pub struct ClusterUsageSnapshot {
209    #[serde(default)]
210    pub source: String,
211    #[serde(default)]
212    pub condition: String,
213    #[serde(default)]
214    pub status: Option<RuntimeCapabilityStatus>,
215    #[serde(default)]
216    pub dirty_pending_buckets: u64,
217    #[serde(default)]
218    pub last_dirty_mark_unix_secs: u64,
219    #[serde(default)]
220    pub last_dirty_clear_unix_secs: u64,
221    #[serde(default)]
222    pub last_cycle_dirty_buckets: u64,
223    #[serde(default)]
224    pub last_cycle_cleared_dirty_buckets: u64,
225    #[serde(default)]
226    pub last_usage_save_unix_secs: u64,
227    #[serde(default)]
228    pub last_usage_save_result: String,
229    #[serde(default)]
230    pub last_success_unix_secs: Option<u64>,
231    #[serde(default)]
232    pub last_error: Option<String>,
233    #[serde(flatten, default)]
234    pub extra: BTreeMap<String, Value>,
235}
236
237/// Default bytes uploaded by each client-devnull request.
238pub const DEFAULT_CLIENT_DEVNULL_BYTES: u64 = 8 * 1024 * 1024;
239/// Maximum bytes uploaded across all concurrent client-devnull requests.
240pub const MAX_CLIENT_DEVNULL_AGGREGATE_BYTES: u64 = 64 * 1024 * 1024;
241/// Default number of concurrent client-devnull requests.
242pub const DEFAULT_CLIENT_DEVNULL_CONCURRENCY: u8 = 1;
243/// Maximum number of concurrent client-devnull requests.
244pub const MAX_CLIENT_DEVNULL_CONCURRENCY: u8 = 4;
245/// Default active probe timeout.
246pub const DEFAULT_CLIENT_DEVNULL_TIMEOUT: Duration = Duration::from_secs(30);
247/// Maximum active probe timeout.
248pub const MAX_CLIENT_DEVNULL_TIMEOUT: Duration = Duration::from_secs(60);
249
250/// Validated limits for one bounded client-to-server devnull probe.
251#[derive(Debug, Clone, Copy, PartialEq, Eq)]
252pub struct ClientDevnullRequest {
253    bytes_per_request: u64,
254    total_bytes: u64,
255    concurrency: u8,
256    timeout: Duration,
257}
258
259impl ClientDevnullRequest {
260    /// Validate probe limits before any server request is made.
261    pub fn new(bytes_per_request: u64, concurrency: u8, timeout: Duration) -> Result<Self> {
262        if bytes_per_request == 0 {
263            return Err(Error::Config(
264                "Client devnull size must be greater than zero".to_string(),
265            ));
266        }
267        if !(1..=MAX_CLIENT_DEVNULL_CONCURRENCY).contains(&concurrency) {
268            return Err(Error::Config(format!(
269                "Client devnull concurrency must be between 1 and {MAX_CLIENT_DEVNULL_CONCURRENCY}"
270            )));
271        }
272        let total_bytes = bytes_per_request
273            .checked_mul(u64::from(concurrency))
274            .ok_or_else(|| {
275                Error::Config("Client devnull aggregate size is too large".to_string())
276            })?;
277        if total_bytes > MAX_CLIENT_DEVNULL_AGGREGATE_BYTES {
278            return Err(Error::Config(format!(
279                "Client devnull aggregate size must not exceed {MAX_CLIENT_DEVNULL_AGGREGATE_BYTES} bytes"
280            )));
281        }
282        if timeout < Duration::from_secs(1) || timeout > MAX_CLIENT_DEVNULL_TIMEOUT {
283            return Err(Error::Config(format!(
284                "Client devnull timeout must be between 1 and {} seconds",
285                MAX_CLIENT_DEVNULL_TIMEOUT.as_secs()
286            )));
287        }
288
289        Ok(Self {
290            bytes_per_request,
291            total_bytes,
292            concurrency,
293            timeout,
294        })
295    }
296
297    /// Bytes sent by each concurrent request.
298    pub const fn bytes_per_request(self) -> u64 {
299        self.bytes_per_request
300    }
301
302    /// Aggregate bytes sent by all requests.
303    pub const fn total_bytes(self) -> u64 {
304        self.total_bytes
305    }
306
307    /// Number of concurrent requests.
308    pub const fn concurrency(self) -> u8 {
309        self.concurrency
310    }
311
312    /// Maximum duration of the active probe.
313    pub const fn timeout(self) -> Duration {
314        self.timeout
315    }
316}
317
318/// Validated aggregate result from a client-to-server devnull probe.
319#[derive(Debug, Clone, Copy, PartialEq)]
320pub struct ClientDevnullResult {
321    pub requested_bytes: u64,
322    pub received_bytes: u64,
323    pub concurrency: u8,
324    pub elapsed_seconds: f64,
325    pub aggregate_throughput_bytes_per_second: f64,
326}
327
328/// Read-only diagnostic adapter boundary.
329#[async_trait]
330pub trait DiagnosticReadApi: Send + Sync {
331    async fn health_snapshot(&self) -> Result<DetailedHealthSnapshot>;
332    async fn cluster_snapshot(&self) -> Result<ClusterSnapshotDocument>;
333    async fn extensions_catalog(&self) -> Result<super::ExtensionsCatalog>;
334}
335
336#[cfg(test)]
337mod tests {
338    use super::*;
339
340    #[test]
341    fn client_devnull_defaults_match_the_bounded_probe_contract() {
342        assert_eq!(DEFAULT_CLIENT_DEVNULL_BYTES, 8 * 1024 * 1024);
343        assert_eq!(MAX_CLIENT_DEVNULL_AGGREGATE_BYTES, 64 * 1024 * 1024);
344        assert_eq!(DEFAULT_CLIENT_DEVNULL_TIMEOUT, Duration::from_secs(30));
345        assert_eq!(DEFAULT_CLIENT_DEVNULL_CONCURRENCY, 1);
346    }
347
348    #[test]
349    fn client_devnull_request_accepts_the_maximum_aggregate_payload() {
350        let request = ClientDevnullRequest::new(16 * 1024 * 1024, 4, Duration::from_secs(60))
351            .expect("maximum aggregate payload should be valid");
352
353        assert_eq!(request.bytes_per_request(), 16 * 1024 * 1024);
354        assert_eq!(request.total_bytes(), 64 * 1024 * 1024);
355        assert_eq!(request.concurrency(), 4);
356        assert_eq!(request.timeout(), Duration::from_secs(60));
357    }
358
359    #[test]
360    fn client_devnull_request_rejects_invalid_limits() {
361        for (bytes, concurrency, timeout) in [
362            (0, 1, Duration::from_secs(30)),
363            (8 * 1024 * 1024, 0, Duration::from_secs(30)),
364            (8 * 1024 * 1024, 5, Duration::from_secs(30)),
365            (32 * 1024 * 1024, 3, Duration::from_secs(30)),
366            (8 * 1024 * 1024, 1, Duration::ZERO),
367            (8 * 1024 * 1024, 1, Duration::from_millis(999)),
368            (8 * 1024 * 1024, 1, Duration::from_secs(61)),
369        ] {
370            assert!(
371                ClientDevnullRequest::new(bytes, concurrency, timeout).is_err(),
372                "bytes={bytes}, concurrency={concurrency}, timeout={timeout:?}"
373            );
374        }
375    }
376}
rc_core/admin/diagnostics.rs

rc_core/admin/
diagnostics.rs
