rocketmq_runtime/
diagnostics.rs

1// Copyright 2023 The RocketMQ Rust Authors
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15mod conversion;
16
17use conversion::*;
18
19use std::sync::atomic::AtomicU64;
20use std::sync::atomic::Ordering;
21use std::sync::Arc;
22use std::time::Duration;
23
24use chrono::DateTime;
25use chrono::Utc;
26use serde::Deserialize;
27use serde::Serialize;
28
29use crate::blocking::BlockingExecutorAggregate;
30use crate::blocking::BlockingExecutorSnapshot;
31use crate::blocking::BlockingKind;
32use crate::blocking::BlockingLane;
33use crate::metadata_io::MetadataIoSnapshot;
34use crate::scheduled::ScheduledTaskSnapshot;
35use crate::shutdown_report::ShutdownReport;
36use crate::task_group::TaskDetailScope;
37use crate::task_group::TaskGroup;
38use crate::task_group::TaskGroupEventCounts;
39use crate::task_group::TaskGroupId;
40use crate::task_group::TaskGroupLifecycleState;
41use crate::task_group::TaskKind;
42
43static NEXT_RUNTIME_DIAGNOSTICS_ID: AtomicU64 = AtomicU64::new(1);
44
45/// Represents runtime diagnostics.
46#[derive(Debug, Clone)]
47pub struct RuntimeDiagnostics {
48    runtime_id: Arc<str>,
49}
50
51/// Represents runtime diagnostics snapshot.
52#[derive(Debug, Clone, Serialize)]
53pub struct RuntimeDiagnosticsSnapshot {
54    /// The runtime identifier.
55    pub runtime_id: String,
56    /// The root name value.
57    pub root_name: String,
58    /// The group identifier.
59    pub group_id: TaskGroupId,
60    /// The parent group identifier.
61    pub parent_group_id: Option<TaskGroupId>,
62    /// The lifecycle state value.
63    pub lifecycle_state: TaskGroupLifecycleState,
64    /// The number of task entries.
65    pub task_count: usize,
66    /// The number of child entries.
67    pub child_count: usize,
68    /// The number of active task entries.
69    pub active_tasks: usize,
70    /// The number of active child registry slots.
71    pub registry_slots: usize,
72    /// Failures absorbed anywhere in the root's tree.
73    pub events: TaskGroupEventCounts,
74    /// The blocking lanes value.
75    pub blocking_lanes: Vec<BlockingExecutorSnapshot>,
76}
77
78/// A stable component identifier used by the sanitized diagnostics view.
79///
80/// The enum deliberately avoids caller-provided labels so diagnostics cannot
81/// disclose deployment names or other high-cardinality runtime data.
82#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
83#[serde(rename_all = "snake_case")]
84pub enum RuntimeComponent {
85    /// A RocketMQ broker process.
86    Broker,
87    /// A RocketMQ NameServer process.
88    NameServer,
89    /// A RocketMQ controller process.
90    Controller,
91    /// A RocketMQ proxy process.
92    Proxy,
93    /// A RocketMQ MCP server process.
94    Mcp,
95    /// An AI SRE control-plane process.
96    SreControlPlane,
97    /// An AI SRE connector process.
98    SreConnector,
99    /// A component without a dedicated bounded identifier.
100    Other,
101}
102
103/// Describes the lifecycle state of the root task group.
104#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
105#[serde(rename_all = "snake_case")]
106pub enum RuntimeLifecycleStateV1 {
107    /// The task group accepts new tasks and child groups.
108    Open,
109    /// Shutdown has started and new work is rejected.
110    Closing,
111    /// The task group is closed and its owned work is draining.
112    Closed,
113    /// Shutdown completed and all bounded cleanup attempts have finished.
114    ShutdownCompleted,
115    /// The lifecycle state could not be mapped to a supported value.
116    Poisoned,
117}
118
119/// Identifies a bounded category of runtime-owned task.
120#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
121#[serde(rename_all = "snake_case")]
122pub enum RuntimeTaskKindV1 {
123    /// A long-lived service task.
124    Service,
125    /// A background worker task.
126    Worker,
127    /// A task that drives a scheduled workload.
128    ScheduledDriver,
129    /// One execution of a scheduled workload.
130    ScheduledRun,
131    /// A task that reaps completed blocking work.
132    BlockingReaper,
133    /// A task that coordinates shutdown.
134    Shutdown,
135    /// A task kind without a dedicated bounded identifier.
136    Other,
137}
138
139/// Identifies a bounded blocking-executor lane.
140#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
141#[serde(rename_all = "snake_case")]
142pub enum RuntimeBlockingLaneV1 {
143    /// Blocking storage input and output.
144    StorageIo,
145    /// Blocking metadata input and output.
146    MetadataIo,
147    /// CPU-intensive cryptographic work.
148    CpuCrypto,
149}
150
151/// Identifies a bounded category of blocking work.
152#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
153#[serde(rename_all = "snake_case")]
154pub enum RuntimeBlockingKindV1 {
155    /// Short blocking input or output.
156    ShortIo,
157    /// CPU-bound work.
158    CpuBound,
159    /// Blocking work expected to run for an extended duration.
160    LongRunning,
161}
162
163/// Bounds applied while creating [`RuntimeDiagnosticsViewV1`].
164#[derive(Debug, Clone, Copy)]
165pub struct RuntimeDiagnosticsViewOptions {
166    /// Elapsed time after which an active task is classified as long-running.
167    pub long_running_threshold: Duration,
168    /// Maximum number of task-kind summaries included in the view.
169    pub max_task_kind_summaries: usize,
170    /// Maximum number of blocking-lane summaries included in the view.
171    pub max_blocking_lane_summaries: usize,
172}
173
174impl Default for RuntimeDiagnosticsViewOptions {
175    fn default() -> Self {
176        Self {
177            long_running_threshold: Duration::from_secs(30),
178            max_task_kind_summaries: TaskKind::COUNT,
179            max_blocking_lane_summaries: BlockingLane::ALL.len(),
180        }
181    }
182}
183
184/// Versioned, bounded, and sanitized runtime diagnostics safe for authenticated
185/// operational APIs.
186///
187/// Unlike [`RuntimeDiagnosticsSnapshot`], this view never exposes runtime IDs,
188/// task IDs, task names, task-group names, executor names, arguments, or
189/// configuration objects.
190#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
191pub struct RuntimeDiagnosticsViewV1 {
192    /// Schema identifier for this serialized diagnostics view.
193    pub schema_version: String,
194    /// UTC timestamp at which the snapshot was observed.
195    pub observed_at: DateTime<Utc>,
196    /// Bounded component category that owns the runtime.
197    pub component: RuntimeComponent,
198    /// Current lifecycle state of the root task group.
199    pub lifecycle_state: RuntimeLifecycleStateV1,
200    /// Number of task groups represented by the snapshot.
201    pub task_group_count: usize,
202    /// Number of active tasks represented by the snapshot.
203    pub task_count: usize,
204    /// Bounded aggregate summaries grouped by task kind.
205    pub task_kinds: Vec<RuntimeTaskKindSummaryV1>,
206    /// Bounded aggregate summaries for blocking-executor lanes.
207    pub blocking_lanes: Vec<RuntimeBlockingLaneSummaryV1>,
208    /// Whether one or more summaries were omitted by configured bounds.
209    pub truncated: bool,
210}
211
212impl RuntimeDiagnosticsViewV1 {
213    /// Stable schema identifier emitted by [`RuntimeDiagnostics::view_v1`].
214    pub const SCHEMA_VERSION: &'static str = "rocketmq.runtime-diagnostics.v1";
215}
216
217/// Aggregated task counts and elapsed-time information for one task kind.
218#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
219pub struct RuntimeTaskKindSummaryV1 {
220    /// Bounded task category.
221    pub kind: RuntimeTaskKindV1,
222    /// Number of active tasks in the category.
223    pub active: usize,
224    /// Number of active tasks exceeding the long-running threshold.
225    pub long_running: usize,
226    /// Maximum elapsed time among active tasks, in milliseconds.
227    pub max_elapsed_millis: u64,
228}
229
230/// Aggregated state for one blocking-executor lane.
231#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
232pub struct RuntimeBlockingLaneSummaryV1 {
233    /// Bounded blocking-executor lane.
234    pub lane: RuntimeBlockingLaneV1,
235    /// Configured concurrency limit when the executor exposes it.
236    #[serde(default, skip_serializing_if = "Option::is_none")]
237    pub max_concurrency: Option<usize>,
238    /// Configured queue-depth limit when the executor exposes it.
239    #[serde(default, skip_serializing_if = "Option::is_none")]
240    pub max_queue_depth: Option<usize>,
241    /// Number of queued blocking tasks.
242    pub queued: usize,
243    /// Number of running blocking tasks.
244    pub running: usize,
245    /// Number of timed-out tasks whose blocking work is still running.
246    pub timed_out_still_running: usize,
247    /// Number of blocking tasks that remain active during observation.
248    pub blocking_still_running: usize,
249    /// Bounded aggregate summaries grouped by blocking-work kind.
250    pub task_kinds: Vec<RuntimeBlockingKindSummaryV1>,
251}
252
253/// Aggregated task counts and elapsed-time information for one blocking-work kind.
254#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
255pub struct RuntimeBlockingKindSummaryV1 {
256    /// Bounded blocking-work category.
257    pub kind: RuntimeBlockingKindV1,
258    /// Number of active blocking tasks in the category.
259    pub active: usize,
260    /// Maximum elapsed time among active tasks, in milliseconds.
261    pub max_elapsed_millis: u64,
262}
263
264impl RuntimeDiagnostics {
265    pub(crate) fn new() -> Self {
266        let runtime_id = NEXT_RUNTIME_DIAGNOSTICS_ID.fetch_add(1, Ordering::Relaxed);
267        Self {
268            runtime_id: Arc::from(format!("rocketmq-runtime-{runtime_id}")),
269        }
270    }
271
272    /// Returns the runtime id.
273    pub fn runtime_id(&self) -> &str {
274        &self.runtime_id
275    }
276
277    /// Returns the snapshot.
278    pub fn snapshot(
279        &self,
280        root: &TaskGroup,
281        blocking_lanes: Vec<BlockingExecutorSnapshot>,
282    ) -> RuntimeDiagnosticsSnapshot {
283        RuntimeDiagnosticsSnapshot {
284            runtime_id: self.runtime_id.to_string(),
285            root_name: root.name().to_string(),
286            group_id: root.id(),
287            parent_group_id: root.parent_id(),
288            lifecycle_state: root.lifecycle_state(),
289            task_count: root.task_count(),
290            child_count: root.component_count(),
291            active_tasks: root.task_count(),
292            registry_slots: root.component_count(),
293            events: root.event_counts(),
294            blocking_lanes,
295        }
296    }
297
298    /// Creates a bounded diagnostics view using the default sanitization limits.
299    pub fn view_v1(
300        &self,
301        component: RuntimeComponent,
302        root: &TaskGroup,
303        blocking_lanes: Vec<BlockingExecutorSnapshot>,
304    ) -> RuntimeDiagnosticsViewV1 {
305        self.view_v1_with_options(
306            component,
307            root,
308            blocking_lanes,
309            RuntimeDiagnosticsViewOptions::default(),
310        )
311    }
312
313    /// Creates a bounded diagnostics view using explicit sanitization limits.
314    pub fn view_v1_with_options(
315        &self,
316        component: RuntimeComponent,
317        root: &TaskGroup,
318        blocking_lanes: Vec<BlockingExecutorSnapshot>,
319        options: RuntimeDiagnosticsViewOptions,
320    ) -> RuntimeDiagnosticsViewV1 {
321        self.view_v1_with_aggregates(
322            component,
323            root,
324            blocking_lanes.into_iter().map(Into::into).collect(),
325            options,
326        )
327    }
328
329    pub(crate) fn view_v1_with_aggregates(
330        &self,
331        component: RuntimeComponent,
332        root: &TaskGroup,
333        blocking_lanes: Vec<BlockingExecutorAggregate>,
334        options: RuntimeDiagnosticsViewOptions,
335    ) -> RuntimeDiagnosticsViewV1 {
336        let task_diagnostics = root.diagnostics(options.long_running_threshold);
337        let task_kind_count = task_diagnostics.task_kinds.len();
338        let task_kinds = task_diagnostics
339            .task_kinds
340            .into_iter()
341            .take(options.max_task_kind_summaries)
342            .map(|summary| RuntimeTaskKindSummaryV1 {
343                kind: runtime_task_kind(summary.kind),
344                active: summary.active,
345                long_running: summary.long_running,
346                max_elapsed_millis: duration_millis(summary.max_elapsed),
347            })
348            .collect();
349
350        let blocking_lane_count = blocking_lanes.len();
351        let blocking_lanes = blocking_lanes
352            .into_iter()
353            .take(options.max_blocking_lane_summaries.min(BlockingLane::ALL.len()))
354            .map(sanitize_blocking_lane)
355            .collect();
356
357        RuntimeDiagnosticsViewV1 {
358            schema_version: RuntimeDiagnosticsViewV1::SCHEMA_VERSION.to_string(),
359            observed_at: Utc::now(),
360            component,
361            lifecycle_state: runtime_lifecycle_state(root.lifecycle_state()),
362            task_group_count: task_diagnostics.group_count,
363            task_count: task_diagnostics.task_count,
364            task_kinds,
365            blocking_lanes,
366            truncated: task_kind_count > options.max_task_kind_summaries
367                || blocking_lane_count > options.max_blocking_lane_summaries.min(3),
368        }
369    }
370}
371
372/// Names the population a V2 diagnostics value covers.
373///
374/// The scope is part of the payload, so a reader never has to infer it from how
375/// the view was produced, and a partial value cannot be read as a total.
376#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
377#[serde(rename_all = "snake_case")]
378pub enum RuntimeDiagnosticsScope {
379    /// Only the calling group's own state.
380    Local,
381    /// The calling group and every descendant group.
382    Subtree,
383    /// State owned once per process and shared by every component.
384    ProcessShared,
385}
386
387/// Bounds applied while creating [`RuntimeDiagnosticsViewV2`].
388///
389/// The detail budgets default to zero, so a routine sample emits only bounded
390/// aggregates and a task list is always an explicit request. Task age and group
391/// summaries still scan the active population; detail bounds do not cap that
392/// aggregate scan. Concurrent submissions and completions can be observed at
393/// different instants across groups; the snapshot is not a global transaction.
394#[derive(Debug, Clone, Copy)]
395pub struct RuntimeDiagnosticsViewOptionsV2 {
396    /// Elapsed time after which an active task is classified as long-running.
397    pub long_running_threshold: Duration,
398    /// Maximum number of task-kind summaries included in the view.
399    pub max_task_kind_summaries: usize,
400    /// Maximum number of blocking-lane summaries included in the view.
401    pub max_blocking_lane_summaries: usize,
402    /// Maximum number of schedule summaries included in the view.
403    pub max_schedule_tasks: usize,
404    /// Maximum number of metadata resources summarized in the view.
405    pub max_metadata_resources: usize,
406    /// Maximum number of task details emitted when a detail list is requested.
407    pub max_detail_entries: usize,
408    /// Maximum number of tasks examined, and of descendant task groups
409    /// visited, when a detail list is requested.
410    pub detail_scan_budget: usize,
411}
412
413impl Default for RuntimeDiagnosticsViewOptionsV2 {
414    fn default() -> Self {
415        Self {
416            long_running_threshold: Duration::from_secs(30),
417            max_task_kind_summaries: TaskKind::COUNT,
418            max_blocking_lane_summaries: BlockingLane::ALL.len(),
419            max_schedule_tasks: 16,
420            max_metadata_resources: 32,
421            max_detail_entries: 0,
422            detail_scan_budget: 0,
423        }
424    }
425}
426
427/// Component-owned inputs for the V2 view.
428///
429/// A section whose input is absent is reported as absent rather than as zero, so
430/// a caller that does not own scheduled work, a metadata actor, or a shutdown
431/// report cannot make the view claim those values are empty.
432#[derive(Debug, Clone, Default)]
433pub struct RuntimeDiagnosticsInputs {
434    /// Snapshots of the caller's scheduled tasks.
435    pub schedule: Vec<ScheduledTaskSnapshot>,
436    /// The snapshot of the caller's metadata actor.
437    pub metadata: Option<MetadataIoSnapshot>,
438    /// The most recent shutdown report for the caller's group.
439    pub shutdown: Option<ShutdownReport>,
440}
441
442/// Bounded task counts for one group population.
443#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
444pub struct RuntimeTaskSectionV2 {
445    /// The population these counts cover.
446    pub scope: RuntimeDiagnosticsScope,
447    /// Number of task groups represented.
448    pub task_group_count: usize,
449    /// Number of active tasks represented.
450    pub task_count: usize,
451    /// Number of active tasks in the calling group alone.
452    ///
453    /// The aggregate `task_count` is scoped to the subtree, so this value lets a
454    /// reader separate the calling group's own work from its descendants without
455    /// requesting a detail list.
456    pub local_task_count: usize,
457    /// Number of active tasks exceeding the long-running threshold.
458    pub long_running: usize,
459    /// Maximum elapsed time among active tasks, in milliseconds.
460    pub max_elapsed_millis: u64,
461    /// Bounded aggregate summaries grouped by task kind.
462    pub task_kinds: Vec<RuntimeTaskKindSummaryV1>,
463    /// Whether summaries were omitted by configured bounds.
464    pub truncated: bool,
465}
466
467/// Bounded blocking-executor state, including work that outlived its wait.
468#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
469pub struct RuntimeBlockingSectionV2 {
470    /// The population these counts cover.
471    pub scope: RuntimeDiagnosticsScope,
472    /// Number of queued blocking tasks.
473    pub queued: usize,
474    /// Number of running blocking tasks.
475    pub running: usize,
476    /// Number of waits that expired while the blocking closure keeps running.
477    pub timed_out_still_running: usize,
478    /// Number of blocking tasks that were still running during observation.
479    pub blocking_still_running: usize,
480    /// Bounded per-lane summaries.
481    pub lanes: Vec<RuntimeBlockingLaneSummaryV1>,
482    /// Whether summaries were omitted by configured bounds.
483    pub truncated: bool,
484}
485
486/// Bounded scheduled-work aggregates.
487#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
488pub struct RuntimeScheduleSectionV2 {
489    /// The population these counts cover.
490    pub scope: RuntimeDiagnosticsScope,
491    /// Number of scheduled tasks examined.
492    pub tasks_scanned: usize,
493    /// Number of scheduled tasks included in the aggregates.
494    pub tasks_emitted: usize,
495    /// Number of runs that are still active.
496    pub active_runs: u64,
497    /// Number of runs started.
498    pub runs: u64,
499    /// Number of skipped ticks.
500    pub skips: u64,
501    /// Number of overlapping runs that were coalesced or refused.
502    pub overlaps: u64,
503    /// Number of failed runs.
504    pub failures: u64,
505    /// Maximum run duration, in milliseconds.
506    pub max_elapsed_millis: u64,
507    /// Whether summaries were omitted by configured bounds.
508    pub truncated: bool,
509}
510
511/// Bounded retained-metadata state.
512#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
513pub struct RuntimeMetadataSectionV2 {
514    /// The population these counts cover.
515    pub scope: RuntimeDiagnosticsScope,
516    /// Whether the metadata actor still accepts work.
517    pub accepting: bool,
518    /// Number of retained operations.
519    pub retained_operations: usize,
520    /// Number of retained payload bytes.
521    pub retained_bytes: usize,
522    /// Number of durability waiters.
523    pub waiters: usize,
524    /// Number of metadata resources examined.
525    pub resources_scanned: usize,
526    /// Number of metadata resources included in the counts.
527    pub resources_emitted: usize,
528    /// Whether resources were omitted by configured bounds.
529    pub truncated: bool,
530}
531
532/// The most recent shutdown result for one group population.
533#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
534pub struct RuntimeShutdownSectionV2 {
535    /// The population this result covers.
536    pub scope: RuntimeDiagnosticsScope,
537    /// Shutdown duration in milliseconds.
538    pub elapsed_millis: u64,
539    /// Number of tasks that were cancelled.
540    pub cancelled: usize,
541    /// Number of tasks that finished with an explicit failure.
542    pub failed: usize,
543    /// Number of tasks that did not finish inside the deadline.
544    pub timed_out: usize,
545    /// Number of tasks that were still tracked after shutdown.
546    pub leaked: usize,
547}
548
549/// One bounded task detail.
550///
551/// Carries a bounded category, the population it was observed in, and its
552/// elapsed time. Task names, group names, and identifiers are deliberately
553/// excluded because they are caller-provided labels.
554#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
555pub struct RuntimeTaskDetailV2 {
556    /// Bounded task category.
557    pub kind: RuntimeTaskKindV1,
558    /// The population the task was observed in.
559    pub scope: RuntimeDiagnosticsScope,
560    /// Elapsed time since the task started, in milliseconds.
561    pub elapsed_millis: u64,
562}
563
564/// Versioned, bounded, and sanitized runtime diagnostics with explicit scopes.
565///
566/// Unlike [`RuntimeDiagnosticsViewV1`], every section states the population it
567/// covers, the payload separates a section that has no input from a section that
568/// is empty, and a bounded detail list reports how much of the tree it examined
569/// instead of presenting a partial list as the whole runtime.
570#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
571pub struct RuntimeDiagnosticsViewV2 {
572    /// Schema identifier for this serialized diagnostics view.
573    pub schema_version: String,
574    /// UTC timestamp at which the snapshot was observed.
575    pub observed_at: DateTime<Utc>,
576    /// Bounded component category that owns the runtime.
577    pub component: RuntimeComponent,
578    /// Current lifecycle state of the root task group.
579    pub lifecycle_state: RuntimeLifecycleStateV1,
580    /// Bounded task counts and their scope.
581    pub tasks: RuntimeTaskSectionV2,
582    /// Bounded blocking state and its scope.
583    pub blocking: RuntimeBlockingSectionV2,
584    /// Bounded scheduled-work aggregates, absent when the caller owns none.
585    #[serde(default, skip_serializing_if = "Option::is_none")]
586    pub schedule: Option<RuntimeScheduleSectionV2>,
587    /// Bounded retained-metadata state, absent when the caller owns no actor.
588    #[serde(default, skip_serializing_if = "Option::is_none")]
589    pub metadata: Option<RuntimeMetadataSectionV2>,
590    /// The most recent shutdown result, absent until shutdown was attempted.
591    #[serde(default, skip_serializing_if = "Option::is_none")]
592    pub shutdown: Option<RuntimeShutdownSectionV2>,
593    /// Bounded task details, empty unless a detail list was requested.
594    pub details: Vec<RuntimeTaskDetailV2>,
595    /// The scan budget applied while collecting details.
596    pub detail_scan_budget: usize,
597    /// Number of tasks examined while collecting details.
598    pub details_scanned: usize,
599    /// Whether any section, including the detail list, was cut short.
600    pub truncated: bool,
601}
602
603/// A bounded detail sample that does not scan aggregate subtree statistics.
604///
605/// This separate API leaves the V1/V2 serialized views and their construction
606/// surfaces unchanged. Labels omit caller-provided names and identifiers.
607#[derive(Debug, Clone, Serialize)]
608pub struct RuntimeTaskDetails {
609    /// Sanitized details that fit the output budget.
610    pub details: Vec<RuntimeTaskDetailV2>,
611    /// Number of task entries examined.
612    pub tasks_scanned: usize,
613    /// Number of descendant registrations examined, including expired entries.
614    pub group_entries_scanned: usize,
615    /// Whether a task, group, or output budget truncated the sample.
616    pub truncated: bool,
617}
618
619impl From<crate::task_group::TaskDetailScan> for RuntimeTaskDetails {
620    fn from(scan: crate::task_group::TaskDetailScan) -> Self {
621        Self {
622            details: scan
623                .details
624                .into_iter()
625                .map(|detail| RuntimeTaskDetailV2 {
626                    kind: runtime_task_kind(detail.kind),
627                    scope: runtime_detail_scope(detail.scope),
628                    elapsed_millis: duration_millis(detail.elapsed),
629                })
630                .collect(),
631            tasks_scanned: scan.scanned,
632            group_entries_scanned: scan.group_entries_scanned,
633            truncated: scan.truncated,
634        }
635    }
636}
637
638impl RuntimeDiagnosticsViewV2 {
639    /// Stable schema identifier emitted by [`RuntimeDiagnostics::view_v2`].
640    pub const SCHEMA_VERSION: &'static str = "rocketmq.runtime-diagnostics.v2";
641}
642
643impl RuntimeDiagnostics {
644    /// Creates a bounded, explicitly scoped diagnostics view.
645    pub fn view_v2(
646        &self,
647        component: RuntimeComponent,
648        root: &TaskGroup,
649        blocking_lanes: Vec<BlockingExecutorSnapshot>,
650        inputs: RuntimeDiagnosticsInputs,
651    ) -> RuntimeDiagnosticsViewV2 {
652        self.view_v2_with_options(
653            component,
654            root,
655            blocking_lanes,
656            inputs,
657            RuntimeDiagnosticsViewOptionsV2::default(),
658        )
659    }
660
661    /// Creates a bounded, explicitly scoped diagnostics view using explicit
662    /// budget limits.
663    pub fn view_v2_with_options(
664        &self,
665        component: RuntimeComponent,
666        root: &TaskGroup,
667        blocking_lanes: Vec<BlockingExecutorSnapshot>,
668        inputs: RuntimeDiagnosticsInputs,
669        options: RuntimeDiagnosticsViewOptionsV2,
670    ) -> RuntimeDiagnosticsViewV2 {
671        self.view_v2_with_aggregates(
672            component,
673            root,
674            blocking_lanes.into_iter().map(Into::into).collect(),
675            inputs,
676            options,
677        )
678    }
679
680    pub(crate) fn view_v2_with_aggregates(
681        &self,
682        component: RuntimeComponent,
683        root: &TaskGroup,
684        blocking_lanes: Vec<BlockingExecutorAggregate>,
685        inputs: RuntimeDiagnosticsInputs,
686        options: RuntimeDiagnosticsViewOptionsV2,
687    ) -> RuntimeDiagnosticsViewV2 {
688        let (tasks, tasks_truncated) = task_section_v2(root, options);
689        let (blocking, blocking_truncated) = blocking_section_v2(blocking_lanes, options);
690        let schedule = (!inputs.schedule.is_empty()).then(|| schedule_section_v2(&inputs.schedule, options));
691        let metadata = inputs
692            .metadata
693            .as_ref()
694            .map(|snapshot| metadata_section_v2(snapshot, options));
695        let shutdown = inputs.shutdown.as_ref().map(shutdown_section_v2);
696        // A zero budget means no detail list was requested, so a routine sample
697        // neither pays for the scan nor reports truncation for a list nobody
698        // asked for.
699        let details = if options.max_detail_entries == 0 && options.detail_scan_budget == 0 {
700            crate::task_group::TaskDetailScan::default()
701        } else {
702            root.bounded_task_details(options.detail_scan_budget, options.max_detail_entries)
703        };
704        let details_truncated = details.truncated;
705
706        let schedule_truncated = schedule.as_ref().is_some_and(|section| section.truncated);
707        let metadata_truncated = metadata.as_ref().is_some_and(|section| section.truncated);
708
709        RuntimeDiagnosticsViewV2 {
710            schema_version: RuntimeDiagnosticsViewV2::SCHEMA_VERSION.to_string(),
711            observed_at: Utc::now(),
712            component,
713            lifecycle_state: runtime_lifecycle_state(root.lifecycle_state()),
714            tasks,
715            blocking,
716            schedule,
717            metadata,
718            shutdown,
719            details: details
720                .details
721                .into_iter()
722                .map(|detail| RuntimeTaskDetailV2 {
723                    kind: runtime_task_kind(detail.kind),
724                    scope: runtime_detail_scope(detail.scope),
725                    elapsed_millis: duration_millis(detail.elapsed),
726                })
727                .collect(),
728            detail_scan_budget: options.detail_scan_budget,
729            details_scanned: details.scanned,
730            truncated: tasks_truncated
731                || blocking_truncated
732                || schedule_truncated
733                || metadata_truncated
734                || details_truncated,
735        }
736    }
737}
738
739#[cfg(test)]
740mod tests {
741    use super::*;
742    use crate::RuntimeContext;
743
744    #[tokio::test]
745    async fn lane_identity_survives_partial_and_reordered_inputs_in_both_schemas() {
746        let context = RuntimeContext::from_current("lane-identity");
747        let diagnostics = RuntimeDiagnostics::new();
748        let snapshot = |lane: BlockingLane| BlockingExecutorSnapshot {
749            name: "private-lane".to_owned(),
750            lane,
751            max_concurrency: 8,
752            max_queue_depth: 32,
753            global_capacity: 8,
754            global_running: 0,
755            global_available: 8,
756            lane_reserved: 1,
757            lane_running: 0,
758            lane_borrowed: 0,
759            queued: lane.index() + 10,
760            running: 0,
761            timed_out_still_running: 0,
762            blocking_still_running: 0,
763            rejected: 0,
764            oldest_queue_wait: Duration::ZERO,
765            tasks: Vec::new(),
766        };
767        for lanes in [
768            vec![BlockingLane::MetadataIo],
769            vec![
770                BlockingLane::CpuCrypto,
771                BlockingLane::StorageIo,
772                BlockingLane::MetadataIo,
773            ],
774            vec![BlockingLane::MetadataIo, BlockingLane::CpuCrypto],
775            vec![],
776        ] {
777            let snapshots: Vec<_> = lanes.iter().copied().map(snapshot).collect();
778            let v1 = diagnostics.view_v1(RuntimeComponent::Broker, context.root_group(), snapshots.clone());
779            let v2 = diagnostics.view_v2(
780                RuntimeComponent::Broker,
781                context.root_group(),
782                snapshots,
783                RuntimeDiagnosticsInputs::default(),
784            );
785            assert_eq!(v1.blocking_lanes, v2.blocking.lanes);
786            assert_eq!(v1.blocking_lanes.len(), lanes.len());
787            for (summary, lane) in v1.blocking_lanes.iter().zip(lanes) {
788                let expected = match lane {
789                    BlockingLane::StorageIo => "storage_io",
790                    BlockingLane::MetadataIo => "metadata_io",
791                    BlockingLane::CpuCrypto => "cpu_crypto",
792                };
793                assert_eq!(serde_json::to_value(summary.lane).unwrap(), expected);
794                assert_eq!(summary.queued, lane.index() + 10);
795            }
796        }
797    }
798
799    #[tokio::test]
800    async fn every_task_kind_keeps_its_own_population_and_wire_label() {
801        let context = RuntimeContext::from_current("kind-population");
802        let root = context.root_group();
803        let labels = [
804            "service",
805            "worker",
806            "scheduled_driver",
807            "scheduled_run",
808            "blocking_reaper",
809            "shutdown",
810            "other",
811        ];
812        for (kind, count) in TaskKind::ALL.into_iter().zip(1..) {
813            for _ in 0..count {
814                let cancellation = root.cancellation_token();
815                root.spawn("pending", kind, async move { cancellation.cancelled().await })
816                    .unwrap();
817            }
818        }
819        let view = RuntimeDiagnostics::new().view_v1(RuntimeComponent::Broker, root, Vec::new());
820        assert_eq!(view.task_kinds.len(), labels.len());
821        for ((summary, label), count) in view.task_kinds.iter().zip(labels).zip(1..) {
822            assert_eq!(serde_json::to_value(summary.kind).unwrap(), label);
823            assert_eq!(summary.active, count);
824        }
825        assert!(root.shutdown(Duration::from_secs(1)).await.is_healthy());
826    }
827
828    #[tokio::test]
829    async fn sanitized_view_does_not_expose_runtime_or_task_names() {
830        let context = RuntimeContext::from_current("sensitive-root-name");
831        let child = context.service_context("sensitive-child-name");
832        child
833            .spawn("sensitive-task-name", TaskKind::Worker, std::future::pending())
834            .expect("task should spawn");
835        let diagnostics = RuntimeDiagnostics::new();
836
837        let view = diagnostics.view_v1(RuntimeComponent::Mcp, context.root_group(), Vec::new());
838        let json = serde_json::to_string(&view).expect("view should serialize");
839
840        assert!(!json.contains("sensitive-root-name"));
841        assert!(!json.contains("sensitive-child-name"));
842        assert!(!json.contains("sensitive-task-name"));
843        assert!(!json.contains("rocketmq-runtime-"));
844        assert!(json.contains("\"lifecycle_state\":\"open\""));
845        assert!(json.contains("\"kind\":\"worker\""));
846        assert!(view.task_group_count >= 2);
847        assert_eq!(view.task_count, 1);
848    }
849
850    #[tokio::test]
851    async fn zero_bounds_produce_an_explicit_truncated_view() {
852        let context = RuntimeContext::from_current("runtime-view-bounds");
853        context
854            .root_group()
855            .spawn("worker", TaskKind::Worker, std::future::pending())
856            .expect("task should spawn");
857        let diagnostics = RuntimeDiagnostics::new();
858        let view = diagnostics.view_v1_with_options(
859            RuntimeComponent::Other,
860            context.root_group(),
861            Vec::new(),
862            RuntimeDiagnosticsViewOptions {
863                max_task_kind_summaries: 0,
864                max_blocking_lane_summaries: 0,
865                ..RuntimeDiagnosticsViewOptions::default()
866            },
867        );
868
869        assert!(view.truncated);
870        assert!(view.task_kinds.is_empty());
871        assert!(view.blocking_lanes.is_empty());
872    }
873
874    #[tokio::test]
875    async fn sanitized_view_exposes_only_bounded_blocking_capacity_and_counts() {
876        let context = RuntimeContext::from_current("runtime-blocking-capacity");
877        let diagnostics = RuntimeDiagnostics::new();
878        let view = diagnostics.view_v1(
879            RuntimeComponent::Broker,
880            context.root_group(),
881            vec![BlockingExecutorSnapshot {
882                name: "sensitive-lane-name".to_owned(),
883                lane: crate::BlockingLane::StorageIo,
884                max_concurrency: 8,
885                max_queue_depth: 32,
886                global_capacity: 8,
887                global_running: 2,
888                global_available: 6,
889                lane_reserved: 8,
890                lane_running: 2,
891                lane_borrowed: 0,
892                queued: 3,
893                running: 2,
894                timed_out_still_running: 1,
895                blocking_still_running: 0,
896                rejected: 0,
897                oldest_queue_wait: Duration::ZERO,
898                tasks: Vec::new(),
899            }],
900        );
901
902        assert_eq!(view.blocking_lanes.len(), 1);
903        assert_eq!(view.blocking_lanes[0].max_concurrency, Some(8));
904        assert_eq!(view.blocking_lanes[0].max_queue_depth, Some(32));
905        let encoded = serde_json::to_string(&view).expect("view should serialize");
906        assert!(!encoded.contains("sensitive-lane-name"));
907        let decoded: RuntimeDiagnosticsViewV1 =
908            serde_json::from_str(&encoded).expect("versioned view should deserialize");
909        assert_eq!(decoded, view);
910
911        let mut legacy = serde_json::to_value(&view).expect("view should encode");
912        let lane = legacy
913            .get_mut("blocking_lanes")
914            .and_then(serde_json::Value::as_array_mut)
915            .and_then(|lanes| lanes.first_mut())
916            .and_then(serde_json::Value::as_object_mut)
917            .expect("blocking lane");
918        lane.remove("max_concurrency");
919        lane.remove("max_queue_depth");
920        let legacy: RuntimeDiagnosticsViewV1 =
921            serde_json::from_value(legacy).expect("older v1 view should remain readable");
922        assert_eq!(legacy.blocking_lanes[0].max_concurrency, None);
923        assert_eq!(legacy.blocking_lanes[0].max_queue_depth, None);
924    }
925
926    fn scheduled_snapshot(name: &str, runs: u64, skips: u64, max_elapsed_ms: u64) -> ScheduledTaskSnapshot {
927        ScheduledTaskSnapshot {
928            name: name.to_string(),
929            mode: crate::scheduled::ScheduleMode::FixedDelay,
930            running: true,
931            active_runs: 1,
932            runs,
933            skips,
934            overlaps: 0,
935            failures: 0,
936            last_drift_ms: 0,
937            last_elapsed_ms: max_elapsed_ms,
938            max_elapsed_ms,
939        }
940    }
941
942    fn metadata_snapshot(waiter_count: usize) -> MetadataIoSnapshot {
943        MetadataIoSnapshot {
944            accepting: true,
945            pending_operations: 2,
946            pending_bytes: 4_096,
947            max_pending_operations: 1_024,
948            max_pending_bytes: 65_536,
949            resources: vec![crate::metadata_io::MetadataIoResourceSnapshot {
950                resource: "sensitive-resource-name".into(),
951                target: None,
952                durable_generation: None,
953                in_flight_generation: None,
954                queued_generation: None,
955                waiter_count,
956            }],
957        }
958    }
959
960    #[tokio::test]
961    async fn v2_sections_state_their_scope_and_absent_inputs_stay_absent() {
962        let context = RuntimeContext::from_current("runtime-v2-scope");
963        context
964            .root_group()
965            .spawn("local-worker", TaskKind::Worker, std::future::pending())
966            .expect("task should spawn");
967        context
968            .service_context("runtime-v2-child")
969            .spawn("child-worker", TaskKind::Worker, std::future::pending())
970            .expect("task should spawn");
971        let diagnostics = RuntimeDiagnostics::new();
972
973        let view = diagnostics.view_v2(
974            RuntimeComponent::Broker,
975            context.root_group(),
976            Vec::new(),
977            RuntimeDiagnosticsInputs::default(),
978        );
979
980        assert_eq!(view.schema_version, "rocketmq.runtime-diagnostics.v2");
981        assert_eq!(view.tasks.scope, RuntimeDiagnosticsScope::Subtree);
982        assert_eq!(view.tasks.task_count, 2);
983        assert_eq!(view.tasks.local_task_count, 1);
984        assert_eq!(view.blocking.scope, RuntimeDiagnosticsScope::ProcessShared);
985        assert!(view.schedule.is_none());
986        assert!(view.metadata.is_none());
987        assert!(view.shutdown.is_none());
988        assert!(view.details.is_empty());
989        // No detail list was requested, so a routine sample reports no truncation.
990        assert!(!view.truncated);
991    }
992
993    #[tokio::test]
994    async fn v2_reports_component_owned_sections_with_their_scope() {
995        let context = RuntimeContext::from_current("runtime-v2-sections");
996        let diagnostics = RuntimeDiagnostics::new();
997
998        let view = diagnostics.view_v2(
999            RuntimeComponent::Broker,
1000            context.root_group(),
1001            Vec::new(),
1002            RuntimeDiagnosticsInputs {
1003                schedule: vec![scheduled_snapshot("sensitive-schedule", 4, 3, 250)],
1004                metadata: Some(metadata_snapshot(2)),
1005                shutdown: Some(ShutdownReport {
1006                    cancelled: 1,
1007                    failed: 1,
1008                    timed_out: 1,
1009                    leaked: 1,
1010                    ..ShutdownReport::new("sensitive-group", Duration::from_millis(120))
1011                }),
1012            },
1013        );
1014
1015        let schedule = view.schedule.clone().expect("a supplied schedule section is present");
1016        assert_eq!(schedule.scope, RuntimeDiagnosticsScope::Local);
1017        assert_eq!(schedule.tasks_scanned, 1);
1018        assert_eq!(schedule.tasks_emitted, 1);
1019        assert_eq!(schedule.runs, 4);
1020        assert_eq!(schedule.skips, 3);
1021        assert_eq!(schedule.max_elapsed_millis, 250);
1022        assert!(!schedule.truncated);
1023
1024        let metadata = view.metadata.clone().expect("a supplied metadata section is present");
1025        assert_eq!(metadata.scope, RuntimeDiagnosticsScope::Local);
1026        assert_eq!(metadata.retained_operations, 2);
1027        assert_eq!(metadata.retained_bytes, 4_096);
1028        assert_eq!(metadata.waiters, 2);
1029        assert!(metadata.accepting);
1030
1031        let shutdown = view.shutdown.clone().expect("a supplied shutdown section is present");
1032        assert_eq!(shutdown.scope, RuntimeDiagnosticsScope::Local);
1033        assert_eq!(shutdown.elapsed_millis, 120);
1034        assert_eq!(shutdown.failed, 1);
1035        assert_eq!(shutdown.timed_out, 1);
1036        assert_eq!(shutdown.leaked, 1);
1037
1038        let json = serde_json::to_string(&view).expect("view should serialize");
1039        assert!(!json.contains("sensitive"));
1040    }
1041
1042    #[tokio::test]
1043    async fn v2_truncation_reports_what_was_scanned_and_emitted() {
1044        let context = RuntimeContext::from_current("runtime-v2-truncation");
1045        let diagnostics = RuntimeDiagnostics::new();
1046        let schedule = vec![
1047            scheduled_snapshot("first", 1, 0, 10),
1048            scheduled_snapshot("second", 1, 0, 20),
1049            scheduled_snapshot("third", 1, 0, 30),
1050        ];
1051
1052        let view = diagnostics.view_v2_with_options(
1053            RuntimeComponent::Other,
1054            context.root_group(),
1055            Vec::new(),
1056            RuntimeDiagnosticsInputs {
1057                schedule,
1058                ..RuntimeDiagnosticsInputs::default()
1059            },
1060            RuntimeDiagnosticsViewOptionsV2 {
1061                max_schedule_tasks: 1,
1062                ..RuntimeDiagnosticsViewOptionsV2::default()
1063            },
1064        );
1065
1066        let section = view.schedule.expect("a supplied schedule section is present");
1067        assert!(section.truncated);
1068        assert_eq!(section.tasks_scanned, 3);
1069        assert_eq!(section.tasks_emitted, 1);
1070        assert_eq!(section.runs, 1);
1071        assert!(view.truncated);
1072    }
1073
1074    #[tokio::test]
1075    async fn v2_detail_lists_stay_bounded_and_redacted() {
1076        let context = RuntimeContext::from_current("sensitive-runtime-name");
1077        let child = context.service_context("sensitive-child-name");
1078        for index in 0..4 {
1079            child
1080                .spawn(
1081                    format!("sensitive-task-{index}"),
1082                    TaskKind::Worker,
1083                    std::future::pending(),
1084                )
1085                .expect("task should spawn");
1086        }
1087        let diagnostics = RuntimeDiagnostics::new();
1088
1089        let view = diagnostics.view_v2_with_options(
1090            RuntimeComponent::Mcp,
1091            context.root_group(),
1092            Vec::new(),
1093            RuntimeDiagnosticsInputs::default(),
1094            RuntimeDiagnosticsViewOptionsV2 {
1095                max_detail_entries: 2,
1096                detail_scan_budget: 4,
1097                ..RuntimeDiagnosticsViewOptionsV2::default()
1098            },
1099        );
1100
1101        assert_eq!(view.details.len(), 2);
1102        assert_eq!(view.details_scanned, 4);
1103        assert_eq!(view.detail_scan_budget, 4);
1104        assert!(view.truncated);
1105        assert!(
1106            view.details
1107                .iter()
1108                .all(|detail| detail.scope == RuntimeDiagnosticsScope::Subtree),
1109            "the tasks belong to a descendant group"
1110        );
1111
1112        let json = serde_json::to_string(&view).expect("view should serialize");
1113        assert!(!json.contains("sensitive"));
1114        assert!(!json.contains("rocketmq-runtime-"));
1115        let decoded: RuntimeDiagnosticsViewV2 = serde_json::from_str(&json).expect("versioned view should deserialize");
1116        assert_eq!(decoded, view);
1117    }
1118
1119    #[tokio::test]
1120    async fn v2_detail_scan_budget_bounds_the_scan_and_not_only_the_output() {
1121        let context = RuntimeContext::from_current("runtime-v2-scan-budget");
1122        for index in 0..4 {
1123            context
1124                .root_group()
1125                .spawn(format!("worker-{index}"), TaskKind::Worker, std::future::pending())
1126                .expect("task should spawn");
1127        }
1128        let diagnostics = RuntimeDiagnostics::new();
1129
1130        let view = diagnostics.view_v2_with_options(
1131            RuntimeComponent::Other,
1132            context.root_group(),
1133            Vec::new(),
1134            RuntimeDiagnosticsInputs::default(),
1135            RuntimeDiagnosticsViewOptionsV2 {
1136                max_detail_entries: 8,
1137                detail_scan_budget: 2,
1138                ..RuntimeDiagnosticsViewOptionsV2::default()
1139            },
1140        );
1141
1142        assert_eq!(view.details_scanned, 2);
1143        assert_eq!(view.details.len(), 2);
1144        assert!(view.truncated, "the scan stopped before every task was examined");
1145        assert!(view
1146            .details
1147            .iter()
1148            .all(|detail| detail.scope == RuntimeDiagnosticsScope::Local));
1149    }
1150
1151    #[tokio::test]
1152    async fn v2_detail_scan_budget_also_bounds_the_groups_it_visits() {
1153        let context = RuntimeContext::from_current("runtime-v2-group-budget");
1154        // Empty groups add no tasks, so only the group bound can stop the scan.
1155        let groups = (0..8)
1156            .map(|index| context.service_context(crate::ScopeId::try_new(format!("empty-{index}")).unwrap()))
1157            .collect::<Vec<_>>();
1158        let diagnostics = RuntimeDiagnostics::new();
1159        let view_with_budget = |detail_scan_budget| {
1160            diagnostics.view_v2_with_options(
1161                RuntimeComponent::Other,
1162                context.root_group(),
1163                Vec::new(),
1164                RuntimeDiagnosticsInputs::default(),
1165                RuntimeDiagnosticsViewOptionsV2 {
1166                    max_detail_entries: 8,
1167                    detail_scan_budget,
1168                    ..RuntimeDiagnosticsViewOptionsV2::default()
1169                },
1170            )
1171        };
1172
1173        let bounded = view_with_budget(3);
1174        assert_eq!(bounded.details_scanned, 0);
1175        assert!(bounded.details.is_empty());
1176        assert!(bounded.truncated, "the scan stopped before every group was visited");
1177        assert_eq!(
1178            bounded.tasks.task_group_count, 9,
1179            "the aggregate section still covers every group"
1180        );
1181
1182        let complete = view_with_budget(8);
1183        assert_eq!(complete.details_scanned, 0);
1184        assert!(!complete.truncated, "a budget covering every group completes the scan");
1185        drop(groups);
1186    }
1187
1188    #[tokio::test]
1189    async fn v2_view_is_a_snapshot_that_late_completions_cannot_rewrite() {
1190        let context = RuntimeContext::from_current("runtime-v2-frozen");
1191        let (release, wait) = tokio::sync::oneshot::channel::<()>();
1192        context
1193            .root_group()
1194            .spawn("finishes-later", TaskKind::Worker, async move {
1195                let _ = wait.await;
1196            })
1197            .expect("task should spawn");
1198        let diagnostics = RuntimeDiagnostics::new();
1199
1200        let view = diagnostics.view_v2(
1201            RuntimeComponent::Other,
1202            context.root_group(),
1203            Vec::new(),
1204            RuntimeDiagnosticsInputs::default(),
1205        );
1206        assert_eq!(view.tasks.task_count, 1);
1207
1208        release.send(()).expect("the task should still be waiting");
1209        for _ in 0..10_000 {
1210            if context.root_group().task_count() == 0 {
1211                break;
1212            }
1213            tokio::task::yield_now().await;
1214        }
1215
1216        assert_eq!(context.root_group().task_count(), 0, "the task really finished");
1217        assert_eq!(
1218            view.tasks.task_count, 1,
1219            "the earlier view keeps what it observed instead of following late completions"
1220        );
1221    }
1222}
rocketmq_runtime/diagnostics.rs

rocketmq_runtime/
diagnostics.rs
