mod conversion;
use conversion::*;
use std::sync::atomic::AtomicU64;
use std::sync::atomic::Ordering;
use std::sync::Arc;
use std::time::Duration;
use chrono::DateTime;
use chrono::Utc;
use serde::Deserialize;
use serde::Serialize;
use crate::blocking::BlockingExecutorAggregate;
use crate::blocking::BlockingExecutorSnapshot;
use crate::blocking::BlockingKind;
use crate::blocking::BlockingLane;
use crate::metadata_io::MetadataIoSnapshot;
use crate::scheduled::ScheduledTaskSnapshot;
use crate::shutdown_report::ShutdownReport;
use crate::task_group::TaskDetailScope;
use crate::task_group::TaskGroup;
use crate::task_group::TaskGroupEventCounts;
use crate::task_group::TaskGroupId;
use crate::task_group::TaskGroupLifecycleState;
use crate::task_group::TaskKind;
static NEXT_RUNTIME_DIAGNOSTICS_ID: AtomicU64 = AtomicU64::new(1);
#[derive(Debug, Clone)]
pub struct RuntimeDiagnostics {
runtime_id: Arc<str>,
}
#[derive(Debug, Clone, Serialize)]
pub struct RuntimeDiagnosticsSnapshot {
pub runtime_id: String,
pub root_name: String,
pub group_id: TaskGroupId,
pub parent_group_id: Option<TaskGroupId>,
pub lifecycle_state: TaskGroupLifecycleState,
pub task_count: usize,
pub child_count: usize,
pub active_tasks: usize,
pub registry_slots: usize,
pub events: TaskGroupEventCounts,
pub blocking_lanes: Vec<BlockingExecutorSnapshot>,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum RuntimeComponent {
Broker,
NameServer,
Controller,
Proxy,
Mcp,
SreControlPlane,
SreConnector,
Other,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum RuntimeLifecycleStateV1 {
Open,
Closing,
Closed,
ShutdownCompleted,
Poisoned,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum RuntimeTaskKindV1 {
Service,
Worker,
ScheduledDriver,
ScheduledRun,
BlockingReaper,
Shutdown,
Other,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum RuntimeBlockingLaneV1 {
StorageIo,
MetadataIo,
CpuCrypto,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum RuntimeBlockingKindV1 {
ShortIo,
CpuBound,
LongRunning,
}
#[derive(Debug, Clone, Copy)]
pub struct RuntimeDiagnosticsViewOptions {
pub long_running_threshold: Duration,
pub max_task_kind_summaries: usize,
pub max_blocking_lane_summaries: usize,
}
impl Default for RuntimeDiagnosticsViewOptions {
fn default() -> Self {
Self {
long_running_threshold: Duration::from_secs(30),
max_task_kind_summaries: TaskKind::COUNT,
max_blocking_lane_summaries: BlockingLane::ALL.len(),
}
}
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct RuntimeDiagnosticsViewV1 {
pub schema_version: String,
pub observed_at: DateTime<Utc>,
pub component: RuntimeComponent,
pub lifecycle_state: RuntimeLifecycleStateV1,
pub task_group_count: usize,
pub task_count: usize,
pub task_kinds: Vec<RuntimeTaskKindSummaryV1>,
pub blocking_lanes: Vec<RuntimeBlockingLaneSummaryV1>,
pub truncated: bool,
}
impl RuntimeDiagnosticsViewV1 {
pub const SCHEMA_VERSION: &'static str = "rocketmq.runtime-diagnostics.v1";
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct RuntimeTaskKindSummaryV1 {
pub kind: RuntimeTaskKindV1,
pub active: usize,
pub long_running: usize,
pub max_elapsed_millis: u64,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct RuntimeBlockingLaneSummaryV1 {
pub lane: RuntimeBlockingLaneV1,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub max_concurrency: Option<usize>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub max_queue_depth: Option<usize>,
pub queued: usize,
pub running: usize,
pub timed_out_still_running: usize,
pub blocking_still_running: usize,
pub task_kinds: Vec<RuntimeBlockingKindSummaryV1>,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct RuntimeBlockingKindSummaryV1 {
pub kind: RuntimeBlockingKindV1,
pub active: usize,
pub max_elapsed_millis: u64,
}
impl RuntimeDiagnostics {
pub(crate) fn new() -> Self {
let runtime_id = NEXT_RUNTIME_DIAGNOSTICS_ID.fetch_add(1, Ordering::Relaxed);
Self {
runtime_id: Arc::from(format!("rocketmq-runtime-{runtime_id}")),
}
}
pub fn runtime_id(&self) -> &str {
&self.runtime_id
}
pub fn snapshot(
&self,
root: &TaskGroup,
blocking_lanes: Vec<BlockingExecutorSnapshot>,
) -> RuntimeDiagnosticsSnapshot {
RuntimeDiagnosticsSnapshot {
runtime_id: self.runtime_id.to_string(),
root_name: root.name().to_string(),
group_id: root.id(),
parent_group_id: root.parent_id(),
lifecycle_state: root.lifecycle_state(),
task_count: root.task_count(),
child_count: root.component_count(),
active_tasks: root.task_count(),
registry_slots: root.component_count(),
events: root.event_counts(),
blocking_lanes,
}
}
pub fn view_v1(
&self,
component: RuntimeComponent,
root: &TaskGroup,
blocking_lanes: Vec<BlockingExecutorSnapshot>,
) -> RuntimeDiagnosticsViewV1 {
self.view_v1_with_options(
component,
root,
blocking_lanes,
RuntimeDiagnosticsViewOptions::default(),
)
}
pub fn view_v1_with_options(
&self,
component: RuntimeComponent,
root: &TaskGroup,
blocking_lanes: Vec<BlockingExecutorSnapshot>,
options: RuntimeDiagnosticsViewOptions,
) -> RuntimeDiagnosticsViewV1 {
self.view_v1_with_aggregates(
component,
root,
blocking_lanes.into_iter().map(Into::into).collect(),
options,
)
}
pub(crate) fn view_v1_with_aggregates(
&self,
component: RuntimeComponent,
root: &TaskGroup,
blocking_lanes: Vec<BlockingExecutorAggregate>,
options: RuntimeDiagnosticsViewOptions,
) -> RuntimeDiagnosticsViewV1 {
let task_diagnostics = root.diagnostics(options.long_running_threshold);
let task_kind_count = task_diagnostics.task_kinds.len();
let task_kinds = task_diagnostics
.task_kinds
.into_iter()
.take(options.max_task_kind_summaries)
.map(|summary| RuntimeTaskKindSummaryV1 {
kind: runtime_task_kind(summary.kind),
active: summary.active,
long_running: summary.long_running,
max_elapsed_millis: duration_millis(summary.max_elapsed),
})
.collect();
let blocking_lane_count = blocking_lanes.len();
let blocking_lanes = blocking_lanes
.into_iter()
.take(options.max_blocking_lane_summaries.min(BlockingLane::ALL.len()))
.map(sanitize_blocking_lane)
.collect();
RuntimeDiagnosticsViewV1 {
schema_version: RuntimeDiagnosticsViewV1::SCHEMA_VERSION.to_string(),
observed_at: Utc::now(),
component,
lifecycle_state: runtime_lifecycle_state(root.lifecycle_state()),
task_group_count: task_diagnostics.group_count,
task_count: task_diagnostics.task_count,
task_kinds,
blocking_lanes,
truncated: task_kind_count > options.max_task_kind_summaries
|| blocking_lane_count > options.max_blocking_lane_summaries.min(3),
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum RuntimeDiagnosticsScope {
Local,
Subtree,
ProcessShared,
}
#[derive(Debug, Clone, Copy)]
pub struct RuntimeDiagnosticsViewOptionsV2 {
pub long_running_threshold: Duration,
pub max_task_kind_summaries: usize,
pub max_blocking_lane_summaries: usize,
pub max_schedule_tasks: usize,
pub max_metadata_resources: usize,
pub max_detail_entries: usize,
pub detail_scan_budget: usize,
}
impl Default for RuntimeDiagnosticsViewOptionsV2 {
fn default() -> Self {
Self {
long_running_threshold: Duration::from_secs(30),
max_task_kind_summaries: TaskKind::COUNT,
max_blocking_lane_summaries: BlockingLane::ALL.len(),
max_schedule_tasks: 16,
max_metadata_resources: 32,
max_detail_entries: 0,
detail_scan_budget: 0,
}
}
}
#[derive(Debug, Clone, Default)]
pub struct RuntimeDiagnosticsInputs {
pub schedule: Vec<ScheduledTaskSnapshot>,
pub metadata: Option<MetadataIoSnapshot>,
pub shutdown: Option<ShutdownReport>,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct RuntimeTaskSectionV2 {
pub scope: RuntimeDiagnosticsScope,
pub task_group_count: usize,
pub task_count: usize,
pub local_task_count: usize,
pub long_running: usize,
pub max_elapsed_millis: u64,
pub task_kinds: Vec<RuntimeTaskKindSummaryV1>,
pub truncated: bool,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct RuntimeBlockingSectionV2 {
pub scope: RuntimeDiagnosticsScope,
pub queued: usize,
pub running: usize,
pub timed_out_still_running: usize,
pub blocking_still_running: usize,
pub lanes: Vec<RuntimeBlockingLaneSummaryV1>,
pub truncated: bool,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct RuntimeScheduleSectionV2 {
pub scope: RuntimeDiagnosticsScope,
pub tasks_scanned: usize,
pub tasks_emitted: usize,
pub active_runs: u64,
pub runs: u64,
pub skips: u64,
pub overlaps: u64,
pub failures: u64,
pub max_elapsed_millis: u64,
pub truncated: bool,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct RuntimeMetadataSectionV2 {
pub scope: RuntimeDiagnosticsScope,
pub accepting: bool,
pub retained_operations: usize,
pub retained_bytes: usize,
pub waiters: usize,
pub resources_scanned: usize,
pub resources_emitted: usize,
pub truncated: bool,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct RuntimeShutdownSectionV2 {
pub scope: RuntimeDiagnosticsScope,
pub elapsed_millis: u64,
pub cancelled: usize,
pub failed: usize,
pub timed_out: usize,
pub leaked: usize,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct RuntimeTaskDetailV2 {
pub kind: RuntimeTaskKindV1,
pub scope: RuntimeDiagnosticsScope,
pub elapsed_millis: u64,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct RuntimeDiagnosticsViewV2 {
pub schema_version: String,
pub observed_at: DateTime<Utc>,
pub component: RuntimeComponent,
pub lifecycle_state: RuntimeLifecycleStateV1,
pub tasks: RuntimeTaskSectionV2,
pub blocking: RuntimeBlockingSectionV2,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub schedule: Option<RuntimeScheduleSectionV2>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub metadata: Option<RuntimeMetadataSectionV2>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub shutdown: Option<RuntimeShutdownSectionV2>,
pub details: Vec<RuntimeTaskDetailV2>,
pub detail_scan_budget: usize,
pub details_scanned: usize,
pub truncated: bool,
}
#[derive(Debug, Clone, Serialize)]
pub struct RuntimeTaskDetails {
pub details: Vec<RuntimeTaskDetailV2>,
pub tasks_scanned: usize,
pub group_entries_scanned: usize,
pub truncated: bool,
}
impl From<crate::task_group::TaskDetailScan> for RuntimeTaskDetails {
fn from(scan: crate::task_group::TaskDetailScan) -> Self {
Self {
details: scan
.details
.into_iter()
.map(|detail| RuntimeTaskDetailV2 {
kind: runtime_task_kind(detail.kind),
scope: runtime_detail_scope(detail.scope),
elapsed_millis: duration_millis(detail.elapsed),
})
.collect(),
tasks_scanned: scan.scanned,
group_entries_scanned: scan.group_entries_scanned,
truncated: scan.truncated,
}
}
}
impl RuntimeDiagnosticsViewV2 {
pub const SCHEMA_VERSION: &'static str = "rocketmq.runtime-diagnostics.v2";
}
impl RuntimeDiagnostics {
pub fn view_v2(
&self,
component: RuntimeComponent,
root: &TaskGroup,
blocking_lanes: Vec<BlockingExecutorSnapshot>,
inputs: RuntimeDiagnosticsInputs,
) -> RuntimeDiagnosticsViewV2 {
self.view_v2_with_options(
component,
root,
blocking_lanes,
inputs,
RuntimeDiagnosticsViewOptionsV2::default(),
)
}
pub fn view_v2_with_options(
&self,
component: RuntimeComponent,
root: &TaskGroup,
blocking_lanes: Vec<BlockingExecutorSnapshot>,
inputs: RuntimeDiagnosticsInputs,
options: RuntimeDiagnosticsViewOptionsV2,
) -> RuntimeDiagnosticsViewV2 {
self.view_v2_with_aggregates(
component,
root,
blocking_lanes.into_iter().map(Into::into).collect(),
inputs,
options,
)
}
pub(crate) fn view_v2_with_aggregates(
&self,
component: RuntimeComponent,
root: &TaskGroup,
blocking_lanes: Vec<BlockingExecutorAggregate>,
inputs: RuntimeDiagnosticsInputs,
options: RuntimeDiagnosticsViewOptionsV2,
) -> RuntimeDiagnosticsViewV2 {
let (tasks, tasks_truncated) = task_section_v2(root, options);
let (blocking, blocking_truncated) = blocking_section_v2(blocking_lanes, options);
let schedule = (!inputs.schedule.is_empty()).then(|| schedule_section_v2(&inputs.schedule, options));
let metadata = inputs
.metadata
.as_ref()
.map(|snapshot| metadata_section_v2(snapshot, options));
let shutdown = inputs.shutdown.as_ref().map(shutdown_section_v2);
let details = if options.max_detail_entries == 0 && options.detail_scan_budget == 0 {
crate::task_group::TaskDetailScan::default()
} else {
root.bounded_task_details(options.detail_scan_budget, options.max_detail_entries)
};
let details_truncated = details.truncated;
let schedule_truncated = schedule.as_ref().is_some_and(|section| section.truncated);
let metadata_truncated = metadata.as_ref().is_some_and(|section| section.truncated);
RuntimeDiagnosticsViewV2 {
schema_version: RuntimeDiagnosticsViewV2::SCHEMA_VERSION.to_string(),
observed_at: Utc::now(),
component,
lifecycle_state: runtime_lifecycle_state(root.lifecycle_state()),
tasks,
blocking,
schedule,
metadata,
shutdown,
details: details
.details
.into_iter()
.map(|detail| RuntimeTaskDetailV2 {
kind: runtime_task_kind(detail.kind),
scope: runtime_detail_scope(detail.scope),
elapsed_millis: duration_millis(detail.elapsed),
})
.collect(),
detail_scan_budget: options.detail_scan_budget,
details_scanned: details.scanned,
truncated: tasks_truncated
|| blocking_truncated
|| schedule_truncated
|| metadata_truncated
|| details_truncated,
}
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::RuntimeContext;
#[tokio::test]
async fn lane_identity_survives_partial_and_reordered_inputs_in_both_schemas() {
let context = RuntimeContext::from_current("lane-identity");
let diagnostics = RuntimeDiagnostics::new();
let snapshot = |lane: BlockingLane| BlockingExecutorSnapshot {
name: "private-lane".to_owned(),
lane,
max_concurrency: 8,
max_queue_depth: 32,
global_capacity: 8,
global_running: 0,
global_available: 8,
lane_reserved: 1,
lane_running: 0,
lane_borrowed: 0,
queued: lane.index() + 10,
running: 0,
timed_out_still_running: 0,
blocking_still_running: 0,
rejected: 0,
oldest_queue_wait: Duration::ZERO,
tasks: Vec::new(),
};
for lanes in [
vec![BlockingLane::MetadataIo],
vec![
BlockingLane::CpuCrypto,
BlockingLane::StorageIo,
BlockingLane::MetadataIo,
],
vec![BlockingLane::MetadataIo, BlockingLane::CpuCrypto],
vec![],
] {
let snapshots: Vec<_> = lanes.iter().copied().map(snapshot).collect();
let v1 = diagnostics.view_v1(RuntimeComponent::Broker, context.root_group(), snapshots.clone());
let v2 = diagnostics.view_v2(
RuntimeComponent::Broker,
context.root_group(),
snapshots,
RuntimeDiagnosticsInputs::default(),
);
assert_eq!(v1.blocking_lanes, v2.blocking.lanes);
assert_eq!(v1.blocking_lanes.len(), lanes.len());
for (summary, lane) in v1.blocking_lanes.iter().zip(lanes) {
let expected = match lane {
BlockingLane::StorageIo => "storage_io",
BlockingLane::MetadataIo => "metadata_io",
BlockingLane::CpuCrypto => "cpu_crypto",
};
assert_eq!(serde_json::to_value(summary.lane).unwrap(), expected);
assert_eq!(summary.queued, lane.index() + 10);
}
}
}
#[tokio::test]
async fn every_task_kind_keeps_its_own_population_and_wire_label() {
let context = RuntimeContext::from_current("kind-population");
let root = context.root_group();
let labels = [
"service",
"worker",
"scheduled_driver",
"scheduled_run",
"blocking_reaper",
"shutdown",
"other",
];
for (kind, count) in TaskKind::ALL.into_iter().zip(1..) {
for _ in 0..count {
let cancellation = root.cancellation_token();
root.spawn("pending", kind, async move { cancellation.cancelled().await })
.unwrap();
}
}
let view = RuntimeDiagnostics::new().view_v1(RuntimeComponent::Broker, root, Vec::new());
assert_eq!(view.task_kinds.len(), labels.len());
for ((summary, label), count) in view.task_kinds.iter().zip(labels).zip(1..) {
assert_eq!(serde_json::to_value(summary.kind).unwrap(), label);
assert_eq!(summary.active, count);
}
assert!(root.shutdown(Duration::from_secs(1)).await.is_healthy());
}
#[tokio::test]
async fn sanitized_view_does_not_expose_runtime_or_task_names() {
let context = RuntimeContext::from_current("sensitive-root-name");
let child = context.service_context("sensitive-child-name");
child
.spawn("sensitive-task-name", TaskKind::Worker, std::future::pending())
.expect("task should spawn");
let diagnostics = RuntimeDiagnostics::new();
let view = diagnostics.view_v1(RuntimeComponent::Mcp, context.root_group(), Vec::new());
let json = serde_json::to_string(&view).expect("view should serialize");
assert!(!json.contains("sensitive-root-name"));
assert!(!json.contains("sensitive-child-name"));
assert!(!json.contains("sensitive-task-name"));
assert!(!json.contains("rocketmq-runtime-"));
assert!(json.contains("\"lifecycle_state\":\"open\""));
assert!(json.contains("\"kind\":\"worker\""));
assert!(view.task_group_count >= 2);
assert_eq!(view.task_count, 1);
}
#[tokio::test]
async fn zero_bounds_produce_an_explicit_truncated_view() {
let context = RuntimeContext::from_current("runtime-view-bounds");
context
.root_group()
.spawn("worker", TaskKind::Worker, std::future::pending())
.expect("task should spawn");
let diagnostics = RuntimeDiagnostics::new();
let view = diagnostics.view_v1_with_options(
RuntimeComponent::Other,
context.root_group(),
Vec::new(),
RuntimeDiagnosticsViewOptions {
max_task_kind_summaries: 0,
max_blocking_lane_summaries: 0,
..RuntimeDiagnosticsViewOptions::default()
},
);
assert!(view.truncated);
assert!(view.task_kinds.is_empty());
assert!(view.blocking_lanes.is_empty());
}
#[tokio::test]
async fn sanitized_view_exposes_only_bounded_blocking_capacity_and_counts() {
let context = RuntimeContext::from_current("runtime-blocking-capacity");
let diagnostics = RuntimeDiagnostics::new();
let view = diagnostics.view_v1(
RuntimeComponent::Broker,
context.root_group(),
vec![BlockingExecutorSnapshot {
name: "sensitive-lane-name".to_owned(),
lane: crate::BlockingLane::StorageIo,
max_concurrency: 8,
max_queue_depth: 32,
global_capacity: 8,
global_running: 2,
global_available: 6,
lane_reserved: 8,
lane_running: 2,
lane_borrowed: 0,
queued: 3,
running: 2,
timed_out_still_running: 1,
blocking_still_running: 0,
rejected: 0,
oldest_queue_wait: Duration::ZERO,
tasks: Vec::new(),
}],
);
assert_eq!(view.blocking_lanes.len(), 1);
assert_eq!(view.blocking_lanes[0].max_concurrency, Some(8));
assert_eq!(view.blocking_lanes[0].max_queue_depth, Some(32));
let encoded = serde_json::to_string(&view).expect("view should serialize");
assert!(!encoded.contains("sensitive-lane-name"));
let decoded: RuntimeDiagnosticsViewV1 =
serde_json::from_str(&encoded).expect("versioned view should deserialize");
assert_eq!(decoded, view);
let mut legacy = serde_json::to_value(&view).expect("view should encode");
let lane = legacy
.get_mut("blocking_lanes")
.and_then(serde_json::Value::as_array_mut)
.and_then(|lanes| lanes.first_mut())
.and_then(serde_json::Value::as_object_mut)
.expect("blocking lane");
lane.remove("max_concurrency");
lane.remove("max_queue_depth");
let legacy: RuntimeDiagnosticsViewV1 =
serde_json::from_value(legacy).expect("older v1 view should remain readable");
assert_eq!(legacy.blocking_lanes[0].max_concurrency, None);
assert_eq!(legacy.blocking_lanes[0].max_queue_depth, None);
}
fn scheduled_snapshot(name: &str, runs: u64, skips: u64, max_elapsed_ms: u64) -> ScheduledTaskSnapshot {
ScheduledTaskSnapshot {
name: name.to_string(),
mode: crate::scheduled::ScheduleMode::FixedDelay,
running: true,
active_runs: 1,
runs,
skips,
overlaps: 0,
failures: 0,
last_drift_ms: 0,
last_elapsed_ms: max_elapsed_ms,
max_elapsed_ms,
}
}
fn metadata_snapshot(waiter_count: usize) -> MetadataIoSnapshot {
MetadataIoSnapshot {
accepting: true,
pending_operations: 2,
pending_bytes: 4_096,
max_pending_operations: 1_024,
max_pending_bytes: 65_536,
resources: vec![crate::metadata_io::MetadataIoResourceSnapshot {
resource: "sensitive-resource-name".into(),
target: None,
durable_generation: None,
in_flight_generation: None,
queued_generation: None,
waiter_count,
}],
}
}
#[tokio::test]
async fn v2_sections_state_their_scope_and_absent_inputs_stay_absent() {
let context = RuntimeContext::from_current("runtime-v2-scope");
context
.root_group()
.spawn("local-worker", TaskKind::Worker, std::future::pending())
.expect("task should spawn");
context
.service_context("runtime-v2-child")
.spawn("child-worker", TaskKind::Worker, std::future::pending())
.expect("task should spawn");
let diagnostics = RuntimeDiagnostics::new();
let view = diagnostics.view_v2(
RuntimeComponent::Broker,
context.root_group(),
Vec::new(),
RuntimeDiagnosticsInputs::default(),
);
assert_eq!(view.schema_version, "rocketmq.runtime-diagnostics.v2");
assert_eq!(view.tasks.scope, RuntimeDiagnosticsScope::Subtree);
assert_eq!(view.tasks.task_count, 2);
assert_eq!(view.tasks.local_task_count, 1);
assert_eq!(view.blocking.scope, RuntimeDiagnosticsScope::ProcessShared);
assert!(view.schedule.is_none());
assert!(view.metadata.is_none());
assert!(view.shutdown.is_none());
assert!(view.details.is_empty());
assert!(!view.truncated);
}
#[tokio::test]
async fn v2_reports_component_owned_sections_with_their_scope() {
let context = RuntimeContext::from_current("runtime-v2-sections");
let diagnostics = RuntimeDiagnostics::new();
let view = diagnostics.view_v2(
RuntimeComponent::Broker,
context.root_group(),
Vec::new(),
RuntimeDiagnosticsInputs {
schedule: vec![scheduled_snapshot("sensitive-schedule", 4, 3, 250)],
metadata: Some(metadata_snapshot(2)),
shutdown: Some(ShutdownReport {
cancelled: 1,
failed: 1,
timed_out: 1,
leaked: 1,
..ShutdownReport::new("sensitive-group", Duration::from_millis(120))
}),
},
);
let schedule = view.schedule.clone().expect("a supplied schedule section is present");
assert_eq!(schedule.scope, RuntimeDiagnosticsScope::Local);
assert_eq!(schedule.tasks_scanned, 1);
assert_eq!(schedule.tasks_emitted, 1);
assert_eq!(schedule.runs, 4);
assert_eq!(schedule.skips, 3);
assert_eq!(schedule.max_elapsed_millis, 250);
assert!(!schedule.truncated);
let metadata = view.metadata.clone().expect("a supplied metadata section is present");
assert_eq!(metadata.scope, RuntimeDiagnosticsScope::Local);
assert_eq!(metadata.retained_operations, 2);
assert_eq!(metadata.retained_bytes, 4_096);
assert_eq!(metadata.waiters, 2);
assert!(metadata.accepting);
let shutdown = view.shutdown.clone().expect("a supplied shutdown section is present");
assert_eq!(shutdown.scope, RuntimeDiagnosticsScope::Local);
assert_eq!(shutdown.elapsed_millis, 120);
assert_eq!(shutdown.failed, 1);
assert_eq!(shutdown.timed_out, 1);
assert_eq!(shutdown.leaked, 1);
let json = serde_json::to_string(&view).expect("view should serialize");
assert!(!json.contains("sensitive"));
}
#[tokio::test]
async fn v2_truncation_reports_what_was_scanned_and_emitted() {
let context = RuntimeContext::from_current("runtime-v2-truncation");
let diagnostics = RuntimeDiagnostics::new();
let schedule = vec![
scheduled_snapshot("first", 1, 0, 10),
scheduled_snapshot("second", 1, 0, 20),
scheduled_snapshot("third", 1, 0, 30),
];
let view = diagnostics.view_v2_with_options(
RuntimeComponent::Other,
context.root_group(),
Vec::new(),
RuntimeDiagnosticsInputs {
schedule,
..RuntimeDiagnosticsInputs::default()
},
RuntimeDiagnosticsViewOptionsV2 {
max_schedule_tasks: 1,
..RuntimeDiagnosticsViewOptionsV2::default()
},
);
let section = view.schedule.expect("a supplied schedule section is present");
assert!(section.truncated);
assert_eq!(section.tasks_scanned, 3);
assert_eq!(section.tasks_emitted, 1);
assert_eq!(section.runs, 1);
assert!(view.truncated);
}
#[tokio::test]
async fn v2_detail_lists_stay_bounded_and_redacted() {
let context = RuntimeContext::from_current("sensitive-runtime-name");
let child = context.service_context("sensitive-child-name");
for index in 0..4 {
child
.spawn(
format!("sensitive-task-{index}"),
TaskKind::Worker,
std::future::pending(),
)
.expect("task should spawn");
}
let diagnostics = RuntimeDiagnostics::new();
let view = diagnostics.view_v2_with_options(
RuntimeComponent::Mcp,
context.root_group(),
Vec::new(),
RuntimeDiagnosticsInputs::default(),
RuntimeDiagnosticsViewOptionsV2 {
max_detail_entries: 2,
detail_scan_budget: 4,
..RuntimeDiagnosticsViewOptionsV2::default()
},
);
assert_eq!(view.details.len(), 2);
assert_eq!(view.details_scanned, 4);
assert_eq!(view.detail_scan_budget, 4);
assert!(view.truncated);
assert!(
view.details
.iter()
.all(|detail| detail.scope == RuntimeDiagnosticsScope::Subtree),
"the tasks belong to a descendant group"
);
let json = serde_json::to_string(&view).expect("view should serialize");
assert!(!json.contains("sensitive"));
assert!(!json.contains("rocketmq-runtime-"));
let decoded: RuntimeDiagnosticsViewV2 = serde_json::from_str(&json).expect("versioned view should deserialize");
assert_eq!(decoded, view);
}
#[tokio::test]
async fn v2_detail_scan_budget_bounds_the_scan_and_not_only_the_output() {
let context = RuntimeContext::from_current("runtime-v2-scan-budget");
for index in 0..4 {
context
.root_group()
.spawn(format!("worker-{index}"), TaskKind::Worker, std::future::pending())
.expect("task should spawn");
}
let diagnostics = RuntimeDiagnostics::new();
let view = diagnostics.view_v2_with_options(
RuntimeComponent::Other,
context.root_group(),
Vec::new(),
RuntimeDiagnosticsInputs::default(),
RuntimeDiagnosticsViewOptionsV2 {
max_detail_entries: 8,
detail_scan_budget: 2,
..RuntimeDiagnosticsViewOptionsV2::default()
},
);
assert_eq!(view.details_scanned, 2);
assert_eq!(view.details.len(), 2);
assert!(view.truncated, "the scan stopped before every task was examined");
assert!(view
.details
.iter()
.all(|detail| detail.scope == RuntimeDiagnosticsScope::Local));
}
#[tokio::test]
async fn v2_detail_scan_budget_also_bounds_the_groups_it_visits() {
let context = RuntimeContext::from_current("runtime-v2-group-budget");
let groups = (0..8)
.map(|index| context.service_context(crate::ScopeId::try_new(format!("empty-{index}")).unwrap()))
.collect::<Vec<_>>();
let diagnostics = RuntimeDiagnostics::new();
let view_with_budget = |detail_scan_budget| {
diagnostics.view_v2_with_options(
RuntimeComponent::Other,
context.root_group(),
Vec::new(),
RuntimeDiagnosticsInputs::default(),
RuntimeDiagnosticsViewOptionsV2 {
max_detail_entries: 8,
detail_scan_budget,
..RuntimeDiagnosticsViewOptionsV2::default()
},
)
};
let bounded = view_with_budget(3);
assert_eq!(bounded.details_scanned, 0);
assert!(bounded.details.is_empty());
assert!(bounded.truncated, "the scan stopped before every group was visited");
assert_eq!(
bounded.tasks.task_group_count, 9,
"the aggregate section still covers every group"
);
let complete = view_with_budget(8);
assert_eq!(complete.details_scanned, 0);
assert!(!complete.truncated, "a budget covering every group completes the scan");
drop(groups);
}
#[tokio::test]
async fn v2_view_is_a_snapshot_that_late_completions_cannot_rewrite() {
let context = RuntimeContext::from_current("runtime-v2-frozen");
let (release, wait) = tokio::sync::oneshot::channel::<()>();
context
.root_group()
.spawn("finishes-later", TaskKind::Worker, async move {
let _ = wait.await;
})
.expect("task should spawn");
let diagnostics = RuntimeDiagnostics::new();
let view = diagnostics.view_v2(
RuntimeComponent::Other,
context.root_group(),
Vec::new(),
RuntimeDiagnosticsInputs::default(),
);
assert_eq!(view.tasks.task_count, 1);
release.send(()).expect("the task should still be waiting");
for _ in 0..10_000 {
if context.root_group().task_count() == 0 {
break;
}
tokio::task::yield_now().await;
}
assert_eq!(context.root_group().task_count(), 0, "the task really finished");
assert_eq!(
view.tasks.task_count, 1,
"the earlier view keeps what it observed instead of following late completions"
);
}
}