inspector_analysis/
contention.rs

1//! Resource-contention rule: scheduling delay growing while tasks pile up.
2//!
3//! Signature this detects: median per-task `scheduled_ns` (time spent woken but
4//! not yet polled) rises sharply between snapshots while busy work stays flat —
5//! tasks are queueing on a contended resource (pool, lock, single-threaded hot
6//! path) rather than doing slow work themselves. Pure function, no I/O, no LLM.
7use inspector_core::{Diagnosis, FindingStatus, Snapshot};
8
9/// Median scheduled-time must grow by at least this ratio to fire.
10pub const DEFAULT_SCHED_RATIO: f64 = 3.0;
11/// Minimum tasks with scheduled data required in each snapshot.
12pub const MIN_TASKS: usize = 5;
13
14pub fn detect_contention(
15    base: &Snapshot,
16    current: &Snapshot,
17    base_id: &str,
18    current_id: &str,
19) -> Diagnosis {
20    detect_contention_with_threshold(base, current, base_id, current_id, DEFAULT_SCHED_RATIO)
21}
22
23pub fn detect_contention_with_threshold(
24    base: &Snapshot,
25    current: &Snapshot,
26    base_id: &str,
27    current_id: &str,
28    threshold_ratio: f64,
29) -> Diagnosis {
30    let evidence = vec![base_id.to_string(), current_id.to_string()];
31    let (bmed, bn) = median_scheduled_ns(base);
32    let (cmed, cn) = median_scheduled_ns(current);
33
34    match (bmed, cmed) {
35        (Some(b), Some(c)) if bn >= MIN_TASKS && cn >= MIN_TASKS && b > 0 => {
36            let ratio = c as f64 / b as f64;
37            if ratio >= threshold_ratio {
38                let res_note = if current.resources.is_empty() {
39                    "no resource records captured; correlate manually in tokio-console Resources view"
40                } else {
41                    "see captured resource records for contended primitives"
42                };
43                Diagnosis {
44                    status: FindingStatus::Hypothesis,
45                    finding: format!(
46                        "median scheduled time grew {b}ns -> {c}ns (x{ratio:.1}) across {cn} tasks"
47                    ),
48                    hypothesis: Some(format!(
49                        "scheduling delay consistent with resource contention, not slow work. {res_note}."
50                    )),
51                    confidence: 0.65,
52                    evidence,
53                    verification: Some(
54                        "widen the bottleneck (pool size, lock scope) and re-capture; scheduled time should fall"
55                            .into(),
56                    ),
57                }
58            } else {
59                Diagnosis {
60                    status: FindingStatus::NoIssue,
61                    finding: format!(
62                        "scheduling delay stable (median {b}ns -> {c}ns, threshold x{threshold_ratio})"
63                    ),
64                    hypothesis: None,
65                    confidence: 0.85,
66                    evidence,
67                    verification: None,
68                }
69            }
70        }
71        _ => Diagnosis {
72            status: FindingStatus::Observed,
73            finding: format!(
74                "scheduled-time data insufficient (base tasks with data: {bn}, current: {cn}; need {MIN_TASKS}); no contention verdict"
75            ),
76            hypothesis: None,
77            confidence: 0.3,
78            evidence,
79            verification: Some("capture a longer window so scheduled stats populate".into()),
80        },
81    }
82}
83
84/// Median of `scheduled_ns` over tasks that report it, plus how many reported.
85fn median_scheduled_ns(snap: &Snapshot) -> (Option<u64>, usize) {
86    let mut v: Vec<u64> = snap.tasks.iter().filter_map(|t| t.scheduled_ns).collect();
87    let n = v.len();
88    if v.is_empty() {
89        return (None, 0);
90    }
91    v.sort_unstable();
92    (Some(v[n / 2]), n)
93}
94
95#[cfg(test)]
96mod tests {
97    use super::*;
98    use inspector_core::{RuntimeOverview, TaskRecord};
99
100    fn task(id: u64, sched: Option<u64>) -> TaskRecord {
101        TaskRecord {
102            task_id: id,
103            name: None,
104            polls: Some(10),
105            wakes: Some(10),
106            self_wakes: Some(0),
107            busy_ns: None,
108            scheduled_ns: sched,
109            state: None,
110            location: None,
111        }
112    }
113
114    fn snap(sched_each: Option<u64>, n: usize) -> Snapshot {
115        Snapshot {
116            overview: RuntimeOverview {
117                service_name: None,
118                console_addr: "t".into(),
119                active_tasks: Some(n as u64),
120                completed_tasks: None,
121                tasks_truncated: false,
122                metric_names: vec![],
123            },
124            tasks: (0..n as u64).map(|i| task(i, sched_each)).collect(),
125            resources: vec![],
126            metrics: vec![],
127        }
128    }
129
130    #[test]
131    fn fires_on_scheduling_blowout() {
132        let d = detect_contention(
133            &snap(Some(1_000_000), 10),
134            &snap(Some(50_000_000), 60),
135            "a",
136            "b",
137        );
138        assert_eq!(d.status, FindingStatus::Hypothesis);
139        assert!(d.is_valid());
140    }
141
142    #[test]
143    fn quiet_when_stable() {
144        let d = detect_contention(
145            &snap(Some(1_000_000), 10),
146            &snap(Some(1_200_000), 10),
147            "a",
148            "b",
149        );
150        assert_eq!(d.status, FindingStatus::NoIssue);
151    }
152
153    #[test]
154    fn missing_data_has_no_verdict() {
155        let d = detect_contention(&snap(None, 10), &snap(Some(50_000_000), 60), "a", "b");
156        assert_eq!(d.status, FindingStatus::Observed);
157    }
158
159    #[test]
160    fn too_few_tasks_has_no_verdict() {
161        let d = detect_contention(&snap(Some(1_000), 2), &snap(Some(99_000), 2), "a", "b");
162        assert_eq!(d.status, FindingStatus::Observed);
163    }
164}
inspector_analysis/contention.rs

inspector_analysis/
contention.rs
