use super::{GpuDb, compute_gpu_gaps, fmt_bytes, fmt_us, parse_count, trunc, xfer_kernel_overlap};
pub fn cmd_transfers(db: &GpuDb, args: &[&str]) {
if db.transfer_count() == 0 {
println!("no memory transfers recorded");
if !db.has_layer("nsys") {
println!("need nsys layer for transfer data");
}
return;
}
let n = parse_count(args);
let transfer_filter = db.timeline_filter_for("transfers");
let (total_bytes, total_time): (i64, f64) = db
.conn
.query_row(
&format!("SELECT COALESCE(SUM(bytes),0), COALESCE(SUM(duration_us),0) FROM transfers WHERE {transfer_filter}"),
[],
|row| Ok((row.get(0)?, row.get(1)?)),
)
.unwrap();
let kernel_time = db.total_gpu_time_us();
let wall_us: f64 = db.meta("wall_time_us").parse().unwrap_or(0.0);
println!(
" Total: {} transfers, {}, {}",
db.transfer_count(),
fmt_bytes(total_bytes),
fmt_us(total_time)
);
if wall_us > 0.0 {
println!(
" {:.1}% of wall time spent on transfers",
total_time / wall_us * 100.0
);
}
if kernel_time > 0.0 {
let ratio = total_time / kernel_time;
let verdict = if ratio > 5.0 {
"BANDWIDTH-BOUND — PCIe dominates"
} else if ratio > 1.5 {
"transfer-heavy — consider async transfers or larger batches"
} else if ratio > 0.5 {
"mixed compute/transfer"
} else {
"compute-dominated"
};
println!(" Transfer:compute ratio = {ratio:.2}:1 ({verdict})");
}
println!();
let kind_sql = format!("SELECT kind, COUNT(*), SUM(bytes), SUM(duration_us),
MIN(bytes), MAX(bytes)
FROM transfers WHERE {transfer_filter} GROUP BY kind ORDER BY SUM(duration_us) DESC");
let kinds: Vec<(String, i64, i64, f64, i64, i64)> = db.query_vec(&kind_sql, [], |row| {
Ok((
row.get(0)?,
row.get(1)?,
row.get(2)?,
row.get(3)?,
row.get(4)?,
row.get(5)?,
))
});
println!(" By Direction:");
println!(" Kind Count Total Time Avg BW Size range");
println!(" ───── ──────── ───────────── ─────────── ──────────── ────────────");
for (kind, cnt, bytes, dur, min_b, max_b) in &kinds {
let bw = if *dur > 0.0 {
format!("{:.1} GB/s", *bytes as f64 / dur / 1000.0)
} else {
"?".into()
};
let range = if min_b == max_b {
fmt_bytes(*min_b)
} else {
format!("{}-{}", fmt_bytes(*min_b), fmt_bytes(*max_b))
};
println!(
" {:<5} {:>8} {:>13} {:>11} {:>12} {}",
kind,
cnt,
fmt_bytes(*bytes),
fmt_us(*dur),
bw,
range
);
}
println!();
let (small_cnt, small_bytes, small_time): (i64, i64, f64) = db
.conn
.query_row(
&format!(
"SELECT COUNT(*), COALESCE(SUM(bytes),0), COALESCE(SUM(duration_us),0)
FROM transfers WHERE bytes < 1048576 AND {transfer_filter}"
), [],
|row| Ok((row.get(0)?, row.get(1)?, row.get(2)?)),
)
.unwrap();
if small_cnt > 0 {
let pct = if total_time > 0.0 {
small_time / total_time * 100.0
} else {
0.0
};
println!(
" Small transfers: {} (<1 MB), {} total, {} time ({pct:.1}% of transfer time)",
small_cnt,
fmt_bytes(small_bytes),
fmt_us(small_time)
);
if small_cnt > 10 {
println!(" → many small transfers — coalesce into fewer batched copies");
}
}
if total_time > 0.0 {
print_transfer_cdf(db, total_time);
}
let sql = format!(
"SELECT kind, bytes, duration_us, stream_id
FROM transfers WHERE {transfer_filter} ORDER BY duration_us DESC LIMIT ?1"
);
let rows: Vec<(String, i64, f64, Option<u32>)> = db.query_vec(&sql, [n as i64], |row| {
Ok((row.get(0)?, row.get(1)?, row.get(2)?, row.get(3)?))
});
println!("\n Top {} by Duration:", rows.len());
println!(" # Kind Size Duration BW Stream Notes");
println!(" ── ───── ────────── ────────── ──────────── ─────── ────────");
for (i, (kind, bytes, dur, sid)) in rows.iter().enumerate() {
let bw_num = if *dur > 0.0 {
*bytes as f64 / dur / 1000.0
} else {
0.0
};
let bw_str = if *dur > 0.0 {
format!("{:.1} GB/s", bw_num)
} else {
"?".into()
};
let notes =
if *bytes >= 16 * 1024 * 1024 && (kind == "H2D" || kind == "D2H") && bw_num < 6.0 {
"pageable? consider cudaMallocHost"
} else if *bytes < 4096 {
"tiny — overhead-dominated"
} else {
""
};
println!(
" {:<2} {:<5} {:>10} {:>10} {:>11} {:>6} {}",
i + 1,
kind,
fmt_bytes(*bytes),
fmt_us(*dur),
bw_str,
sid.map(|s| s.to_string()).unwrap_or_else(|| "?".into()),
notes
);
}
}
fn print_transfer_cdf(db: &GpuDb, total_time: f64) {
let buckets: [(i64, &str); 7] = [
(4 * 1024, "<4 KB"),
(64 * 1024, "<64 KB"),
(1024 * 1024, "<1 MB"),
(16 * 1024 * 1024, "<16 MB"),
(128 * 1024 * 1024, "<128 MB"),
(1024 * 1024 * 1024, "<1 GB"),
(i64::MAX, ">=1 GB"),
];
let transfer_filter = db.timeline_filter_for("transfers");
let rows: Vec<(i64, f64)> = db.query_vec(
&format!(
"SELECT bytes, duration_us FROM transfers WHERE {transfer_filter} ORDER BY bytes ASC"
),
[],
|row| Ok((row.get(0)?, row.get(1)?)),
);
if rows.is_empty() {
return;
}
println!("\n Cumulative time by transfer size:");
println!(" Size bucket Count Time Bucket % Cumulative %");
println!(" ────────────── ──────── ──────────── ───────── ────────────");
let mut idx = 0usize;
let mut cum = 0.0;
for &(limit, label) in &buckets {
let mut cnt = 0i64;
let mut bucket_time = 0.0;
while idx < rows.len() && rows[idx].0 < limit {
bucket_time += rows[idx].1;
cnt += 1;
idx += 1;
}
if cnt == 0 {
continue;
}
cum += bucket_time;
let bpct = bucket_time / total_time * 100.0;
let cpct = cum / total_time * 100.0;
println!(
" {:<14} {:>8} {:>12} {:>8.1}% {:>11.1}%",
label,
cnt,
fmt_us(bucket_time),
bpct,
cpct
);
}
}
pub fn cmd_gaps(db: &GpuDb, args: &[&str]) {
if !db.has_layer("nsys") {
println!("no timeline data — need nsys layer");
return;
}
let n = parse_count(args);
let mut rows = compute_gpu_gaps(db);
if rows.is_empty() {
println!("no GPU idle gaps detected");
return;
}
let total_gap: f64 = rows.iter().map(|r| r.1).sum();
let total_count = rows.len();
rows.sort_by(|a, b| b.1.partial_cmp(&a.1).unwrap());
let shown = rows.len().min(n);
rows.truncate(n);
println!(
" {} GPU idle gaps (total idle: {})",
total_count,
fmt_us(total_gap)
);
if shown < total_count {
println!(" Showing top {shown} by duration:\n");
} else {
println!();
}
println!(" # Start Duration Before → After");
println!(" ── ──────────── ──────────── ────────────────────────────────");
for (i, (start, dur)) in rows.iter().enumerate() {
let before = kernel_ending_at_or_before(db, *start);
let after = kernel_starting_at_or_after(db, *start + *dur);
let edge = format!(
"{} → {}",
before
.as_deref()
.map(|n| trunc(n, 22))
.unwrap_or_else(|| "—".into()),
after
.as_deref()
.map(|n| trunc(n, 22))
.unwrap_or_else(|| "—".into())
);
println!(
" {:<2} {:>12} {:>12} {}",
i + 1,
fmt_us(*start),
fmt_us(*dur),
edge
);
}
}
fn kernel_ending_at_or_before(db: &GpuDb, t: f64) -> Option<String> {
let tl = db.timeline_filter();
let sql = format!(
"SELECT kernel_name FROM launches
WHERE start_us IS NOT NULL AND (start_us + duration_us) <= ?1 + 0.5 AND {tl}
ORDER BY (start_us + duration_us) DESC LIMIT 1"
);
db.conn
.query_row(&sql, [t], |row| row.get::<_, String>(0))
.ok()
}
fn kernel_starting_at_or_after(db: &GpuDb, t: f64) -> Option<String> {
let tl = db.timeline_filter();
let sql = format!(
"SELECT kernel_name FROM launches
WHERE start_us IS NOT NULL AND start_us >= ?1 - 0.5 AND {tl}
ORDER BY start_us ASC LIMIT 1"
);
db.conn
.query_row(&sql, [t], |row| row.get::<_, String>(0))
.ok()
}
pub fn cmd_overlap(db: &GpuDb) {
if !db.has_layer("nsys") {
println!("no timeline data — need nsys layer");
return;
}
let gpu_us = db.total_gpu_time_us();
let wall_us: f64 = db.meta("wall_time_us").parse().unwrap_or(0.0);
let xfer_time: f64 = db
.conn
.query_row(
&format!(
"SELECT COALESCE(SUM(duration_us),0) FROM transfers WHERE {}",
db.timeline_filter_for("transfers")
),
[],
|row| row.get(0),
)
.unwrap();
let (_, overlap_us) = xfer_kernel_overlap(db, None);
println!(" Compute/Transfer Overlap:");
println!(" GPU kernel time: {}", fmt_us(gpu_us));
println!(" Transfer time: {}", fmt_us(xfer_time));
if xfer_time > 0.0 && overlap_us > 0.0 {
println!(
" Concurrent: {} ({:.1}% of transfers overlapped with compute)",
fmt_us(overlap_us),
overlap_us / xfer_time * 100.0
);
} else if xfer_time > 0.0 {
println!(" Concurrent: none (transfers and compute are serialized)");
}
if wall_us > 0.0 {
println!(" GPU utilization: {:.1}%", gpu_us / wall_us * 100.0);
}
let kinds: Vec<String> = db.query_vec(
&format!(
"SELECT DISTINCT kind FROM transfers WHERE start_us IS NOT NULL AND {}",
db.timeline_filter_for("transfers")
),
[],
|row| row.get(0),
);
if !kinds.is_empty() {
println!("\n By direction:");
println!(" Kind Transfer Overlap %");
println!(" ────── ────────── ────────── ──────");
for kind in &kinds {
let (dir_time, dir_overlap) = xfer_kernel_overlap(db, Some(kind));
if dir_time <= 0.0 {
continue;
}
let pct = dir_overlap / dir_time * 100.0;
println!(
" {:<6} {:>10} {:>10} {:>5.1}%",
kind,
fmt_us(dir_time),
fmt_us(dir_overlap),
pct
);
}
}
}
pub fn cmd_streams(db: &GpuDb) {
let tl = db.timeline_filter();
let sql = format!(
"SELECT stream_id, COUNT(*) as cnt, SUM(duration_us) as total
FROM launches WHERE stream_id IS NOT NULL AND {tl}
GROUP BY stream_id ORDER BY total DESC"
);
let rows: Vec<(u32, i64, f64)> =
db.query_vec(&sql, [], |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?)));
if rows.is_empty() {
println!("no stream data");
return;
}
println!(" Stream Kernels Active Time");
println!(" ─────── ──────── ────────────");
for (sid, cnt, total) in &rows {
println!(" {:>6} {:>7} {:>11}", sid, cnt, fmt_us(*total));
}
}
pub fn cmd_timeline(db: &GpuDb, args: &[&str]) {
let n = parse_count(args);
let tl = db.timeline_filter();
let sql = format!(
"SELECT kernel_name, start_us, duration_us, stream_id
FROM launches WHERE start_us IS NOT NULL AND {tl}
ORDER BY start_us LIMIT ?1"
);
let rows: Vec<(String, f64, f64, Option<u32>)> = db.query_vec(&sql, [n as i64], |row| {
Ok((row.get(0)?, row.get(1)?, row.get(2)?, row.get(3)?))
});
if rows.is_empty() {
println!("no timeline data");
return;
}
println!(" # Start Duration Stream Kernel");
println!(" ── ──────────── ────────── ──────── ────────────────────────────────");
for (i, (name, start, dur, sid)) in rows.iter().enumerate() {
println!(
" {:<2} {:>12} {:>10} {:>7} {}",
i + 1,
fmt_us(*start),
fmt_us(*dur),
sid.map(|s| s.to_string()).unwrap_or_else(|| "?".into()),
trunc(name, 40)
);
}
}