use crate::analysis::value_counts::{LineKind, Number, Order, Summary, ValueCounts};
use crate::analysis::value_counts_modal::ValueCountsModal;
use crate::numfmt::{self, CellFormatter};
use crate::render::context::RenderContext;
use polars::prelude::{AnyValue, DataType};
use ratatui::buffer::Buffer;
use ratatui::layout::Rect;
use ratatui::style::{Modifier, Style};
use ratatui::text::{Line, Span};
use ratatui::widgets::{Clear, Paragraph, Widget};
const GAP: usize = 3;
pub fn render(area: Rect, buf: &mut Buffer, app: &mut crate::App, ctx: &RenderContext) {
let g = crate::glyphs::get();
let spinner = g.spinner[app.throbber_frame as usize % g.spinner.len()];
draw(area, buf, &mut app.value_counts, ctx, spinner);
if app.value_counts.shows_histogram() {
draw_histogram(area, buf, app, ctx);
}
if matches!(app.overlay, crate::Overlay::Export { .. }) {
let dialog = crate::widgets::export::dialog_area(area);
crate::widgets::export::render_export_modal(dialog, buf, &mut app.export_modal, ctx);
}
}
pub fn draw(
area: Rect,
buf: &mut Buffer,
modal: &mut ValueCountsModal,
ctx: &RenderContext,
spinner: &str,
) {
Clear.render(area, buf);
if area.height == 0 || area.width < 4 {
return;
}
let g = crate::glyphs::get();
let column = modal.column().unwrap_or_default().to_string();
let counts = modal.current().cloned();
let count_fmt = ctx.number_format.formatter_for("", &DataType::UInt64);
let chrome = numfmt::group_chrome;
let read = match &counts {
Some(c) => match c.sampled_of {
Some(of) => format!(
"sample of {} of {} rows",
chrome(c.summary.rows),
chrome(of)
),
None => format!("all {} rows", chrome(c.summary.rows)),
},
None if modal.failed.as_ref().is_some_and(|(c, _)| *c == column) => {
"not counted".to_string()
}
None => "counting".to_string(),
};
let view = match counts.as_ref().and_then(|c| c.histogram.as_ref()) {
Some(_) if !modal.shows_histogram() => format!(" {} List", g.middot),
Some(h) => format!(" {} Histogram, {} bins", g.middot, h.bins.len()),
None => String::new(),
};
let mut header = format!(
"Value Counts {} {column}{view} {} {read}",
g.middot, g.middot
);
if counts.is_some() && modal.counting() {
header.push_str(&format!(" {} {spinner} counting every row", g.middot));
}
let header_style = Style::default().bg(ctx.controls_bg).fg(ctx.table_header);
buf.set_style(Rect { height: 1, ..area }, header_style);
Paragraph::new(crate::glyphs::fit_cells(&header, area.width as usize, g.ellipsis).into_owned())
.style(header_style.add_modifier(Modifier::BOLD))
.render(Rect { height: 1, ..area }, buf);
let body = Rect {
x: area.x + 1,
y: area.y + 1,
width: area.width - 1,
height: area.height - 1,
};
let Some(counts) = counts else {
let line = match &modal.failed {
Some((failed, why)) if *failed == column => Line::from(Span::styled(
format!("Could not count {column}: {why}"),
Style::default().fg(ctx.warning),
)),
_ => {
let seen = modal
.computing
.as_ref()
.and_then(|c| c.watch.rows_seen())
.map(|rows| format!(" {} {} rows read", g.middot, chrome(rows)))
.unwrap_or_default();
Line::from(Span::styled(
format!("{spinner} Counting {column}...{seen}"),
Style::default().fg(ctx.text_secondary),
))
}
};
if body.height > 1 {
Paragraph::new(line).render(
Rect {
y: body.y + 1,
height: 1,
..body
},
buf,
);
}
return;
};
let width = body.width as usize;
let strip = summary_items(
&counts.summary,
&column,
&counts.dtype,
counts.is_sample(),
ctx,
crate::glyphs::get(),
body.width,
);
let strip_lines = pack(&strip, width);
let mut y = body.y;
let bottom = body.y + body.height;
for items in &strip_lines {
if y >= bottom {
return;
}
let mut spans = Vec::new();
for (i, (label, value)) in items.iter().enumerate() {
if i > 0 {
spans.push(Span::raw(" ".repeat(GAP)));
}
spans.push(Span::styled(
format!("{label} "),
Style::default().fg(ctx.text_secondary),
));
spans.push(Span::styled(
value.clone(),
Style::default().fg(ctx.text_primary),
));
}
Paragraph::new(Line::from(spans)).render(Rect::new(body.x, y, body.width, 1), buf);
y += 1;
}
if body.height as usize > strip_lines.len() + 3 {
y += 1;
}
modal.body_top = y;
if y >= bottom || modal.shows_histogram() {
return;
}
let lines = counts.lines(modal.order);
let value_fmt = ctx.number_format.formatter_for(&column, &counts.dtype);
let labels: Vec<(String, Style)> = lines
.iter()
.map(|line| label(&counts, line.kind, &value_fmt, ctx, body.width))
.collect();
let numbers: Vec<String> = lines
.iter()
.map(|l| count_text(l.rows, &count_fmt))
.collect();
let total = counts.summary.rows.max(1) as f64;
let order_mark = |order: Order| {
if modal.order == order {
if order == Order::Count {
g.sort_desc
} else {
g.sort_asc
}
} else {
""
}
};
let value_head = format!("{column}{}", order_mark(Order::Value));
let count_head = format!("Count{}", order_mark(Order::Count));
let count_w = numbers
.iter()
.map(|n| crate::glyphs::display_width(n))
.chain([crate::glyphs::display_width(&count_head)])
.max()
.unwrap_or(5);
const PCT_W: usize = 6;
let value_cap = (width * 2 / 5).max(8);
let value_w = labels
.iter()
.map(|(l, _)| crate::glyphs::display_width(l))
.chain([crate::glyphs::display_width(&value_head)])
.max()
.unwrap_or(1)
.clamp(1, value_cap);
let count_x = value_w + GAP;
let pct_x = count_x + count_w + GAP;
let cum_x = pct_x + PCT_W + 1;
let bar_x = cum_x + PCT_W + GAP;
const BAR_MAX: usize = 60;
let bar_w = width.saturating_sub(bar_x).min(BAR_MAX);
let put = |buf: &mut Buffer, x: usize, y: u16, s: &str, style: Style| {
if x < width {
buf.set_stringn(body.x + x as u16, y, s, width - x, style);
}
};
let right = |buf: &mut Buffer, x: usize, w: usize, y: u16, s: &str, style: Style| {
let pad = w.saturating_sub(crate::glyphs::display_width(s));
put(buf, x + pad, y, s, style);
};
let numbers_right = ctx.number_format.align_numeric_right
&& (counts.dtype.is_primitive_numeric() || matches!(counts.dtype, DataType::Decimal(..)));
let head_style = Style::default()
.fg(ctx.table_header)
.add_modifier(Modifier::BOLD);
buf.set_style(
Rect::new(body.x, y, body.width, 1),
Style::default().bg(ctx.table_header_bg),
);
let fit = |s: &str, w: usize| crate::glyphs::fit_cells(s, w, g.ellipsis).into_owned();
let value_head = fit(&value_head, value_w);
let value_head_style = head_style.fg(ctx.type_color(&counts.dtype));
if numbers_right {
right(buf, 0, value_w, y, &value_head, value_head_style);
} else {
put(buf, 0, y, &value_head, value_head_style);
}
right(buf, count_x, count_w, y, &count_head, head_style);
right(buf, pct_x, PCT_W, y, "%", head_style);
right(buf, cum_x, PCT_W, y, "Cum %", head_style);
y += 1;
if y >= bottom {
return;
}
let height = (bottom - y) as usize;
modal.scroll_into_view(height);
let peak = lines
.iter()
.filter(|l| !matches!(l.kind, LineKind::Other(_)))
.map(|l| l.rows)
.max()
.unwrap_or(1)
.max(1) as f64;
let bar_style = Style::default().fg(ctx.primary_chart_series_color);
let text = Style::default().fg(ctx.text_primary);
for (i, line) in lines.iter().enumerate().skip(modal.offset).take(height) {
let selected = i == modal.selected;
if selected {
buf.set_style(Rect::new(area.x, y, area.width, 1), ctx.highlight_style());
buf.set_string(area.x, y, g.rail, Style::default().fg(ctx.accent));
}
let (label, style) = &labels[i];
if numbers_right && matches!(line.kind, LineKind::Value(_)) {
right(buf, 0, value_w, y, &fit(label, value_w), *style);
} else {
put(buf, 0, y, &fit(label, value_w), *style);
}
right(buf, count_x, count_w, y, &numbers[i], text);
right(
buf,
pct_x,
PCT_W,
y,
&crate::numfmt::percent(line.rows as f64 / total),
text,
);
right(
buf,
cum_x,
PCT_W,
y,
&crate::numfmt::percent(line.cumulative as f64 / total),
Style::default().fg(ctx.text_secondary),
);
if bar_w > 0 && !matches!(line.kind, LineKind::Other(_)) {
let eighths = ((line.rows as f64 / peak) * (bar_w * 8) as f64).round() as usize;
let eighths = eighths.max(usize::from(line.rows > 0)).min(bar_w * 8);
let mut bar = g.bar_eighths[7].repeat(eighths / 8);
if let Some(part) = (eighths % 8).checked_sub(1) {
bar.push_str(g.bar_eighths[part]);
}
put(buf, bar_x, y, &bar, bar_style);
}
y += 1;
}
}
fn draw_histogram(area: Rect, buf: &mut Buffer, app: &crate::App, ctx: &RenderContext) {
let modal = &app.value_counts;
let Some(counts) = modal.current() else {
return;
};
let Some(histogram) = &counts.histogram else {
return;
};
let top = modal.body_top.max(area.y + 1);
if top >= area.bottom() {
return;
}
let mut plot = Rect {
x: area.x + 1,
y: top,
width: area.width.saturating_sub(2),
height: area.bottom() - top,
};
let notes = crate::chart::chart_data::chart_notes(
&Default::default(),
histogram.clipped.as_ref(),
crate::glyphs::get().middot,
);
if !notes.is_empty() && plot.height > 4 {
plot.height -= 1;
Paragraph::new(notes.join(" "))
.style(Style::default().fg(ctx.dimmed))
.right_aligned()
.render(
Rect {
y: plot.bottom(),
height: 1,
..plot
},
buf,
);
}
let schema = app.data_table_state.as_ref().map(|s| s.schema().as_ref());
let x = crate::widgets::axis_numbers::AxisNumbers::column(
&ctx.number_format,
schema,
&counts.column,
);
crate::widgets::chart::render_histogram(plot, buf, &app.theme, ctx, histogram, x);
}
fn label(
counts: &ValueCounts,
kind: LineKind,
fmt: &CellFormatter,
ctx: &RenderContext,
width: u16,
) -> (String, Style) {
let g = crate::glyphs::get();
match kind {
LineKind::Value(at) => {
let text = match counts.value(at) {
Ok(value) => {
let mut scratch = String::new();
let shown = numfmt::format_any_value(fmt, &value, &mut scratch);
crate::exact::cell_text(shown, g, crate::exact::cell_cut(width))
}
Err(_) => String::new(),
};
(text, Style::default().fg(ctx.text_primary))
}
LineKind::Null => (g.null.to_string(), Style::default().fg(ctx.dimmed)),
LineKind::Other(values) => (
format!(
"other ({} values)",
count_text(
values as u64,
&ctx.number_format.formatter_for("", &DataType::UInt64)
)
),
Style::default().fg(ctx.text_secondary),
),
}
}
pub fn summary_items(
summary: &Summary,
column: &str,
dtype: &DataType,
sample: bool,
ctx: &RenderContext,
g: &crate::glyphs::Glyphs,
width: u16,
) -> Vec<(&'static str, String)> {
let mut items = vec![
("Rows", crate::numfmt::group_chrome(summary.rows)),
("Distinct", crate::numfmt::group_chrome(summary.distinct)),
("Nulls", crate::numfmt::group_chrome(summary.nulls)),
];
let floats = ctx.number_format.formatter_for(column, &DataType::Float64);
if let Some(sum) = summary.sum.filter(|_| !sample) {
let text = match sum {
Number::Int(n) => match i64::try_from(n) {
Ok(n) => {
let ints = ctx.number_format.formatter_for(column, &DataType::Int64);
let mut scratch = String::new();
numfmt::format_any_value(&ints, &AnyValue::Int64(n), &mut scratch).into_owned()
}
Err(_) => n.to_string(),
},
Number::Float(f) => float_text(f, &floats),
};
items.push(("Sum", text));
}
if let Some(mean) = summary.mean {
items.push(("Mean", float_text(mean, &floats)));
}
let values = ctx.number_format.formatter_for(column, dtype);
for (name, value) in [("Min", &summary.min), ("Max", &summary.max)] {
if let Some(value) = value {
let mut scratch = String::new();
let text = numfmt::format_any_value(&values, value, &mut scratch);
items.push((
name,
crate::exact::cell_text(text, g, crate::exact::cell_cut(width)),
));
}
}
items
}
fn pack(items: &[(&'static str, String)], width: usize) -> Vec<Vec<(&'static str, String)>> {
let mut lines: Vec<Vec<(&'static str, String)>> = Vec::new();
let mut used = 0;
for item in items {
let w = crate::glyphs::display_width(item.0) + 1 + crate::glyphs::display_width(&item.1);
match lines.last_mut() {
Some(line) if used + GAP + w <= width => {
line.push(item.clone());
used += GAP + w;
}
_ => {
lines.push(vec![item.clone()]);
used = w;
}
}
}
lines
}
fn count_text(n: u64, fmt: &CellFormatter) -> String {
let mut scratch = String::new();
numfmt::format_any_value(fmt, &AnyValue::UInt64(n), &mut scratch).into_owned()
}
pub fn float_text(v: f64, fmt: &CellFormatter) -> String {
if !v.is_finite() {
return v.to_string();
}
if v != 0.0 && (v.abs() >= 1e15 || v.abs() < 1e-4) {
return format!("{v:.4e}");
}
let mut text = format!("{v:.4}");
if text.contains('.') {
let trimmed = text.trim_end_matches('0').trim_end_matches('.').len();
text.truncate(trimmed);
}
if text == "-0" {
text = "0".to_string();
}
match fmt {
CellFormatter::Number(nf) => {
let mut out = String::new();
nf.regroup_decimal(&text, &mut out);
out
}
CellFormatter::Passthrough => text,
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::analysis::sampling::ReadWatch;
use polars::prelude::*;
fn screen(modal: &mut ValueCountsModal, width: u16, height: u16) -> Vec<String> {
let ctx = RenderContext::for_test();
let area = Rect::new(0, 0, width, height);
let mut buf = Buffer::empty(area);
draw(area, &mut buf, modal, &ctx, "*");
(0..height)
.map(|y| {
(0..width)
.map(|x| buf[(x, y)].symbol().to_string())
.collect::<String>()
.trim_end()
.to_string()
})
.collect()
}
fn counted(df: DataFrame, column: &str) -> ValueCountsModal {
let counts = crate::analysis::value_counts::Plan {
lf: df.lazy(),
column: column.to_string(),
read: crate::analysis::value_counts::Read::Exact,
known_total: None,
streaming: false,
}
.run(&ReadWatch::default())
.unwrap();
let mut modal = ValueCountsModal::default();
modal.open(vec![column.to_string()], 0, 1);
modal.hold(counts);
modal
}
#[test]
fn the_screen_lists_values_with_the_summary_above() {
let mut modal = counted(
df!("pay" => [Some(1i64), Some(1), Some(2), None, Some(1)]).unwrap(),
"pay",
);
modal.view = Some(crate::analysis::value_counts_modal::CountsView::Listing);
let rows = screen(&mut modal, 80, 12);
let g = crate::glyphs::get();
assert!(rows[0].starts_with("Value Counts"), "{rows:#?}");
assert!(rows[0].contains("all 5 rows"), "{rows:#?}");
assert!(rows[1].contains("Rows 5") && rows[1].contains("Distinct 2"));
assert!(rows[1].contains("Nulls 1") && rows[1].contains("Sum 5"));
assert!(rows[1].contains("Mean 1.25") && rows[1].contains("Max 2"));
let head = rows.iter().position(|r| r.contains("Cum %")).unwrap();
assert!(rows[head].contains(&format!("Count{}", g.sort_desc)));
let first = &rows[head + 1];
assert!(first.starts_with(g.rail), "the cursor's rail: {first:?}");
assert!(first.contains("60.0%"), "{first:?}");
assert!(rows[head + 2].contains("20.0%") && rows[head + 2].contains("80.0%"));
assert!(rows[head + 3].contains(g.null), "nulls on their own line");
assert!(rows[head + 3].contains("100.0%"));
}
#[test]
fn the_header_names_a_numbers_view() {
let mut modal = counted(df!("pay" => [1i64, 1, 2, 5]).unwrap(), "pay");
let header = |modal: &mut ValueCountsModal| screen(modal, 80, 12).remove(0);
let dot = crate::glyphs::get().middot;
let shown = header(&mut modal);
assert!(shown.contains(&format!("pay {dot} Histogram, ")), "{shown}");
assert!(
shown.contains(&format!(" bins {dot} all 4 rows")),
"{shown}"
);
modal.toggle_view();
let shown = header(&mut modal);
assert!(
shown.contains(&format!("pay {dot} List {dot} all 4 rows")),
"{shown}"
);
let mut words = counted(df!("tag" => ["a", "b"]).unwrap(), "tag");
let shown = header(&mut words);
assert!(
!shown.contains("List") && !shown.contains("Histogram"),
"{shown}"
);
}
#[test]
fn a_multi_line_min_is_marked_and_cut() {
let ctx = RenderContext::for_test();
let summary = crate::analysis::value_counts::Summary {
rows: 2,
distinct: 2,
nulls: 0,
sum: None,
mean: None,
min: Some(AnyValue::StringOwned("line one\nline two".into())),
max: Some(AnyValue::StringOwned("x".repeat(1 << 20).into())),
};
for g in [crate::glyphs::unicode(), crate::glyphs::ascii()] {
let items = summary_items(&summary, "s", &DataType::String, false, &ctx, g, 80);
let value = |name: &str| &items.iter().find(|(n, _)| *n == name).unwrap().1;
assert_eq!(value("Min"), &format!("line one{}line two", g.newline_mark));
let max = value("Max");
assert!(max.ends_with(g.ellipsis), "{} bytes", max.len());
assert!(max.len() < 300, "{} bytes", max.len());
}
}
#[test]
fn the_strip_groups_its_counts() {
let mut modal = counted(df!("n" => (0..1_500i64).collect::<Vec<_>>()).unwrap(), "n");
modal.view = Some(crate::analysis::value_counts_modal::CountsView::Listing);
let rows = screen(&mut modal, 100, 12);
assert!(rows[1].contains("Rows 1,500"), "{rows:#?}");
assert!(rows[1].contains("Distinct 1,500"), "{rows:#?}");
}
#[test]
fn a_narrow_screen_wraps_the_strip_and_keeps_the_listing() {
let mut modal = counted(
df!("amount" => [1.5f64, 2.25, 1.5, 1000.0]).unwrap(),
"amount",
);
modal.view = Some(crate::analysis::value_counts_modal::CountsView::Listing);
let rows = screen(&mut modal, 40, 12);
assert!(rows[1].contains("Rows 4"));
assert!(rows.iter().any(|r| r.contains("Sum 1005.25")), "{rows:#?}");
assert!(rows.iter().any(|r| r.contains("Cum %")));
assert!(rows.iter().all(|r| crate::glyphs::display_width(r) <= 40));
}
#[test]
fn a_sample_says_so_in_the_header() {
let mut modal = counted(df!("k" => ["a", "b"]).unwrap(), "k");
let mut counts = (**modal.current().unwrap()).clone();
counts.sampled_of = Some(1_000_000);
modal.hold(counts);
let rows = screen(&mut modal, 80, 8);
assert!(
rows[0].contains("sample of 2 of 1,000,000 rows"),
"{rows:#?}"
);
let mut modal = counted(df!("n" => [1i64, 2]).unwrap(), "n");
let mut counts = (**modal.current().unwrap()).clone();
counts.sampled_of = Some(10);
modal.hold(counts);
let rows = screen(&mut modal, 80, 8);
assert!(
rows[1].contains("Mean 1.5") && !rows[1].contains("Sum"),
"{rows:#?}"
);
}
#[test]
fn before_the_counts_the_screen_says_it_is_counting() {
let mut modal = ValueCountsModal::default();
modal.open(vec!["k".to_string()], 0, 1);
modal.computing = Some(crate::analysis::value_counts_modal::Computing {
column: "k".to_string(),
exact: false,
watch: ReadWatch::default(),
file_starts: None,
});
let rows = screen(&mut modal, 60, 6);
assert!(rows[2].contains("Counting k..."), "{rows:#?}");
modal.computing = None;
modal.failed = Some(("k".to_string(), "boom".to_string()));
let rows = screen(&mut modal, 60, 6);
assert!(rows[2].contains("Could not count k: boom"), "{rows:#?}");
assert!(!rows[0].contains("counting"), "nothing runs: {rows:#?}");
}
#[test]
fn the_other_line_has_no_bar() {
let ids: Vec<i64> = (0..crate::analysis::value_counts::TOP_N as i64 + 50).collect();
let mut modal = counted(df!("id" => ids).unwrap(), "id");
modal.view = Some(crate::analysis::value_counts_modal::CountsView::Listing);
modal.move_to_end();
let rows = screen(&mut modal, 80, 8);
let g = crate::glyphs::get();
let at = rows
.iter()
.position(|r| r.contains("other (50 values)"))
.unwrap();
assert!(!rows[at].contains(g.bar_eighths[7]), "{rows:#?}");
assert!(
rows[at - 1].contains(g.bar_eighths[7]),
"a value's line has one: {rows:#?}"
);
}
#[test]
fn shares_and_means_read_plainly() {
let plain = CellFormatter::Passthrough;
assert_eq!(float_text(1.25, &plain), "1.25");
assert_eq!(float_text(3.0, &plain), "3");
assert_eq!(float_text(2.0 / 3.0, &plain), "0.6667");
assert_eq!(float_text(1e20, &plain), "1.0000e20");
}
}