use std::collections::BTreeMap;
use std::io::Write;
use std::sync::PoisonError;
use std::time::Duration;
use tokio::sync::mpsc::UnboundedReceiver;
use crate::command::_helpers::logging::LIVE_BLOCK;
use crate::command::_helpers::output::print_lines;
use crate::command::constants::PLAN_TREE_HINT;
use crate::load_events::main::event_line::event_line;
use crate::load_events::models::{EventKind, LoadEvent};
use crate::loader::constants::{
BOLD, CYAN, DICT_HINT_MILLISECONDS, DICT_HINT_ROWS, DIM, INVERSE, RED, RESET, YELLOW,
};
use crate::loader::models::ProjectPlan;
use crate::number_text::main::memory_text::memory_text;
use crate::number_text::main::thousands::thousands;
use crate::python_sources::models::TransformTime;
const VERB_COLUMN: usize = 8;
const SUMMARY_LABEL: usize = 13;
const LIVE_REDRAW: std::time::Duration = std::time::Duration::from_millis(100);
const LIVE_HZ: u8 = 10;
const MAX_LIVE_WIDTH: usize = 112;
const NAMED_TABLES: usize = 3;
const SPINNER: [char; 10] = ['⠋', '⠙', '⠹', '⠸', '⠼', '⠴', '⠦', '⠧', '⠇', '⠏'];
const QUEUED_NAMED: usize = 3;
const NAME_COLUMN: usize = 28;
const MAX_WIDTH: usize = 112;
const RATE_RECENT: Duration = Duration::from_millis(1500);
const BAR_LINE_GAPS: usize = 3 + 1;
#[derive(Debug)]
pub(crate) struct LoadOutput {
pub(crate) name_width: usize,
pub(crate) rows_width: usize,
pub(crate) print: bool,
pub(crate) no_color: bool,
pub(crate) events: Option<EventsTarget>,
pub(crate) live: Option<LiveState>,
}
#[derive(Debug)]
pub(crate) enum EventsTarget {
Stderr,
File(std::fs::File),
}
impl EventsTarget {
fn write_line(&mut self, line: &str) {
let _ = match self {
Self::Stderr => {
let mut stderr = std::io::stderr().lock();
writeln!(stderr, "{line}").and_then(|()| stderr.flush())
}
Self::File(file) => writeln!(file, "{line}").and_then(|()| file.flush()),
};
}
}
#[derive(Debug, Default, Clone, PartialEq, Eq)]
pub(crate) struct Tally {
pub(crate) loaded: usize,
pub(crate) failed: usize,
pub(crate) not_started: Vec<String>,
pub(crate) rows: u64,
pub(crate) slowest: Option<(String, u64)>,
pub(crate) peak_memory_bytes: Option<u64>,
}
impl Tally {
pub(crate) fn add(&mut self, event: &LoadEvent) {
let table = event.table.clone().unwrap_or_default();
match &event.kind {
EventKind::TableDone {
rows, elapsed_ms, ..
} => {
self.loaded += 1;
self.rows += rows;
if self
.slowest
.as_ref()
.is_none_or(|(_, slowest)| elapsed_ms > slowest)
{
self.slowest = Some((table, *elapsed_ms));
}
}
EventKind::TableFailed { .. } => self.failed += 1,
EventKind::TableNotStarted { .. } => self.not_started.push(table),
EventKind::RunFinished {
peak_memory_bytes, ..
} => self.peak_memory_bytes = *peak_memory_bytes,
_ => {}
}
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
struct Running {
table: String,
strategy: String,
rows: u64,
doing: String,
committing: bool,
written: (u64, u64),
}
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub(crate) struct LiveState {
waiting: Vec<String>,
total: usize,
done: usize,
finished_rows: u64,
running: Vec<Running>,
fail_fast: bool,
stopped_after: Option<String>,
rates: BTreeMap<String, (String, Option<Duration>)>,
follow: bool,
}
impl LiveState {
pub(crate) fn new(tables: Vec<String>, fail_fast: bool) -> Self {
Self {
total: tables.len(),
waiting: tables,
fail_fast,
..Self::default()
}
}
pub(crate) fn following(self) -> Self {
Self {
follow: true,
..self
}
}
pub(crate) fn apply(&mut self, event: &LoadEvent, at: Duration) {
let table = event.table.clone().unwrap_or_default();
match &event.kind {
EventKind::TableStarted { strategy } => {
self.waiting.retain(|waiting| *waiting != table);
self.running.push(Running {
table,
strategy: strategy.clone(),
rows: 0,
doing: String::new(),
committing: false,
written: (0, 0),
});
}
EventKind::TableProgress { unit, count, rows } => {
if let Some(running) = self.running_mut(&table) {
running.rows = running.written.0 + rows;
running.doing = format!("{unit} {count}");
}
}
EventKind::TableCommitting { rows } => {
if let Some(running) = self.running_mut(&table) {
running.rows = running.written.0 + rows;
running.doing = "committing".to_owned();
running.committing = true;
}
}
EventKind::TableBatchWritten { rows, .. } => {
if let Some(running) = self.running_mut(&table) {
running.written = (running.written.0 + rows, running.written.1 + 1);
running.rows = running.written.0;
running.doing = match running.written.1 {
1 => "1 batch written".to_owned(),
batches => format!("{batches} batches written"),
};
running.committing = false;
}
}
EventKind::RateLimited {
connection, rate, ..
} => {
self.rates
.insert(connection.clone(), (rate.clone(), Some(at)));
}
EventKind::TableDone { rows, .. } => {
self.running.retain(|running| running.table != table);
self.done += 1;
self.finished_rows += rows;
}
EventKind::TableFailed { .. } => {
self.running.retain(|running| running.table != table);
if self.fail_fast && self.stopped_after.is_none() {
self.stopped_after = Some(table);
}
}
EventKind::TableNotStarted { .. } => {
self.waiting.retain(|waiting| *waiting != table);
}
EventKind::RunStarted { .. } | EventKind::RunFinished { .. } => {}
}
}
fn running_mut(&mut self, table: &str) -> Option<&mut Running> {
self.running
.iter_mut()
.find(|running| running.table == table)
}
}
pub(crate) fn event_lines(
event: &LoadEvent,
(name_width, rows_width): (usize, usize),
) -> Vec<String> {
let name = event.table.as_deref().unwrap_or_default();
let (verb, detail) = match &event.kind {
EventKind::TableDone {
rows,
elapsed_ms,
position,
changes,
transforms,
peak_memory_bytes,
} => {
let word = match rows {
1 => "row",
_ => "rows",
};
let seconds = format!("{:.1}s", *elapsed_ms as f64 / 1000.0);
let note: Vec<String> = position
.iter()
.cloned()
.chain(peak_memory_bytes.map(peak_note))
.collect();
let detail = format!(
"{}{}",
format!(
"{:>rows_width$} {word:<5}{seconds:<6}{}",
thousands(*rows),
note.join(" ")
)
.trim_end(),
transform_lines(transforms)
);
let mut lines: Vec<String> = changes
.iter()
.map(|change| change_tally_line(change, name_width))
.collect();
lines.push(tally_line(("loaded", name), &detail, name_width));
return lines;
}
EventKind::TableBatchWritten { rows, behind_secs } => {
let word = match rows {
1 => "row",
_ => "rows",
};
let behind = behind_secs
.map(|secs| format!("{secs}s behind"))
.unwrap_or_default();
(
"written",
format!("{:>rows_width$} {word:<5}{behind}", thousands(*rows))
.trim_end()
.to_owned(),
)
}
EventKind::TableFailed {
reason,
peak_memory_bytes,
..
} => (
"failed",
match peak_memory_bytes {
Some(bytes) => format!("{reason}\n{}", peak_note(*bytes)),
None => reason.clone(),
},
),
_ => return Vec::new(),
};
vec![tally_line((verb, name), &detail, name_width)]
}
fn peak_note(bytes: u64) -> String {
format!("peak memory {}", memory_text(bytes))
}
fn transform_lines(times: &[TransformTime]) -> String {
let seconds = |time: &TransformTime| time.milliseconds as f64 / 1000.0;
let mut lines: String = times
.iter()
.map(|time| format!("\ntransform {} {:.2}s", time.file, seconds(time)))
.collect();
let slow = times
.iter()
.filter(|time| {
time.dicts && time.rows >= DICT_HINT_ROWS && time.milliseconds >= DICT_HINT_MILLISECONDS
})
.max_by_key(|time| time.milliseconds);
if let Some(time) = slow {
lines.push_str(&format!(
"\nhint: the transform returned dicts for {} ({} rows, {:.1}s); a pyarrow Table or a DataFrame transforms faster",
time.file,
thousands(time.rows),
seconds(time)
));
}
lines
}
fn tally_line((verb, name): (&str, &str), detail: &str, name_width: usize) -> String {
let verb_width = VERB_COLUMN.max(verb.len() + 2);
let name_width = name_width.max(name.len() + 2);
let continued = format!("\n{:width$}", "", width = verb_width + name_width);
format!(
"{verb:<verb_width$}{name:<name_width$}{}",
detail.replace('\n', &continued)
)
.trim_end()
.to_owned()
}
pub(crate) fn change_tally_line(line: &str, name_width: usize) -> String {
let mut parts = line.splitn(2, char::is_whitespace);
let verb = parts.next().unwrap_or_default();
let rest = parts.next().unwrap_or_default().trim_start();
let (name, detail) = rest.split_once(" ").unwrap_or((rest, ""));
tally_line((verb, name), detail.trim_start(), name_width)
}
pub(crate) fn start_lines(plan_lines: Vec<String>) -> Vec<String> {
let dropped = ["Plan ready", "Objects (", "Loads (", "Skipped ("];
let mut kept: Vec<String> = Vec::new();
for section in plan_lines.split(String::is_empty) {
let first = section.first().map_or("", String::as_str);
if section.is_empty() || dropped.iter().any(|heading| first.starts_with(heading)) {
continue;
}
kept.extend(section.iter().cloned());
kept.push(String::new());
}
kept.pop();
if let Some(closing) = kept.last_mut()
&& closing.starts_with('✓')
{
closing.push_str(&format!(" {PLAN_TREE_HINT}"));
}
kept
}
pub(crate) async fn write_events(
mut output: LoadOutput,
mut receiver: UnboundedReceiver<LoadEvent>,
) -> (Vec<String>, Tally) {
let mut kept: Vec<String> = Vec::new();
let mut tally = Tally::default();
let started = std::time::Instant::now();
let block = output.live.as_ref().map(|_| live_bar());
if let Some(bar) = &block {
*LIVE_BLOCK.lock().unwrap_or_else(PoisonError::into_inner) = Some(bar.clone());
}
let width = console::Term::stderr()
.size_checked()
.map_or(MAX_LIVE_WIDTH, |(_, columns)| usize::from(columns));
let mut ticks = tokio::time::interval(LIVE_REDRAW);
let mut frame: usize = 0;
let coloured = !output.no_color && std::env::var_os("NO_COLOR").is_none();
loop {
let event = tokio::select! {
event = receiver.recv() => event,
_ = ticks.tick(), if block.is_some() => {
frame += 1;
if let (Some(bar), Some(state)) = (&block, &output.live) {
bar.set_message(live_lines(state, (started.elapsed(), frame), (width, coloured)).join("\n"));
}
continue;
}
};
let Some(event) = event else {
break;
};
if let Some(events) = output.events.as_mut() {
events.write_line(&event_line(&event));
}
tally.add(&event);
if let Some(state) = output.live.as_mut() {
state.apply(&event, started.elapsed());
}
let lines = event_lines(&event, (output.name_width, output.rows_width));
match (output.print, &block) {
(true, _) if lines.is_empty() => {}
(true, Some(bar)) => bar.suspend(|| print_lines(lines, output.no_color)),
(true, None) => print_lines(lines, output.no_color),
(false, _) => kept.extend(lines),
}
}
if let Some(bar) = block {
*LIVE_BLOCK.lock().unwrap_or_else(PoisonError::into_inner) = None;
bar.finish_and_clear();
}
(kept, tally)
}
pub(crate) fn above_live_block<T>(write: impl FnOnce() -> T) -> T {
let drawn = LIVE_BLOCK
.lock()
.unwrap_or_else(PoisonError::into_inner)
.clone();
match drawn {
Some(bar) => bar.suspend(write),
None => write(),
}
}
fn live_bar() -> indicatif::ProgressBar {
let bar = indicatif::ProgressBar::with_draw_target(
None,
indicatif::ProgressDrawTarget::stderr_with_hz(LIVE_HZ),
);
if let Ok(style) = indicatif::ProgressStyle::with_template("{msg}") {
bar.set_style(style);
}
bar
}
pub(crate) fn summary_lines(tally: &Tally, elapsed_ms: u64, exit_code: u8) -> Vec<String> {
let mut clauses: Vec<String> = Vec::new();
clauses.extend((tally.loaded > 0).then(|| format!("{} loaded", tally.loaded)));
clauses.extend((tally.failed > 0).then(|| format!("{} failed", tally.failed)));
clauses.extend(
(!tally.not_started.is_empty()).then(|| format!("{} not started", tally.not_started.len())),
);
clauses.extend((tally.loaded > 0).then(|| match tally.rows {
1 => "1 row".to_owned(),
rows => format!("{} rows", thousands(rows)),
}));
clauses.push(format!("{:.1}s", elapsed_ms as f64 / 1000.0));
let failures = tally.failed > 0 || !tally.not_started.is_empty();
let mut branches: Vec<(&str, String)> = Vec::new();
if !tally.not_started.is_empty() {
let named: Vec<&str> = tally
.not_started
.iter()
.take(NAMED_TABLES)
.map(String::as_str)
.collect();
let more = match tally.not_started.len().saturating_sub(NAMED_TABLES) {
0 => String::new(),
more => format!(", +{more} more"),
};
branches.push((
"not started",
format!("{}{more} (--fail-fast)", named.join(", ")),
));
}
if let Some((table, elapsed_ms)) = &tally.slowest {
branches.push((
"slowest",
format!("{table} {:.1}s", *elapsed_ms as f64 / 1000.0),
));
}
if let Some(bytes) = tally.peak_memory_bytes {
branches.push(("peak memory", memory_text(bytes)));
}
if failures && exit_code == crate::command::constants::RETRYABLE_EXIT {
branches.push((
"retry",
format!("every failure is retryable, so exit {exit_code}: run the same command again"),
));
}
if failures {
let unchanged = match tally.failed + tally.not_started.len() {
1 => "the 1 unchanged table loads".to_owned(),
count => format!("the {count} unchanged tables load"),
};
branches.push((
"next",
match tally.loaded {
0 => unchanged,
1 => format!("{unchanged}; the 1 that committed carries on from its position"),
count => {
format!("{unchanged}; the {count} that committed carry on from their positions")
}
},
));
}
let mut lines = vec![match tally.failed {
0 => format!("✓ Completed {}", clauses.join(", ")),
_ => format!("✗ Completed with failures {}", clauses.join(", ")),
}];
for (index, (label, text)) in branches.iter().enumerate() {
let branch = match index + 1 == branches.len() {
true => "└── ",
false => "├── ",
};
lines.push(format!("{branch}{label:<SUMMARY_LABEL$}{text}"));
}
lines
}
pub(crate) async fn planning<T>(
work: impl std::future::Future<Output = T>,
sources: usize,
(draws, no_color): (bool, bool),
) -> T {
if !draws {
return work.await;
}
let bar = indicatif::ProgressBar::with_draw_target(
None,
indicatif::ProgressDrawTarget::stderr_with_hz(LIVE_HZ),
);
let template = match no_color {
true => "{spinner} Planning {msg}",
false => "{spinner:.cyan} Planning {msg:.dim}",
};
if let Ok(style) = indicatif::ProgressStyle::with_template(template) {
bar.set_style(style.tick_chars("⠋⠙⠹⠸⠼⠴⠦⠧⠇⠏ "));
}
let done = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false));
let drawer = {
let (bar, done) = (bar.clone(), done.clone());
std::thread::spawn(move || {
while !done.load(std::sync::atomic::Ordering::Relaxed) {
bar.set_message(format!(
"checking sources, {} of {sources} done",
ProjectPlan::planned_sources()
));
bar.tick();
std::thread::sleep(LIVE_REDRAW);
}
})
};
let output = work.await;
done.store(true, std::sync::atomic::Ordering::Relaxed);
if drawer.join().is_err() {
bar.abandon();
}
bar.finish_and_clear();
output
}
pub(crate) fn live_lines(
state: &LiveState,
(elapsed, frame): (Duration, usize),
(width, coloured): (usize, bool),
) -> Vec<String> {
let paint = |code: &str, text: String| match coloured && !text.is_empty() {
true => format!("{code}{text}{RESET}"),
false => text,
};
let width = width.min(MAX_WIDTH);
let spinner = paint(CYAN, SPINNER[frame % SPINNER.len()].to_string());
let rows = state.finished_rows
+ state
.running
.iter()
.map(|running| running.rows)
.sum::<u64>();
let seconds = elapsed.as_secs();
let counts = format!(
"{}/{} tables · {:>5} rows · {:02}:{:02}",
state.done,
state.total,
thousands(rows),
seconds / 60,
seconds % 60
);
let badge = match state.follow {
true => " FOLLOWING ",
false => " LOADING ",
};
let bar_width = width
.saturating_sub(badge.len() + BAR_LINE_GAPS + counts.chars().count())
.max(10);
let filled = match state.total {
0 => bar_width,
total => bar_width * state.done / total,
};
let mut lines = vec![
String::new(),
format!(
"{} {}{} {counts}",
paint(&format!("{BOLD}{INVERSE}{CYAN}"), badge.to_owned()),
paint(
match state.stopped_after {
Some(_) => RED,
None => CYAN,
},
"━".repeat(filled)
),
paint(DIM, "━".repeat(bar_width - filled)),
),
String::new(),
];
for running in &state.running {
lines.push(format!(
" {spinner} {:<NAME_COLUMN$} {:<8} {:>6} {}",
running.table,
running.strategy,
thousands(running.rows),
paint(
match running.committing {
true => CYAN,
false => DIM,
},
running.doing.clone()
)
));
}
if let Some(failed) = &state.stopped_after {
lines.push(paint(
YELLOW,
format!(
" ⊘ fail-fast: {} will not start after {failed} failed; running tables finish",
match state.waiting.len() {
1 => "1 table".to_owned(),
count => format!("{count} tables"),
}
),
));
} else if !state.waiting.is_empty() {
let named: Vec<&str> = state
.waiting
.iter()
.take(QUEUED_NAMED)
.map(String::as_str)
.collect();
let more = match state.waiting.len() > QUEUED_NAMED {
true => ", …",
false => "",
};
lines.push(paint(
DIM,
format!(
" · {} queued: {}{more}",
state.waiting.len(),
named.join(", ")
),
));
}
lines.push(String::new());
if !state.rates.is_empty() {
let meters: Vec<String> = state
.rates
.iter()
.map(|(connection, (rate, waited))| {
let busy = waited.is_some_and(|at| elapsed.saturating_sub(at) < RATE_RECENT);
match busy {
true => format!(
"{connection} {}",
paint(YELLOW, format!("▮▮▮▮▮▮▮▮ at {rate}"))
),
false => format!(
"{connection} {}",
paint(DIM, format!("▯▯▯▯▯▯▯▯ under {rate}"))
),
}
})
.collect();
lines.push(format!(
" {}{}",
paint(DIM, "rate ".to_owned()),
meters.join(" ")
));
}
lines.push(paint(
DIM,
match state.follow {
true => " ctrl-c writes what it read and stops · a second ctrl-c stops at once",
false => " ctrl-c finishes running tables and stops · a second ctrl-c stops at once",
}
.to_owned(),
));
lines
}