use crate::adapters::contract::models::{MergeDedupe, WriteMode};
use crate::loader::models::{CommitRule, PlanDetail, TablePlan};
use crate::number_text::main::memory_text::memory_text;
use crate::plan_text::constants::{
BATCH, FILES, INFERRED_SCHEMA, MAXIMUM, MIB, OWN_ROUTES, QUIET_LABELS, ROUTE, SCHEMA,
};
use crate::plan_text::models::LoadedTable;
use crate::project::models::{BatchStep, NullCursor, OnCursorOutOfRange};
use crate::row_deletes::main::deletes_note::deletes_note;
use crate::run_log::models::RunEvent;
use crate::schema_plan::models::TableAction;
fn child(label: &str, text: impl Into<String>) -> PlanDetail {
PlanDetail {
label: label.to_owned(),
text: text.into(),
}
}
pub(crate) fn source_line(source: &str, table: &TablePlan) -> String {
let spec = &table.spec;
let what = match (&spec.files, &spec.python, &spec.warehouse_source) {
(Some(files), _, _) => files.location.to_string(),
(None, Some(python), _) => python.module.clone(),
(None, None, Some(warehouse)) => warehouse.system.to_owned(),
(None, None, None) if spec.sheet.is_some() => "google_sheets".to_owned(),
(None, None, None) => match &spec.cdc {
Some(cdc) => format!("postgres_cdc slot {}", cdc.slot),
None => "postgres".to_owned(),
},
};
match &spec.tunnel {
Some(route) => format!("{source} {what} via {route}"),
None => format!("{source} {what}"),
}
}
pub(crate) fn table_names(
source: &str,
tables: &[&LoadedTable<'_>],
) -> (Option<String>, Vec<String>) {
let prefix = format!("{}__", source.to_lowercase());
let mut split: Vec<(String, String)> = Vec::new();
for loaded in tables {
let (schema, table) = loaded.name.rsplit_once('.').unwrap_or(("", &loaded.name));
let short = match table.get(..prefix.len()) {
Some(head) if head.eq_ignore_ascii_case(&prefix) && table.len() > prefix.len() => {
&table[prefix.len()..]
}
_ => table,
};
split.push((schema.to_owned(), short.to_owned()));
}
let first = split
.first()
.map(|(schema, _)| schema.clone())
.unwrap_or_default();
if split.iter().any(|(schema, _)| *schema != first) {
let names: Vec<String> = split
.into_iter()
.map(|(schema, table)| match schema.is_empty() {
true => table,
false => format!("{schema}.{table}"),
})
.collect();
return (None, names);
}
let names: Vec<String> = split.into_iter().map(|(_, table)| table).collect();
match first.is_empty() || first.eq_ignore_ascii_case(source) {
true => (None, names),
false => (Some(first), names),
}
}
pub(crate) fn action(loaded: &LoadedTable<'_>) -> String {
match &loaded.table.mode {
WriteMode::Merge { unique_key, .. } => format!("merge on {}", unique_key.join(", ")),
WriteMode::DeleteInsert { replace_key, .. } if !replace_key.is_empty() => {
format!("delete_insert on {}", replace_key.join(", "))
}
WriteMode::DeleteInsert {
cursor_column: Some(column),
..
} => format!("delete_insert on {column}"),
WriteMode::DeleteInsert { .. } => "delete_insert".to_owned(),
WriteMode::Append => "append".to_owned(),
_ => loaded.item.action.clone(),
}
}
pub(crate) fn reads(loaded: &LoadedTable<'_>) -> String {
let table = loaded.table;
if table.logged.is_some() {
return logged_reads(table);
}
if table.window.is_none() {
return String::new();
}
let text = loaded
.reads
.clone()
.unwrap_or_default()
.replacen(" >= ", " \u{2265} ", 1);
let (maximum, lookback) = MAXIMUM;
match text.split_once(maximum) {
Some((bound, rest)) => match rest.strip_prefix(lookback) {
Some(lookback) => format!("{bound} (max \u{2212} {lookback}"),
None => format!("{bound} (max{rest}"),
},
None => text,
}
}
fn logged_reads(table: &TablePlan) -> String {
let Some(record) = table
.logged
.as_ref()
.and_then(|logged| logged.current.as_ref())
else {
return "from the start (no run logged yet)".to_owned();
};
let cursor = record
.cursor_column
.as_ref()
.map(|column| {
let at = record.cursor_after.as_deref().unwrap_or("not set");
format!("{column} {at}, ")
})
.unwrap_or_default();
let origin = match record.event {
RunEvent::Reset => "state reset ",
RunEvent::Reconcile => "reconciled ",
RunEvent::Run | RunEvent::Backfill => "run ",
};
format!(
"from {cursor}state {} ({origin}{})",
record.state.as_deref().unwrap_or("not set"),
record.run_id.chars().take(8).collect::<String>()
)
}
pub(crate) fn files_read(loaded: &LoadedTable<'_>) -> (Option<String>, Vec<PlanDetail>) {
let mut details: Vec<PlanDetail> = loaded.item.details.clone();
let count = |text: &str| {
text.ends_with(" listed") && text.contains(" of ") && !text.contains([',', ':', '('])
};
let Some(index) = details
.iter()
.position(|detail| detail.label == FILES && count(&detail.text))
.filter(|_| loaded.table.spec.files.is_some())
else {
return (None, details);
};
let text = details.remove(index).text;
let read = match text.strip_suffix(" listed") {
Some(read) => format!("{read} files"),
None => text,
};
(Some(read), details)
}
pub(crate) fn children(table: &TablePlan, details: &[PlanDetail]) -> Vec<PlanDetail> {
let spec = &table.spec;
let mut lines: Vec<PlanDetail> = cursor_lines(table);
lines.extend(merge_lines(table));
if let Some(note) = deletes_note(spec).strip_prefix(", then ") {
lines.push(child("deletes", note.replace(", then ", "; then ")));
}
if let Some(transform) = &spec.transform {
lines.push(child("transform", transform.function.as_str()));
}
lines.extend(memory_line(table));
for detail in details {
if shown(detail) && !(detail.label == BATCH && spec.files.is_some()) {
lines.push(detail.clone());
}
}
lines
}
fn cursor_lines(table: &TablePlan) -> Vec<PlanDetail> {
let spec = &table.spec;
let mut lines: Vec<PlanDetail> = Vec::new();
let (Some(cursor), Some(window)) = (spec.cursor.as_ref(), table.window.as_ref()) else {
return lines;
};
match cursor.null_cursor {
NullCursor::Include if spec.files.is_none() => {
lines.push(child("", "NULL cursors included"))
}
NullCursor::Ignore if spec.files.is_none() => lines.push(child("", "NULL cursors skipped")),
_ => {}
}
match cursor.out_of_range {
OnCursorOutOfRange::Skip if spec.python.is_some() => {
lines.push(child("", "rows outside a window are skipped"));
}
OnCursorOutOfRange::Warn if spec.python.is_some() => {
lines.push(child("", "rows outside a window load, with a warning"));
}
_ => {}
}
if let Some((zone, at)) = &spec.timezone {
lines.push(child(
"",
format!("times without an offset in {zone} (the {at}'s timezone)"),
));
}
if let (CommitRule::Windows(_), Some(step)) = (&table.commits, &spec.batch) {
match step {
BatchStep::Duration(step) => {
lines.push(child("", format!("commits every {}", every(*step))));
}
BatchStep::Integer(step) => {
lines.push(child(
"",
format!("commits every {step} of {}", window.column),
));
}
BatchStep::Whole => {}
}
}
lines
}
fn merge_lines(table: &TablePlan) -> Vec<PlanDetail> {
let mut lines: Vec<PlanDetail> = Vec::new();
let WriteMode::Merge { dedupe, .. } = &table.mode else {
return lines;
};
match dedupe {
None => lines.push(child("", "repeated keys fail")),
Some(MergeDedupe { newest_by, .. }) if !newest_by.is_empty() => {
lines.push(child(
"",
format!("newest by {}", newest_by.join(", then ")),
));
}
Some(_) => {}
}
if table.schema.action == TableAction::Recreate {
lines.push(child(
"",
"into a new table, swapped in at the end (--reload)",
));
}
lines
}
fn memory_line(table: &TablePlan) -> Option<PlanDetail> {
let fetch = table.fetch_memory.as_ref()?;
let estimate = fetch.estimate.filter(|estimate| *estimate > fetch.budget)?;
Some(child(
"memory",
format!(
"~{} per fetch, over the {} fetch buffer\nset max_batch_memory = \"{}\" on the source",
memory_text(estimate as u64),
memory_text(fetch.budget as u64),
enough(estimate as u64)
),
))
}
fn shown(detail: &PlanDetail) -> bool {
let label = detail.label.as_str();
if QUIET_LABELS.contains(&label) {
return false;
}
match label {
SCHEMA => !detail.text.starts_with(INFERRED_SCHEMA),
ROUTE => OWN_ROUTES.iter().any(|words| detail.text.contains(words)),
_ => true,
}
}
fn every(step: std::time::Duration) -> String {
let seconds = step.as_secs();
let units = [(86_400, "day"), (3_600, "hour"), (60, "minute")];
let Some((size, unit)) = units
.iter()
.find(|(size, _)| seconds.is_multiple_of(*size) && seconds >= *size)
else {
return humantime::format_duration(step).to_string();
};
match seconds / size {
1 => (*unit).to_owned(),
count => format!("{count} {unit}s"),
}
}
fn enough(bytes: u64) -> String {
let (mib, gib) = MIB;
let needed = bytes.div_ceil(mib).next_power_of_two();
match needed.checked_div(gib).filter(|whole| *whole > 0) {
Some(whole) => format!("{whole}GiB"),
None => format!("{needed}MiB"),
}
}