use std::collections::{BTreeMap, BTreeSet};
use std::sync::Arc;
use tokio::task::{Id, JoinError};
use tokio_util::sync::CancellationToken;
use crate::adapters::contract::models::WriteMode;
use crate::cdc::main::follow_cdc::follow_cdc;
use crate::cdc::main::load_cdc::load_cdc;
use crate::load_events::main::emit::emit;
use crate::load_events::models::EventKind;
use crate::loader::_helpers::run_log::record_numbered;
use crate::loader::constants::DEFAULT_THREADS;
use crate::loader::main::_load_table::load_one;
use crate::loader::models::{LoadRun, ProjectPlan, SharedRun, TableOutcome, TablePlan};
use crate::peak_memory::main::with_table_peak::with_table_peak;
use crate::project::models::ProjectConfig;
pub(crate) fn thread_count(run: &LoadRun<'_>) -> usize {
if run.options.memory_per_table {
return 1;
}
run.options
.concurrency
.or(run.project.concurrency)
.unwrap_or(DEFAULT_THREADS)
.max(1)
}
pub(crate) fn shared_run(run: &LoadRun<'_>) -> Arc<SharedRun> {
Arc::new(SharedRun {
project: run.project.clone(),
destinations: run.destinations.clone(),
host: run.host.clone(),
options: run.options.clone(),
})
}
pub(crate) async fn load_task(shared: Arc<SharedRun>, table: TablePlan) -> TableOutcome {
let run = LoadRun {
project: &shared.project,
destinations: &shared.destinations,
host: shared.host.clone(),
options: &shared.options,
};
let started = std::time::Instant::now();
let (loaded, peak_memory_bytes) = with_table_peak(
shared.options.memory_per_table,
Box::pin(load_one(&run, &table)),
)
.await;
match loaded {
Ok((name, loaded)) => TableOutcome {
table: name,
origin: Some(table.origin()),
loaded: Ok(loaded),
retryable: false,
elapsed: started.elapsed(),
skipped: false,
peak_memory_bytes,
},
Err(error) => TableOutcome {
table: table.outcome_name(),
origin: Some(table.origin()),
retryable: error.is_retryable(),
loaded: Err(error.to_string()),
elapsed: started.elapsed(),
skipped: false,
peak_memory_bytes,
},
}
}
pub(crate) async fn follow_tables(
run: &LoadRun<'_>,
plan: &ProjectPlan,
stop: &CancellationToken,
) -> Vec<TableOutcome> {
for table in plan
.tables
.iter()
.filter(|table| table.schema.blocked.is_none())
{
emit(
Some(&table.outcome_name()),
EventKind::TableStarted {
strategy: strategy_name(table).to_owned(),
},
);
}
record_cdc_names(run, plan).await;
let outcomes = Box::pin(follow_cdc(run, plan.clone(), stop)).await;
outcomes.iter().for_each(emit_outcome);
outcomes
}
async fn record_cdc_names(run: &LoadRun<'_>, plan: &ProjectPlan) {
let captured = plan
.tables
.iter()
.filter(|table| table.spec.cdc.is_some() && table.schema.blocked.is_none());
for table in captured {
let Some(destination) = run.destinations.get(&table.spec.target) else {
continue;
};
if let Err(error) = Box::pin(record_numbered(table, destination)).await {
tracing::warn!(
table = %table.spec.destination.table,
"could not record the names numbering gave its columns: {error}"
);
}
}
}
pub(crate) fn strategy_name(table: &TablePlan) -> &'static str {
match table.mode {
WriteMode::Merge { .. } => "merge",
WriteMode::DeleteInsert { .. } => "delete_insert",
WriteMode::Append => "append",
WriteMode::Replace => "replace",
WriteMode::Recreate => "recreate",
WriteMode::Changes(_) => "changes",
WriteMode::MarkMissing(_) => "sync_deletes",
}
}
pub(crate) fn emit_outcome(outcome: &TableOutcome) {
let elapsed_ms = u64::try_from(outcome.elapsed.as_millis()).unwrap_or(u64::MAX);
let kind = match &outcome.loaded {
Ok(loaded) => EventKind::TableDone {
rows: loaded.rows,
elapsed_ms,
position: loaded.note.clone(),
changes: loaded.changes.clone(),
transforms: loaded.transforms.clone(),
names: loaded.names.clone(),
warnings: loaded.warnings.clone(),
peak_memory_bytes: outcome.peak_memory_bytes,
},
Err(reason) if outcome.skipped => EventKind::TableNotStarted {
reason: reason.clone(),
},
Err(reason) => EventKind::TableFailed {
reason: reason.clone(),
retryable: outcome.retryable,
elapsed_ms,
peak_memory_bytes: outcome.peak_memory_bytes,
},
};
emit(Some(&outcome.table), kind);
}
fn destination_key(table: &TablePlan) -> String {
let destination = &table.spec.destination;
format!(
"{}\u{0}{}\u{0}{}\u{0}{}",
table.spec.target,
destination.database.as_deref().unwrap_or_default(),
destination.schema,
destination.table
)
}
fn tombstones_loading(table: &TablePlan, unfinished: &[&TablePlan]) -> bool {
let from = table
.spec
.deletes
.as_ref()
.and_then(|deletes| deletes.from.as_ref());
let Some((destination, _)) = from.and_then(|from| from.loaded.as_ref()) else {
return false;
};
unfinished.iter().any(|other| {
other.spec.target == table.spec.target && other.spec.destination == *destination
})
}
pub(crate) struct TableQueue<'a> {
project: &'a ProjectConfig,
tables: Vec<&'a TablePlan>,
waiting: Vec<usize>,
busy: BTreeSet<&'a str>,
writing: BTreeSet<String>,
started: BTreeMap<Id, usize>,
outcomes: Vec<(usize, TableOutcome)>,
fail_fast: bool,
stopped: Option<String>,
}
impl<'a> TableQueue<'a> {
pub(crate) fn new(project: &'a ProjectConfig, plan: &'a ProjectPlan, fail_fast: bool) -> Self {
let tables: Vec<&TablePlan> = plan
.tables
.iter()
.filter(|table| table.schema.blocked.is_none())
.collect();
Self {
project,
waiting: (0..tables.len()).collect(),
tables,
busy: BTreeSet::new(),
writing: BTreeSet::new(),
started: BTreeMap::new(),
outcomes: Vec::new(),
fail_fast,
stopped: None,
}
}
pub(crate) async fn cdc_outcomes(
&self,
run: &LoadRun<'_>,
plan: &ProjectPlan,
) -> Vec<TableOutcome> {
let captured = plan.tables.iter().filter(|table| {
table.spec.cdc.is_some() && table.schema.blocked.is_none() && self.stopped.is_none()
});
for table in captured {
emit(
Some(&table.outcome_name()),
EventKind::TableStarted {
strategy: strategy_name(table).to_owned(),
},
);
}
if self.stopped.is_none() {
record_cdc_names(run, plan).await;
}
let outcomes = load_cdc(run, plan, self.stopped.clone()).await;
outcomes.iter().for_each(emit_outcome);
outcomes
}
pub(crate) fn next(&mut self) -> Option<(usize, &'a TablePlan)> {
if self.stopped.is_some() {
return None;
}
let tables = &self.tables;
let busy = &self.busy;
let writing = &self.writing;
let unfinished: Vec<&TablePlan> = self
.waiting
.iter()
.chain(self.started.values())
.map(|index| tables[*index])
.collect();
let position = self.waiting.iter().position(|index| {
let table = tables[*index];
!busy.contains(table.spec.source_name.as_str())
&& !writing.contains(&destination_key(table))
&& !tombstones_loading(table, &unfinished)
})?;
let index = self.waiting.remove(position);
let table = self.tables[index];
emit(
Some(&table.outcome_name()),
EventKind::TableStarted {
strategy: strategy_name(table).to_owned(),
},
);
self.writing.insert(destination_key(table));
let source = self.project.sources.get(&table.spec.source_name);
if source.is_some_and(|source| source.shared_client && !source.parallel_tables) {
self.busy.insert(table.spec.source_name.as_str());
}
Some((index, table))
}
pub(crate) fn started(&mut self, id: Id, index: usize) {
self.started.insert(id, index);
}
pub(crate) fn finish(&mut self, joined: Result<(Id, TableOutcome), JoinError>) {
let (id, outcome) = match joined {
Ok((id, outcome)) => (id, Ok(outcome)),
Err(error) => (error.id(), Err(error.to_string())),
};
let Some(index) = self.started.remove(&id) else {
return;
};
let table = self.tables[index];
self.busy.remove(table.spec.source_name.as_str());
self.writing.remove(&destination_key(table));
let outcome = outcome.unwrap_or_else(|reason| TableOutcome {
table: table.outcome_name(),
origin: Some(table.origin()),
loaded: Err(reason),
retryable: false,
elapsed: std::time::Duration::ZERO,
skipped: false,
peak_memory_bytes: None,
});
emit_outcome(&outcome);
if self.fail_fast && self.stopped.is_none() && outcome.loaded.is_err() {
self.stopped = Some(outcome.table.clone());
}
self.outcomes.push((index, outcome));
}
pub(crate) fn outcomes(mut self) -> Vec<TableOutcome> {
if let Some(failed) = &self.stopped {
for index in &self.waiting {
let skipped = TableOutcome::skipped(self.tables[*index], failed);
emit_outcome(&skipped);
self.outcomes.push((*index, skipped));
}
}
self.outcomes.sort_by_key(|(index, _)| *index);
self.outcomes
.into_iter()
.map(|(_, outcome)| outcome)
.collect()
}
}