sqlload 0.70.1

SQL data loading tooling.
//! Load every unblocked table of a plan, reporting each table's outcome.

use tokio_util::sync::CancellationToken;

use crate::loader::_helpers::destination::clean_stages;
use crate::loader::_helpers::destination::clean_staging;
use crate::loader::_helpers::table_queue::{
    TableQueue, emit_outcome, follow_tables, shared_run, thread_count, watch_load,
};
use crate::loader::models::{LoadRun, ProjectPlan, TableOutcome};
use crate::memory_budget::main::run_stopped::run_stopped;

/// Load each unblocked table, Python tables through `host`, each on a thread of its own, up to `concurrency` at once, or fewer when the memory budget's table shares don't fit that many (one table's batches commit one at a time), none starting while the watchdog holds the load at 80% of its budget, so tables use several cores (decoding, masking and metadata included) and a slow one never holds up the others; tables of a Python source sharing a client run one at a time unless it sets `parallel_tables`. A table that fails is reported and the others still load, unless `--fail-fast` starts no table after it (those not started, `postgres_cdc` sources' included, are reported skipped); outcomes are in plan order. When the memory watchdog stops the run, the tables running fail and none other starts. Load folders and staging tables crashed runs left are removed first. `postgres_cdc` sources load after the other tables, each as one unit. Given `follow` (`load --follow`), the plan's one `postgres_cdc` source is followed instead until it is cancelled (see [`follow_tables`]).
pub async fn load_tables(
    run: &LoadRun<'_>,
    plan: &ProjectPlan,
    follow: Option<&CancellationToken>,
) -> Vec<TableOutcome> {
    clean_stages(run, plan).await;
    clean_staging(run, plan).await;
    if let Some(stop) = follow {
        return follow_tables(run, plan, stop).await;
    }
    let captured = plan;
    let plan = &plan.uncaptured();
    let guard = watch_load(run);
    let threads = thread_count(run);
    let shared = shared_run(run, guard.clone());
    let mut queue = TableQueue::new(run.project, plan, run.options.fail_fast);
    queue.load_all(threads, (shared, guard)).await;
    let cdc = match run_stopped() {
        true => (captured.tables.iter())
            .filter(|table| table.spec.cdc.is_some() && table.schema.blocked.is_none())
            .map(TableOutcome::run_stopped)
            .inspect(emit_outcome)
            .collect(),
        false => queue.cdc_outcomes(run, captured).await,
    };
    let mut outcomes = queue.outcomes();
    outcomes.extend(cdc);
    outcomes
}