sqlload 0.69.1

SQL data loading tooling.
//! Load every unblocked table of a plan, reporting each table's outcome.

use tokio_util::sync::CancellationToken;

use crate::loader::_helpers::destination::clean_stages;
use crate::loader::_helpers::destination::clean_staging;
use crate::loader::_helpers::table_queue::{
    TableQueue, follow_tables, shared_run, thread_count, watch_load,
};
use crate::loader::models::{LoadRun, ProjectPlan, TableOutcome};

/// Load each unblocked table, Python tables through `host`, each on a thread of its own, up to `concurrency` at once, or fewer when the memory budget's table shares don't fit that many (one table's batches commit one at a time), none starting while the watchdog holds the load at 80% of its budget, so tables use several cores (decoding, masking and metadata included) and a slow one never holds up the others; tables of a Python source sharing a client run one at a time unless it sets `parallel_tables`. A table that fails is reported and the others still load, unless `--fail-fast` starts no table after it (those not started, `postgres_cdc` sources' included, are reported skipped); outcomes are in plan order. Load folders and staging tables crashed runs left are removed first. `postgres_cdc` sources load after the other tables, each as one unit. Given `follow` (`load --follow`), the plan's one `postgres_cdc` source is followed instead until it is cancelled (see [`follow_tables`]).
pub async fn load_tables(
    run: &LoadRun<'_>,
    plan: &ProjectPlan,
    follow: Option<&CancellationToken>,
) -> Vec<TableOutcome> {
    clean_stages(run, plan).await;
    clean_staging(run, plan).await;
    if let Some(stop) = follow {
        return follow_tables(run, plan, stop).await;
    }
    let captured = plan;
    let plan = &plan.uncaptured();
    let guard = watch_load(run);
    let threads = thread_count(run);
    let shared = shared_run(run, guard.clone());
    let mut queue = TableQueue::new(run.project, plan, run.options.fail_fast);
    queue.load_all(threads, (shared, guard)).await;
    let cdc = queue.cdc_outcomes(run, captured).await;
    let mut outcomes = queue.outcomes();
    outcomes.extend(cdc);
    outcomes
}