sqlload 0.70.3

SQL data loading tooling.
use std::path::PathBuf;
use std::time::Duration;

use clap::{Parser, Subcommand, ValueEnum};

/// Load data into warehouses: the Kafka exporter, and the loader for projects with [sources].
#[derive(Debug, Parser)]
#[command(name = "sqlload", bin_name = "sqlload", version, about)]
pub struct Cli {
    /// The configuration file.
    #[arg(
        long,
        short,
        global = true,
        default_value = "sqlload.toml",
        env = "SQLLOAD_CONFIG"
    )]
    pub config: PathBuf,
    /// Log output format.
    #[arg(long, global = true, value_enum, default_value_t = LogFormat::Text)]
    pub log_format: LogFormat,
    /// Print without colour (also when `NO_COLOR` is set or the output is not a terminal).
    #[arg(long, global = true)]
    pub no_color: bool,
    #[command(subcommand)]
    pub command: Command,
}

/// The subcommands.
#[derive(Debug, Subcommand)]
pub enum Command {
    /// Validate the configuration and test every connection.
    Debug {
        /// Only validate the configuration; connect to nothing.
        #[arg(long)]
        no_connection: bool,
    },
    /// Show what `apply` and `load` would do (exporter: each topic's table and objects).
    Plan(LoaderArgs),
    /// Print `topic<TAB>table<TAB>stream<TAB>group` per topic (`-` when skipped), without connecting to Snowflake, Databricks or BigQuery.
    Tables {
        /// Read topic names, one per line, from this file instead of Kafka.
        #[arg(long)]
        topics_file: Option<PathBuf>,
    },
    /// Create or change tables (exporter: create missing tables, stages and pipes); never loads data.
    Apply(LoaderArgs),
    /// Load the selected tables of a loader project.
    Load(LoaderArgs),
    /// Export until stopped.
    Run {
        /// Stop after this long, e.g. `10m`.
        #[arg(long, value_parser = humantime::parse_duration)]
        max_duration: Option<Duration>,
        /// This instance's index below `source.shards`.
        #[arg(long, env = "SQLLOAD_SHARD")]
        shard: Option<u32>,
    },
    /// Show per-partition lag and committed offsets.
    Status,
    /// Inspect or change the run log of `state = "logged"` tables.
    LoggedState {
        #[command(subcommand)]
        action: StateAction,
    },
    /// Compare each table's declared columns with its live table, or adopt the live columns into the declarations; exits 1 when they differ.
    Contract {
        #[command(subcommand)]
        action: ContractAction,
    },
    /// Copy one table once, between connections named in sqlload.toml or URIs (sqlload.toml is optional).
    Copy(CopyArgs),
    /// Create a self-contained project that loads CSV files into DuckDB, or with --change add the next change to its data.
    Playground(PlaygroundArgs),
}

/// `sqlload playground`'s folder and template, or `--change`.
#[derive(Debug, Clone, PartialEq, Eq, clap::Args)]
pub struct PlaygroundArgs {
    /// The new project's folder (default: shop).
    #[arg(value_name = "NAME", value_parser = clap::builder::NonEmptyStringValueParser::new())]
    pub name: Option<String>,
    /// The project to create; `shop` is the only one.
    #[arg(long, value_enum, default_value_t = PlaygroundTemplate::Shop)]
    pub template: PlaygroundTemplate,
    /// Add the next change to the data of the playground in this folder, for an incremental load.
    #[arg(long, conflicts_with_all = ["name", "template"])]
    pub change: bool,
}

/// The projects `sqlload playground` creates.
#[derive(Debug, Clone, Copy, PartialEq, Eq, ValueEnum)]
pub enum PlaygroundTemplate {
    /// Customers and orders as CSV files, merged into DuckDB.
    Shop,
}

/// What `sqlload contract` does.
#[derive(Debug, Clone, PartialEq, Eq, Subcommand)]
pub enum ContractAction {
    /// Report each selected table's declared columns that differ from its live table, and on `contract = "enforced"` tables its undeclared live columns.
    Diff(ContractArgs),
    /// Propose declared `columns` from each selected table's live table; `--write` writes them into sqlload.toml (Python tables print theirs to paste).
    Generate(ContractGenerateArgs),
}

/// `sqlload contract generate`'s selection and what it may change.
#[derive(Debug, Clone, Default, PartialEq, Eq, clap::Args)]
pub struct ContractGenerateArgs {
    #[command(flatten)]
    pub tables: ContractArgs,
    /// Write the declarations into sqlload.toml.
    #[arg(long)]
    pub write: bool,
    /// Also retype declared columns whose live type differs, and remove declared columns the live table lacks.
    #[arg(long, requires = "write")]
    pub overwrite: bool,
}

/// Table selection and output for `sqlload contract`.
#[derive(Debug, Clone, Default, PartialEq, Eq, clap::Args)]
pub struct ContractArgs {
    /// Sources or tables to compare, as for `plan`; all when neither this nor --select is given.
    #[arg(value_name = "SELECTOR")]
    pub positional_select: Vec<String>,
    /// Sources or tables to compare, combined with any given positionally.
    #[arg(long, short = 's', num_args = 1.., value_name = "SELECTOR")]
    pub select: Vec<String>,
    /// Sources or tables to leave out; wins over --select.
    #[arg(long, num_args = 1.., value_name = "SELECTOR")]
    pub exclude: Vec<String>,
    /// Read every table as if its source loaded into this target, in that target's database and schema.
    #[arg(long, value_name = "TARGET")]
    pub from: Option<String>,
    /// Print the report as one JSON document.
    #[arg(long)]
    pub json: bool,
    /// On files targets, also read every file's footer and report the files whose columns differ from the newest load's.
    #[arg(long)]
    pub all_files: bool,
    /// Also report where what each table's source would load differs from the live table, for columns no declaration names.
    #[arg(long)]
    pub source: bool,
}

/// `sqlload copy`'s endpoints and write settings.
#[derive(Debug, Clone, Default, PartialEq, Eq, clap::Args)]
pub struct CopyArgs {
    /// What to copy: `connection:schema.table`, `postgresql://user:${ENV:PGPASSWORD}@host/db#schema.table`, `s3://bucket/prefix/`, `file:///folder` or a folder.
    pub source: String,
    /// Where to: `connection:SCHEMA.TABLE`, `duckdb://file.duckdb#SCHEMA.TABLE`, `s3://bucket/prefix/`, `file:///folder` or a folder.
    pub destination: String,
    /// append, merge, delete_insert or replace (default: the loader's).
    #[arg(long)]
    pub write_strategy: Option<String>,
    /// The key columns for merge and delete_insert.
    #[arg(long, num_args = 1.., value_delimiter = ',')]
    pub unique_key: Vec<String>,
    /// The column incremental copies continue from.
    #[arg(long)]
    pub cursor_column: Option<String>,
    /// The format of a files source's files: parquet, ndjson, json or csv.
    #[arg(long)]
    pub format: Option<String>,
    /// Print the copy as a project entry for sqlload.toml instead of running it.
    #[arg(long)]
    pub print_toml: bool,
}

/// What `sqlload logged-state` does.
#[derive(Debug, Clone, PartialEq, Eq, Subcommand)]
pub enum StateAction {
    /// Show each logged table's run log and current state; read-only.
    Show(StateArgs),
    /// Compare each logged table's logged cursor with the data; read-only, exits 1 when they disagree.
    Verify(StateArgs),
    /// Append a reconciliation adopting the data's position where the log disagrees (with --yes; otherwise only shows it).
    Reconcile(StateArgs),
    /// Append a reset: the next run reads from --to, or from the beginning with --clear (with --yes; otherwise only shows it).
    Reset(ResetArgs),
}

/// Table selection for `sqlload logged-state`, and confirmation for the commands that append.
#[derive(Debug, Clone, Default, PartialEq, Eq, clap::Args)]
pub struct StateArgs {
    /// Sources or tables to select; all when empty.
    pub select: Vec<String>,
    /// Append the events instead of only showing them.
    #[arg(long)]
    pub yes: bool,
}

/// `sqlload logged-state reset`'s position.
#[derive(Debug, Clone, Default, PartialEq, Eq, clap::Args)]
pub struct ResetArgs {
    #[command(flatten)]
    pub tables: StateArgs,
    /// The position to read from next: a cursor value for tables with a cursor_column, else the `ctx.state` value as JSON.
    #[arg(long, required_unless_present = "clear", conflicts_with = "clear")]
    pub to: Option<String>,
    /// Forget the position: the next run reads from the beginning (cursor_start, or everything).
    #[arg(long)]
    pub clear: bool,
}

/// Table selection and policy flags for loader commands.
#[derive(Debug, Clone, Default, PartialEq, Eq, clap::Args)]
pub struct LoaderArgs {
    /// Sources or tables to select (source names, destination table names or `schema.table`), the same as --select; all when neither is given.
    #[arg(value_name = "SELECTOR")]
    pub positional_select: Vec<String>,
    /// Sources or tables to select, combined with any given positionally.
    #[arg(long, short = 's', num_args = 1.., value_name = "SELECTOR")]
    pub select: Vec<String>,
    /// Sources or tables to leave out, named as for --select; wins over --select.
    #[arg(long, num_args = 1.., value_name = "SELECTOR")]
    pub exclude: Vec<String>,
    /// Allow table rebuilds that `type_change_rebuild = "require_confirmation"` would otherwise block.
    #[arg(long)]
    pub allow_type_change_rebuild: bool,
    /// Rebuild the selected tables from the whole source, with the current columns and types; asks first unless --yes.
    #[arg(long)]
    pub reload: bool,
    /// Rebuild with --reload or --resnapshot without asking.
    #[arg(long)]
    pub yes: bool,
    /// postgres_cdc sources: drop and recreate the replication slot, then copy every table again (after a snapshot stopped part-way, or a lost slot); asks first unless --yes.
    #[arg(long)]
    pub resnapshot: bool,
    /// After each selected table with `deletes` merges, read every key from its source and mark the table's rows whose key it no longer holds.
    #[arg(long, conflicts_with = "follow")]
    pub sync_deletes: bool,
    /// Backfill from this cursor value instead of the destination's maximum, typed from each selected table's cursor (an integer, or RFC 3339 or a date).
    #[arg(long, value_name = "VALUE")]
    pub start_cursor: Option<String>,
    /// Backfill only cursor values below this one, typed from each selected table's cursor.
    #[arg(long, value_name = "VALUE")]
    pub end_cursor: Option<String>,
    /// Backfill the selected date and timestamp cursor tables from this RFC 3339 timestamp or date; not with --start-cursor/--end-cursor.
    #[arg(long, value_name = "TIMESTAMP")]
    pub start_cursor_ts: Option<String>,
    /// Backfill the selected date and timestamp cursor tables only below this RFC 3339 timestamp or date.
    #[arg(long, value_name = "TIMESTAMP")]
    pub end_cursor_ts: Option<String>,
    /// Backfill the selected integer cursor tables from this integer; not with --start-cursor/--end-cursor.
    #[arg(long, value_name = "INTEGER", allow_hyphen_values = true)]
    pub start_cursor_int: Option<String>,
    /// Backfill the selected integer cursor tables only below this integer.
    #[arg(long, value_name = "INTEGER", allow_hyphen_values = true)]
    pub end_cursor_int: Option<String>,
    /// `load`: once a table fails, start no other table; tables already loading finish, and the run exits 1, or 75 when the failure is retryable.
    #[arg(long, conflicts_with = "follow")]
    pub fail_fast: bool,
    /// The most tables loading at once (default: the project's `concurrency`, else 4); one table's batches always commit one at a time.
    #[arg(long, value_parser = clap::value_parser!(u16).range(1..))]
    pub concurrency: Option<u16>,
    /// The memory the run stays under, such as `4GiB`, or `off` for no budget, overriding the project's `memory_limit` (default: 80% of a container's or job object's limit, else 50% of the machine's memory).
    #[arg(long, value_name = "SIZE|off")]
    pub memory_limit: Option<crate::memory_budget::models::MemorySetting>,
    /// Print one JSON document on stdout: the plan for `plan`; for `apply` and `load`, the plan and what was created, altered and loaded.
    #[arg(long)]
    pub json: bool,
    /// `load` of one postgres_cdc source only: keep streaming after catching up, writing a batch every `batch_interval`, until --max-duration passes or Ctrl-C/SIGTERM (a second one stops at once).
    #[arg(long, conflicts_with_all = ["reload", "start_cursor", "end_cursor", "start_cursor_ts", "end_cursor_ts", "start_cursor_int", "end_cursor_int"])]
    pub follow: bool,
    /// With --follow: stop after this long, e.g. `10m`, once the buffered changes are written.
    #[arg(long, requires = "follow", value_parser = humantime::parse_duration)]
    pub max_duration: Option<Duration>,
    /// `load`: write each load event as one line of JSON to stderr (`jsonl`), apart from the logs.
    #[arg(long, value_enum, value_name = "FORMAT", conflicts_with = "follow")]
    pub events: Option<EventsFormat>,
    /// `load`: write each load event as one line of JSON to this file.
    #[arg(long, value_name = "PATH", conflicts_with_all = ["events", "follow"])]
    pub events_file: Option<PathBuf>,
    /// `load`: draw the live block of running tables on stderr: on a terminal without --json or --log-format json (`auto`), always, or never.
    #[arg(long, value_enum, default_value_t = Progress::Auto)]
    pub progress: Progress,
    /// `load`, Linux only: load one table at a time and report each table's peak memory (the run's peak is always reported).
    #[arg(long, conflicts_with = "follow")]
    pub memory_per_table: bool,
    /// The bytes a batch holds before it may commit, instead of about 1 GB (for tests).
    #[arg(long, hide = true, value_parser = clap::value_parser!(u64).range(1..))]
    pub batch_bytes: Option<u64>,
    /// Show every line of the plan, the defaults each table follows too, and every connection's and discovery's own line.
    #[arg(long, short = 'v')]
    pub verbose: bool,
}

/// How `--events` writes load events.
#[derive(Debug, Clone, Copy, PartialEq, Eq, ValueEnum)]
pub enum EventsFormat {
    /// One JSON object per line, on stderr.
    Jsonl,
}

/// When `load` draws its live block.
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, ValueEnum)]
pub enum Progress {
    /// When stderr is a terminal, and neither --json nor --log-format json is set.
    #[default]
    Auto,
    Always,
    Never,
}

/// How logs are written.
#[derive(Debug, Clone, Copy, PartialEq, Eq, ValueEnum)]
pub enum LogFormat {
    Text,
    Json,
}

/// A loader command to run on a project's text.
#[derive(Debug)]
pub struct LoaderRequest<'a> {
    pub text: &'a str,
    /// The project file, whose folder `python_paths` are relative to.
    pub config: &'a std::path::Path,
    pub command: Command,
    /// Runs Python sources; `None` in the standalone executable.
    pub host: Option<std::sync::Arc<dyn crate::python_sources::types::PythonHost>>,
    /// `--no-color`, for the plan a `--reload` prompt shows.
    pub no_color: bool,
    /// Print `load`'s lines as tables finish, rather than return them all at the end (the command line).
    pub streams: bool,
    /// `--log-format json`, under which no live block is drawn.
    pub log_json: bool,
    /// Values for `${ENV:NAME}` references, read before the process environment (`sqlload copy`'s literal URI passwords).
    pub variables: std::collections::BTreeMap<String, String>,
    /// `text` is a project `sqlload copy` wrote, not the user's file: hints name no line of it and no variable it reads.
    pub generated: bool,
}