1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
use std::path::PathBuf;
use std::time::Duration;
use clap::{Parser, Subcommand, ValueEnum};
/// Load data into warehouses: the Kafka exporter, and the loader for projects with [sources].
#[derive(Debug, Parser)]
#[command(name = "sqlload", version, about)]
pub struct Cli {
/// The configuration file.
#[arg(
long,
short,
global = true,
default_value = "sqlload.toml",
env = "SQLLOAD_CONFIG"
)]
pub config: PathBuf,
/// Log output format.
#[arg(long, global = true, value_enum, default_value_t = LogFormat::Text)]
pub log_format: LogFormat,
/// Print without colour (also when `NO_COLOR` is set or the output is not a terminal).
#[arg(long, global = true)]
pub no_color: bool,
#[command(subcommand)]
pub command: Command,
}
/// The subcommands.
#[derive(Debug, Subcommand)]
pub enum Command {
/// Validate the configuration and test every connection.
Debug {
/// Only validate the configuration; connect to nothing.
#[arg(long)]
no_connection: bool,
},
/// Show what `apply` and `load` would do (exporter: each topic's table and objects).
Plan(LoaderArgs),
/// Print `topic<TAB>table<TAB>stream<TAB>group` per topic (`-` when skipped), without Snowflake.
Tables {
/// Read topic names, one per line, from this file instead of Kafka.
#[arg(long)]
topics_file: Option<PathBuf>,
},
/// Create or change tables (exporter: create missing tables, stages and pipes); never loads data.
Apply(LoaderArgs),
/// Load the selected tables of a loader project.
Load(LoaderArgs),
/// Export until stopped.
Run {
/// Stop after this long, e.g. `10m`.
#[arg(long, value_parser = humantime::parse_duration)]
max_duration: Option<Duration>,
/// This instance's index below `source.shards`.
#[arg(long, env = "SQLLOAD_SHARD")]
shard: Option<u32>,
},
/// Show per-partition lag and committed offsets.
Status,
/// Inspect or change the run log of `state = "logged"` tables.
LoggedState {
#[command(subcommand)]
action: StateAction,
},
/// Compare each table's declared columns with its live table, or adopt the live columns into the declarations; exits 1 when they differ.
Contract {
#[command(subcommand)]
action: ContractAction,
},
/// Copy one table once, between connections named in sqlload.toml or URIs (sqlload.toml is optional).
Copy(CopyArgs),
}
/// What `sqlload contract` does.
#[derive(Debug, Clone, PartialEq, Eq, Subcommand)]
pub enum ContractAction {
/// Report each selected table's declared columns that differ from its live table, and on `contract = "enforced"` tables its undeclared live columns.
Diff(ContractArgs),
/// Propose declared `columns` from each selected table's live table; `--write` writes them into sqlload.toml (Python tables print theirs to paste).
Generate(ContractGenerateArgs),
}
/// `sqlload contract generate`'s selection and what it may change.
#[derive(Debug, Clone, Default, PartialEq, Eq, clap::Args)]
pub struct ContractGenerateArgs {
#[command(flatten)]
pub tables: ContractArgs,
/// Write the declarations into sqlload.toml.
#[arg(long)]
pub write: bool,
/// Also retype declared columns whose live type differs, and remove declared columns the live table lacks.
#[arg(long, requires = "write")]
pub overwrite: bool,
}
/// Table selection and output for `sqlload contract`.
#[derive(Debug, Clone, Default, PartialEq, Eq, clap::Args)]
pub struct ContractArgs {
/// Sources or tables to compare, as for `plan`; all when neither this nor --select is given.
#[arg(value_name = "SELECTOR")]
pub positional_select: Vec<String>,
/// Sources or tables to compare, combined with any given positionally.
#[arg(long, short = 's', num_args = 1.., value_name = "SELECTOR")]
pub select: Vec<String>,
/// Sources or tables to leave out; wins over --select.
#[arg(long, num_args = 1.., value_name = "SELECTOR")]
pub exclude: Vec<String>,
/// Read every table as if its source loaded into this target, in that target's database and schema.
#[arg(long, value_name = "TARGET")]
pub from: Option<String>,
/// Print the report as one JSON document.
#[arg(long)]
pub json: bool,
/// On files targets, also read every file's footer and report the files whose columns differ from the newest load's.
#[arg(long)]
pub all_files: bool,
/// Also report where what each table's source would load differs from the live table, for columns no declaration names.
#[arg(long)]
pub source: bool,
}
/// `sqlload copy`'s endpoints and write settings.
#[derive(Debug, Clone, Default, PartialEq, Eq, clap::Args)]
pub struct CopyArgs {
/// What to copy: `connection:schema.table`, `postgresql://user:${ENV:PGPASSWORD}@host/db#schema.table`, `s3://bucket/prefix/`, `file:///folder` or a folder.
pub source: String,
/// Where to: `connection:SCHEMA.TABLE`, `duckdb://file.duckdb#SCHEMA.TABLE`, `s3://bucket/prefix/`, `file:///folder` or a folder.
pub destination: String,
/// append, merge, delete_insert or replace (default: the loader's).
#[arg(long)]
pub write_strategy: Option<String>,
/// The key columns for merge and delete_insert.
#[arg(long, num_args = 1.., value_delimiter = ',')]
pub unique_key: Vec<String>,
/// The column incremental copies continue from.
#[arg(long)]
pub cursor_column: Option<String>,
/// The format of a files source's files: parquet, ndjson, json or csv.
#[arg(long)]
pub format: Option<String>,
/// Print the copy as a project entry for sqlload.toml instead of running it.
#[arg(long)]
pub print_toml: bool,
}
/// What `sqlload logged-state` does.
#[derive(Debug, Clone, PartialEq, Eq, Subcommand)]
pub enum StateAction {
/// Show each logged table's run log and current state; read-only.
Show(StateArgs),
/// Compare each logged table's logged cursor with the data; read-only, exits 1 when they disagree.
Verify(StateArgs),
/// Append a reconciliation adopting the data's position where the log disagrees (with --yes; otherwise only shows it).
Reconcile(StateArgs),
/// Append a reset: the next run reads from --to, or from the beginning with --clear (with --yes; otherwise only shows it).
Reset(ResetArgs),
}
/// Table selection for `sqlload logged-state`, and confirmation for the commands that append.
#[derive(Debug, Clone, Default, PartialEq, Eq, clap::Args)]
pub struct StateArgs {
/// Sources or tables to select; all when empty.
pub select: Vec<String>,
/// Append the events instead of only showing them.
#[arg(long)]
pub yes: bool,
}
/// `sqlload logged-state reset`'s position.
#[derive(Debug, Clone, Default, PartialEq, Eq, clap::Args)]
pub struct ResetArgs {
#[command(flatten)]
pub tables: StateArgs,
/// The position to read from next: a cursor value for tables with a cursor_column, else the `ctx.state` value as JSON.
#[arg(long, required_unless_present = "clear", conflicts_with = "clear")]
pub to: Option<String>,
/// Forget the position: the next run reads from the beginning (cursor_start, or everything).
#[arg(long)]
pub clear: bool,
}
/// Table selection and policy flags for loader commands.
#[derive(Debug, Clone, Default, PartialEq, Eq, clap::Args)]
pub struct LoaderArgs {
/// Sources or tables to select (source names, destination table names or `schema.table`), the same as --select; all when neither is given.
#[arg(value_name = "SELECTOR")]
pub positional_select: Vec<String>,
/// Sources or tables to select, combined with any given positionally.
#[arg(long, short = 's', num_args = 1.., value_name = "SELECTOR")]
pub select: Vec<String>,
/// Sources or tables to leave out, named as for --select; wins over --select.
#[arg(long, num_args = 1.., value_name = "SELECTOR")]
pub exclude: Vec<String>,
/// Allow table rebuilds that `type_change_rebuild = "require_confirmation"` would otherwise block.
#[arg(long)]
pub allow_type_change_rebuild: bool,
/// Rebuild the selected tables from the whole source, with the current columns and types; asks first unless --yes.
#[arg(long)]
pub reload: bool,
/// Rebuild with --reload or --resnapshot without asking.
#[arg(long)]
pub yes: bool,
/// postgres_cdc sources: drop and recreate the replication slot, then copy every table again (after a snapshot stopped part-way, or a lost slot); asks first unless --yes.
#[arg(long)]
pub resnapshot: bool,
/// Backfill from this cursor value instead of the destination's maximum, typed from each selected table's cursor (an integer, or RFC 3339 or a date).
#[arg(long, value_name = "VALUE")]
pub start_cursor: Option<String>,
/// Backfill only cursor values below this one, typed from each selected table's cursor.
#[arg(long, value_name = "VALUE")]
pub end_cursor: Option<String>,
/// The most tables loading at once (default: the project's `concurrency`, else 4); one table's batches always commit one at a time.
#[arg(long, value_parser = clap::value_parser!(u16).range(1..))]
pub concurrency: Option<u16>,
/// Print one JSON document on stdout: the plan for `plan`; for `apply` and `load`, the plan and what was created, altered and loaded.
#[arg(long)]
pub json: bool,
/// `load` of one postgres_cdc source only: keep streaming after catching up, writing a batch every `batch_interval`, until --max-duration passes or Ctrl-C/SIGTERM (a second one stops at once).
#[arg(long, conflicts_with_all = ["reload", "start_cursor", "end_cursor"])]
pub follow: bool,
/// With --follow: stop after this long, e.g. `10m`, once the buffered changes are written.
#[arg(long, requires = "follow", value_parser = humantime::parse_duration)]
pub max_duration: Option<Duration>,
/// The bytes a batch holds before it may commit, instead of about 1 GB (for tests).
#[arg(long, hide = true, value_parser = clap::value_parser!(u64).range(1..))]
pub batch_bytes: Option<u64>,
}
/// How logs are written.
#[derive(Debug, Clone, Copy, PartialEq, Eq, ValueEnum)]
pub enum LogFormat {
Text,
Json,
}
/// A loader command to run on a project's text.
#[derive(Debug)]
pub struct LoaderRequest<'a> {
pub text: &'a str,
/// The project file, whose folder `python_paths` are relative to.
pub config: &'a std::path::Path,
pub command: Command,
/// Runs Python sources; `None` in the standalone executable.
pub host: Option<std::sync::Arc<dyn crate::python_sources::types::PythonHost>>,
/// `--no-color`, for the plan a `--reload` prompt shows.
pub no_color: bool,
/// Values for `${ENV:NAME}` references, read before the process environment (`sqlload copy`'s literal URI passwords).
pub variables: std::collections::BTreeMap<String, String>,
}