sqlload 0.70.1

SQL data loading tooling.
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
use crate::load_naming::models::{Compat, MetadataRole};
use std::collections::{BTreeMap, BTreeSet};
use std::time::Duration;

use arrow_schema::SchemaRef;
use chrono::NaiveDate;
use parquet::arrow::ArrowWriter;

use crate::adapters::contract::models::WriteMode;
use crate::adapters::files::_helpers::layout::swap::{
    exchange, exchange_support, local_folder, staged_stem,
};
use crate::adapters::files::types::{CommitFiles, Exchange};
use crate::column_types::models::ColumnType;
use crate::file_store::models::{OpenFolder, StoredFile};
use crate::project::models::{OutputEncoding, WriteStrategy};
use crate::schema_plan::models::DesiredColumn;
use crate::store_paths::main::escape_partition_value::escape_partition_value;
use crate::store_paths::main::unescape_partition_value::unescape_partition_value;
use crate::store_paths::models::{Location, StoreKind};

/// A target that writes tables as Parquet files into an s3 or filesystem connection.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct FilesDestination {
    /// The connection's name, for messages and paths.
    pub connection: String,
    pub store: StoreKind,
    /// The most footers `plan` reads for one table.
    pub max_footer_reads: usize,
    /// The format and options of the files written.
    pub encoding: OutputEncoding,
}

/// How a partition folder's value is taken from the cursor.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum PartitionBy {
    /// `{cursor}`: the cursor value itself (int, date or text cursors).
    Value,
    /// `{cursor:date}`: the cursor's date (date or timestamp cursors).
    Date,
}

/// The partition folder of a table: `prefix`, the cursor's value, then `suffix`.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct PartitionFolder {
    pub prefix: String,
    pub suffix: String,
    pub by: PartitionBy,
}

impl PartitionFolder {
    /// The folder name for partition `value`, escaped as Hive escapes partition values.
    pub fn name(&self, value: &str) -> String {
        format!(
            "{}{}{}",
            self.prefix,
            escape_partition_value(value),
            self.suffix
        )
    }

    /// The partition value of folder `name` (escapes undone), if the name has this folder's shape.
    pub fn value_of(&self, name: &str) -> Option<String> {
        name.strip_prefix(self.prefix.as_str())?
            .strip_suffix(self.suffix.as_str())
            .filter(|value| !value.is_empty())
            .map(unescape_partition_value)
    }
}

impl std::fmt::Display for PartitionFolder {
    fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
        let placeholder = match self.by {
            PartitionBy::Value => "{cursor}",
            PartitionBy::Date => "{cursor:date}",
        };
        write!(formatter, "{}{placeholder}{}/", self.prefix, self.suffix)
    }
}

/// Where a table's files go: its folder, and the partition folder below it if the path has one.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct TablePath {
    pub folder: Location,
    pub partition: Option<PartitionFolder>,
}

impl TablePath {
    /// The hidden folder beside the table folder that a `replace` or `--reload` of a local directory writes the table into, to swap it in at commit: `.<table folder>.sqlload-<load id>/`; `None` for other write modes, other stores and a table at the directory's root.
    pub fn staged_folder(&self, mode: &WriteMode, load_id: &str) -> Option<Location> {
        let rewrites = matches!(mode, WriteMode::Replace | WriteMode::Recreate);
        let (_, parent, name) = local_folder(&self.folder).filter(|_| rewrites)?;
        Some(Location {
            store: self.folder.store.clone(),
            prefix: format!("{parent}{}{load_id}/", staged_stem(name)),
        })
    }

    /// The folder this load stages its table in (see `staged_folder`) when the filesystem swaps two folders in one rename, checked once, before the load writes anything; `None` where it can't (NFS, Windows), so the load writes into the table folder, then deletes the older files, as in S3. The folder is created here, with an open handle on it the swap checks it against.
    pub fn staging(
        &self,
        mode: &WriteMode,
        load_id: &str,
    ) -> std::io::Result<Option<(Location, std::fs::File)>> {
        self.staging_with(mode, load_id, exchange)
    }

    /// `staging`, checking the filesystem with `exchange`.
    pub fn staging_with(
        &self,
        mode: &WriteMode,
        load_id: &str,
        exchange: Exchange,
    ) -> std::io::Result<Option<(Location, std::fs::File)>> {
        let (Some(staged), Some((live, _, name))) = (
            self.staged_folder(mode, load_id),
            local_folder(&self.folder),
        ) else {
            return Ok(None);
        };
        let (Some((path, _, _)), true) = (
            local_folder(&staged),
            exchange_support(&live, &staged_stem(name), exchange)?,
        ) else {
            return Ok(None);
        };
        std::fs::create_dir(&path)?;
        Ok(Some((staged, std::fs::File::open(&path)?)))
    }
}

/// What a table's path template is filled with.
#[derive(Debug, Clone, Copy)]
pub struct TablePathRequest<'a> {
    pub template: &'a str,
    pub connection: &'a str,
    pub store: &'a StoreKind,
    pub source: &'a str,
    pub schema: &'a str,
    pub table: &'a str,
}

/// A partition value in comparable form.
#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)]
pub enum PartitionKey {
    Int(i64),
    Date(NaiveDate),
    Text(String),
}

/// One partition's data files; `value` is `None` for a table without partitions.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Partition {
    pub value: Option<String>,
    pub files: Vec<StoredFile>,
}

/// A table's existing data files by partition, oldest partition first.
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub struct TableFiles {
    pub partitions: Vec<Partition>,
    /// Files that are not where the path puts them, so the table's layout cannot be trusted.
    pub problems: Vec<String>,
}

impl TableFiles {
    /// Every data file's key below the table folder.
    pub fn keys(&self) -> Vec<String> {
        self.partitions
            .iter()
            .flat_map(|partition| partition.files.iter())
            .map(|file| file.relative_key.clone())
            .collect()
    }
}

/// What decides the cursor window of a `delete_insert` into partition folders.
#[derive(Debug, Clone, Copy)]
pub struct PartitionWindowRequest<'a> {
    pub by: PartitionBy,
    pub cursor_column: &'a str,
    pub cursor_type: &'a ColumnType,
    /// The newest existing partition's value; `None` when the table has none (or `--reload`).
    pub newest: Option<&'a str>,
    pub lookback: Option<Duration>,
    /// `--start-cursor-*`, already normalised for the cursor type.
    pub start: Option<&'a str>,
    /// The flag `start` came from, as `plan` names it: `--start-cursor`, `--start-cursor-ts` or `--start-cursor-int`.
    pub start_flag: &'a str,
    /// `--end-cursor-*`, already normalised for the cursor type.
    pub end: Option<&'a str>,
    pub cursor_start: Option<&'a str>,
    pub reload: bool,
}

/// The cursor range read, widened to whole partitions, and a note for `plan`.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct PartitionWindow {
    pub from: Option<String>,
    pub to: Option<String>,
    pub note: String,
}

/// What a load writes: the table, its columns, and the load's identity.
#[derive(Debug, Clone)]
pub struct WriterRequest<'a> {
    pub folder: OpenFolder,
    pub path: &'a TablePath,
    /// The cursor column partitions are taken from, when the path has a partition folder.
    pub cursor_column: Option<&'a str>,
    pub columns: &'a [DesiredColumn],
    /// How the table names the columns a load adds.
    pub compat: Compat,
    pub load_id: &'a str,
    /// Names the files and the pending marker (see [`FilesWriter::file_id`]).
    pub file_id: &'a str,
    /// The load time as an ISO 8601 UTC timestamp, written to every row.
    pub loaded_at: &'a str,
    pub encoding: &'a OutputEncoding,
}

/// A file being filled in memory.
#[derive(Debug)]
pub struct OpenFile {
    pub body: FileBody,
    pub rows: usize,
}

/// A file's contents so far, in its format.
#[derive(Debug)]
pub enum FileBody {
    Parquet(Box<ArrowWriter<Vec<u8>>>),
    Csv(CsvOut),
}

/// CSV text written so far, compressed as it is written.
#[derive(Debug)]
pub enum CsvOut {
    Plain(Vec<u8>),
    Gzip(Box<flate2::write::GzEncoder<Vec<u8>>>),
    /// zstd frames compressed so far, and text waiting for the next frame.
    Zstd {
        done: Vec<u8>,
        pending: Vec<u8>,
    },
}

/// A load's writer: one open file per partition, flushed to the store as files fill up.
#[derive(Debug)]
pub struct FilesWriter {
    pub folder: OpenFolder,
    pub encoding: OutputEncoding,
    /// With `single_file`, each file's name before its suffix: the table folder's own name.
    pub single_name: String,
    pub partition: Option<PartitionFolder>,
    pub cursor_column: Option<String>,
    pub schema: SchemaRef,
    /// What each of `schema`'s fields holds when the load adds it.
    pub metadata: Vec<Option<MetadataRole>>,
    /// The load id the rows carry in their load id column.
    pub load_id: String,
    /// Names this writer's files and its pending marker: one per commit of a load, so repairing an unfinished commit never touches an earlier one's files.
    pub file_id: String,
    pub loaded_at_micros: i64,
    /// Open files by partition value (empty for a table without partitions).
    pub open: BTreeMap<String, OpenFile>,
    /// Keys of the files written so far, below the table folder.
    pub written: Vec<String>,
    /// Partition values this load wrote rows to.
    pub partitions: BTreeSet<String>,
    /// By folder (as [`CommitFiles`]), the source files (name and version) whose rows this commit wrote there, from the rows' source file columns.
    pub sources: BTreeMap<String, BTreeSet<(String, String)>>,
    pub rows: u64,
}

/// What decides whether a table's strategy and cursor suit its path.
#[derive(Debug, Clone, Copy)]
pub struct OutputCheck<'a> {
    pub strategy: WriteStrategy,
    pub cursor_column: Option<&'a str>,
    /// The cursor's type, when the cursor column is loaded.
    pub cursor_type: Option<&'a ColumnType>,
    pub partition: Option<&'a PartitionFolder>,
}

/// How far an unfinished load got, from its pending marker.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum PendingPhase {
    /// Writing its new files: they are incomplete, so they are deleted.
    Writing,
    /// Deleting the files it replaces: its new files are complete, so the deletes are finished.
    Replacing,
    /// Found writing and repaired: any file it still stores is deleted, until the marker expires.
    Aborted,
}

/// A load that left its pending marker behind, and the files repairing it deletes.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct PendingLoad {
    pub load_id: String,
    pub phase: PendingPhase,
    /// Keys below the table folder that repair deletes: the load's own files while writing, the replaced ones while replacing.
    pub deletes: Vec<String>,
    /// The run record a logged table's load writes once its files are complete; repair writes it if the load stopped first.
    pub run: Option<PendingRun>,
    /// An `aborted` marker repair deletes: old enough, or a withdrawn decision.
    pub expired: bool,
}

/// What a pending marker records.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Marker {
    pub phase: PendingPhase,
    pub keys: Vec<String>,
    pub run: Option<PendingRun>,
}

/// A run record waiting in a `replacing` marker: the record object's folder (in the same store), its name and its JSON.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct PendingRun {
    pub prefix: String,
    pub name: String,
    pub json: String,
}

/// A files table as listed: its trusted files by partition, loads to repair, and whether any data file exists.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct TableState {
    /// Data files, without those an unfinished load leaves for repair to delete.
    pub listed: TableFiles,
    pub pending: Vec<PendingLoad>,
    pub exists: bool,
    /// The commits (load id and commit number) whose files the table holds.
    pub commits: BTreeSet<String>,
    /// Source files loads from a files source loaded, whose rows the table still holds.
    pub loaded: Vec<crate::files_source::models::LoadedFile>,
    /// Why the table's records cannot be read: one is in an older SQLLoad's format.
    pub old_records: Option<String>,
    /// The records still holding a loaded file, and what they hold.
    pub records: LiveRecords,
    /// Keys of load records none of whose files the table holds any more.
    pub stale_loaded: Vec<String>,
}

/// A table's records that still hold loaded files: their keys below the table folder, and the groups the table still holds.
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub struct LiveRecords {
    pub keys: Vec<String>,
    pub groups: CommitFiles,
}

/// A record of loaded source files, as stored: `{"commits": {commit: {folder: [[file, version], ...]}}}`.
#[derive(Debug, Clone, Default, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
pub struct LoadedRecord {
    pub commits: CommitFiles,
}

/// The files a write replaces, worked out before anything is written.
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub struct ReplacedFiles {
    /// Replaced whatever the load writes (everything, or partitions in the cursor window).
    pub always: Vec<String>,
    /// Each other partition's files, replaced if the load writes rows to it.
    pub by_partition: BTreeMap<String, Vec<String>>,
}

impl ReplacedFiles {
    /// Every replaced key, given the partitions the load wrote.
    pub fn keys(&self, written: &BTreeSet<String>) -> Vec<String> {
        let mut keys = self.always.clone();
        for value in written {
            keys.extend(self.by_partition.get(value).into_iter().flatten().cloned());
        }
        keys
    }
}

/// A table's folder, and what is needed to explain a clash with another table's.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct FolderClaim {
    /// Tells tables apart: the source and the source table.
    pub id: String,
    /// The table as `plan` names it.
    pub name: String,
    pub folder: Location,
    pub template: String,
    /// The source table's schema.
    pub schema: String,
    /// The folder a files source reads the table from; `None` for other sources.
    pub source: Option<Location>,
}

/// What decides the files a write replaces.
#[derive(Debug, Clone, Copy)]
pub struct ReplaceRequest<'a> {
    pub mode: &'a WriteMode,
    pub listed: &'a TableFiles,
    pub partition: Option<&'a PartitionFolder>,
    pub cursor_type: Option<&'a ColumnType>,
}

/// Parquet files' types: Arrow's, with no SQL of their own.
#[derive(Debug, Clone, Copy, Default)]
pub struct ParquetFiles;