1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
/// A file is written out once its rows reach this many.
pub const FILE_MAX_ROWS: usize = 1_000_000;
/// A file is written out once its encoded bytes reach about this many (128 MiB).
pub const FILE_MAX_BYTES: usize = 128 * 1024 * 1024;
/// The largest file is written out while all open files together hold more than this (512 MiB).
pub const OPEN_FILES_MAX_BYTES: usize = 512 * 1024 * 1024;
/// The name every written file starts with, followed by the load id and a part number.
pub const PART_PREFIX: &str = "part-";
/// The file name suffix of the Parquet files written.
pub const PARQUET_SUFFIX: &str = ".parquet";
/// The placeholder for the cursor's value in a path template.
pub const CURSOR_PLACEHOLDER: &str = "{cursor}";
/// The placeholder for the cursor's date in a path template.
pub const CURSOR_DATE_PLACEHOLDER: &str = "{cursor:date}";
/// The placeholder for the table's name in a path template.
pub const TABLE_PLACEHOLDER: &str = "{table}";
/// What every cursor placeholder starts with.
pub const CURSOR_PLACEHOLDER_START: &str = "{cursor";
/// The time of day partition bounds of timestamp cursors are written with.
pub const MIDNIGHT: &str = "00:00:00.000000";
/// The offset zoned timestamp bounds end with.
pub const UTC_SUFFIX: &str = "+00";
/// Seconds in a day, to turn Parquet date statistics (days since 1970) into dates.
pub const SECONDS_PER_DAY: i64 = 86_400;
/// The folder below a table's folder holding, per commit of a files source's load, the source files whose rows that commit's files hold.
pub const LOADED_FOLDER: &str = "_sqlload/loaded/";
/// The suffix of a commit's record in [`LOADED_FOLDER`], which no data file has.
pub const LOADED_SUFFIX: &str = ".json";
/// The folder below a table's folder holding one marker per load that has not finished.
pub const PENDING_FOLDER: &str = "_sqlload/pending/";
/// The folder below a table's folder holding how each commit of a load ended: `replacing` once the load committed its files, `aborted` once another load repaired it. Each is created only if absent, so of a load committing late and a load repairing it, exactly one decides.
pub const DECIDED_FOLDER: &str = "_sqlload/decided/";
/// The most a `single_file` commit holds in memory across its open files (2 GiB); each file is stored in one put at the commit, well under S3's 5 GiB limit for one put.
pub const SINGLE_FILE_MAX_BYTES: usize = 2 * 1024 * 1024 * 1024;
/// Marks the hidden folder a filesystem `replace` stages the table in, beside the table folder: `.<table folder>.sqlload-<load id>`.
pub const STAGED_FOLDER_MARK: &str = ".sqlload-";
/// The length of a load id, a hyphenated UUID, as staging folder names end with it.
pub const LOAD_ID_LENGTH: usize = 36;
/// Added to a staging folder's name for the old table while a filesystem without a one-rename swap moves it aside.
pub const MOVED_ASIDE_SUFFIX: &str = "-old";
/// The first line of a pending marker while the load writes its files.
pub const PENDING_WRITING: &str = "writing";
/// The first line of a pending marker while the load deletes the files it replaces.
pub const PENDING_REPLACING: &str = "replacing";
/// The first line of the marker repair leaves for a load it found writing: the load may only have lost its lease and still be writing, so any file it stores later is deleted too.
pub const PENDING_ABORTED: &str = "aborted";
/// How long repair keeps an `aborted` marker (7 days), well past any lease a load still writing could have held.
pub const ABORTED_KEPT_MS: i64 = 7 * 24 * 60 * 60 * 1000;
/// At most this many partitions have a file open at once; beyond it the largest is written out.
pub const OPEN_FILES_MAX: usize = 256;
/// Memory an open file holds beyond its buffered data (writer state, page buffers), for the memory estimate.
pub const OPEN_FILE_OVERHEAD_BYTES: usize = 1024 * 1024;
/// The start of a pending marker line holding a run record (keys never start with `@`).
pub const PENDING_RUN_PREFIX: &str = "@run\t";
/// Records of loaded source files read at once while planning.
pub const RECORD_READS: usize = 16;
/// The start of a record's name that merges the records of several commits, which no commit's own record starts with (load ids are UUIDs).
pub const COMPACTED_PREFIX: &str = "compacted.";
/// A load merges a table's live records into one once there are more than this many.
pub const RECORDS_COMPACTED_ABOVE: usize = 64;
/// The file name suffix of the CSV files written, before any compression suffix.
pub const CSV_SUFFIX: &str = ".csv";
/// The suffix of gzip-compressed CSV files.
pub const GZIP_SUFFIX: &str = ".gz";
/// The suffix of zstd-compressed CSV files.
pub const ZSTD_SUFFIX: &str = ".zst";
/// The name suffixes of a files table's data files, whatever format wrote them, so a replace removes files of an earlier format too.
pub const DATA_SUFFIXES: & = &;
/// CSV text compressed into one zstd frame once this many bytes wait (4 MiB).
pub const ZSTD_FRAME_BYTES: usize = 4 * 1024 * 1024;
/// The file name of a `single_file` table whose folder has no name of its own.
pub const SINGLE_FILE_STEM: &str = "data";
/// How timestamps with a zone are written to CSV: RFC 3339 in UTC.
pub const CSV_TIMESTAMP_TZ: &str = "%Y-%m-%dT%H:%M:%S%.6fZ";
/// How timestamps without a zone are written to CSV.
pub const CSV_TIMESTAMP: &str = "%Y-%m-%dT%H:%M:%S%.6f";
/// How dates are written to CSV.
pub const CSV_DATE: &str = "%Y-%m-%d";
/// How times of day are written to CSV.
pub const CSV_TIME: &str = "%H:%M:%S%.6f";