1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
use crate;
/// The column holding each row's source file, used to skip files already loaded by `append`.
pub const SOURCE_FILE_COLUMN: &str = "_sqlload_source_file";
/// The column holding the version of each row's source file (its ETag, or size and modification time), to notice files rewritten in place.
pub const SOURCE_VERSION_COLUMN: &str = "_sqlload_source_version";
/// The column holding a row's number in its source file, from 1.
pub const SOURCE_ROW_COLUMN: &str = "_sqlload_source_row";
/// Separates a file name from its version in the loaded-files query; versions never contain it.
pub const VERSION_SEPARATOR: char = '|';
/// The column a `shape = "record"` source loads each record into, unless `record_column` names another.
pub const RECORD_COLUMN: &str = "record";
/// The file name suffix of Parquet data files.
pub const PARQUET_SUFFIX: &str = ".parquet";
/// The file name suffixes of JSON data files, for both `ndjson` and `json`.
pub const JSON_SUFFIXES: & = &;
/// The file name suffix of CSV data files.
pub const CSV_SUFFIX: &str = ".csv";
/// The suffix of Excel workbooks.
pub const XLSX_SUFFIX: &str = ".xlsx";
/// The suffix of XML documents.
pub const XML_SUFFIX: &str = ".xml";
/// The suffix of ZIP archives.
pub const ZIP_SUFFIX: &str = ".zip";
/// The largest xlsx workbook, XML document or ZIP archive member read: each is read whole into memory.
pub const MAX_WHOLE_FILE_BYTES: u64 = 1024 * 1024 * 1024;
/// The bytes of a ZIP archive each range request reads: its index, at its end, then one member.
pub const ZIP_READ_BLOCK_BYTES: u64 = 4 * 1024 * 1024;
/// The most non-empty cells read from one worksheet, each held in memory.
pub const MAX_SHEET_CELLS: usize = 20_000_000;
/// The key an XML element's own text loads under when it also has attributes or child elements.
pub const XML_TEXT_KEY: &str = "#text";
/// The XML Schema instance namespace, whose attributes such as `xsi:type` describe an element rather than hold data.
pub const XSI_NAMESPACE: &str = "http://www.w3.org/2001/XMLSchema-instance";
/// The `xsi:` attribute that marks an element NULL.
pub const XSI_NIL: &str = "nil";
/// The values of `xsi:nil` that make an element NULL.
pub const XML_NIL_VALUES: = ;
/// The file name suffix of gzip-compressed files.
pub const GZIP_SUFFIX: &str = ".gz";
/// The file name suffix of zstd-compressed files.
pub const ZSTD_SUFFIX: &str = ".zst";
/// Records read from each sampled JSON or CSV file to work out its column types.
pub const SAMPLE_ROWS: usize = 10_000;
/// The longest JSON object or CSV record read, so a broken file (an unclosed quote or brace) fails instead of filling memory.
pub const MAX_RECORD_BYTES: usize = 64 * 1024 * 1024;
/// Bytes read from a JSON or CSV file (after decompression) at a time.
pub const READ_CHUNK_BYTES: usize = 256 * 1024;
/// Files sampled to work out the schema (their footers, or their first records): the newest by key, so recent schema changes are seen.
pub const SCHEMA_SAMPLE_FILES: usize = 20;
/// Bytes read from the end of a Parquet file for its footer: one request holds a typical footer, and stays within a new connection's first TCP window (a larger read costs round trips on a distant store); a larger footer takes a second request.
pub const FOOTER_PREFETCH_BYTES: usize = 8 * 1024;
/// Parquet footers (or CSV header lines) read at once, at plan and for each batch.
pub const CONCURRENT_FOOTER_READS: usize = 16;
/// JSON and CSV files sampled at once at plan: each sample holds up to `SAMPLE_ROWS` records, so fewer than footers.
pub const CONCURRENT_TEXT_SAMPLES: usize = 4;
/// Columns the loader writes itself (load metadata and source tracking): a files source drops them from its files, matched ignoring case, and this load fills its own.
pub const LOADER_COLUMNS: & = &;
/// The largest decimal precision a direct `COPY` loads exactly.
pub const MAX_DIRECT_DECIMAL_PRECISION: u8 = 38;
/// The decimal digits of a second a nanosecond time or timestamp holds; a live column with fewer gets values cut to microseconds.
pub const NANOSECOND_SCALE: u8 = 9;
/// A `path_columns` segment matching any number of folders.
pub const ANY_FOLDERS: &str = "**";
/// The type of a `path_columns` capture that names none.
pub const DEFAULT_CAPTURE_TYPE: &str = "text";
/// How a captured or constant date is stored.
pub const NORMAL_DATE: &str = "%Y-%m-%d";
/// How a captured or constant time is stored.
pub const NORMAL_TIME: &str = "%H:%M:%S%.6f";
/// How a captured or constant timestamp without a zone is stored.
pub const NORMAL_TIMESTAMP: &str = "%Y-%m-%d %H:%M:%S%.6f";
/// How a captured or constant timestamp with a zone is stored: in UTC, the offset right after the time, which every destination reads.
pub const NORMAL_TIMESTAMP_TZ: &str = "%Y-%m-%dT%H:%M:%S%.6f%:z";
/// Rows of a nested Parquet column written as JSON text at a time.
pub const NESTED_JSON_SLICE_ROWS: usize = 1024;
/// The most text one Arrow text array holds: its offsets are 32-bit.
pub const MAX_TEXT_ARRAY_BYTES: usize = i32MAX as usize;
/// The encoding CSV files are read in unless `csv.encoding` says otherwise.
pub const DEFAULT_CSV_ENCODING: &str = "utf-8";
/// `csv.encoding` labels read as ISO-8859-1 itself, byte for byte; the Encoding Standard reads these as windows-1252, which differs in bytes 0x80 to 0x9F.
pub const LATIN1_LABELS: = ;
/// The UTF-16 byte order marks, little-endian then big-endian.
pub const UTF16_BOMS: = ;
/// `csv.encoding` labels read as US-ASCII itself, failing on any byte above 127; the Encoding Standard reads these as windows-1252.
pub const ASCII_LABELS: = ;
/// The windows-1252 bytes no character is assigned to, which Python's cp1252 refuses to decode.
pub const CP1252_UNDEFINED: = ;
/// The fix an error about bytes that do not decode ends with.
pub const ENCODING_HINT: &str =
"set csv.encoding to the files' encoding, such as \"cp1252\", \"latin-1\" or \"utf-16le\"";
/// The quote byte the CSV decoder gets when `quote = false`: one valid UTF-8 never holds, so no field opens a quote.
pub const QUOTE_OFF_BYTE: u8 = 0xFF;
/// The fix for a backslash-escaped quote, which a CSV error about quoting suggests unless `escape` is set.
pub const ESCAPE_HINT: &str =
"if this file escapes quotes with a backslash, set csv = { escape = \"\\\\\" }";
/// The fix for a file that never quotes values, which a CSV error about quoting suggests.
pub const QUOTE_OFF_HINT: &str = "if it never quotes values, set csv = { quote = false }";
/// The last C1 control character: bytes 0x80 to this one are controls in `latin-1` and `iso-8859-15`, which no text holds.
pub const LAST_C1_CONTROL: u8 = 0x9F;
/// Of a file's bytes above 0x7F read in a single-byte encoding, the share in valid UTF-8 sequences that says the file is UTF-8.
pub const UTF8_SHARE: f64 = 0.9;
/// Floats below this hold every whole number exactly: 2^53.
pub const MAX_EXACT_FLOAT_INT: f64 = 9_007_199_254_740_992.0;
/// One day in Excel's date numbers; a date-formatted value below it is a time of day.
pub const EXCEL_DAY: f64 = 1.0;
/// How a column flattening made is held, which a destination reading the files itself would not find.
pub const FLATTENED_KEY: &str = "a key of an object flatten flattens";