1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
/// A file format SQLLoad reads or writes.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum FileFormat {
Parquet,
/// Newline-delimited JSON, optionally gzip-compressed.
Json,
Csv,
}
impl std::fmt::Display for FileFormat {
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
formatter.write_str(match self {
Self::Parquet => "parquet",
Self::Json => "json",
Self::Csv => "csv",
})
}
}
/// Where files live.
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum Store {
/// An object store prefix such as `s3://bucket/path/`, `gs://...` or `az://...`.
ObjectStore(String),
/// A directory on the machine SQLLoad runs on.
Local(String),
/// A warehouse's own internal stage, such as Snowflake `@~` or a named internal stage.
InternalStage(String),
}
impl Store {
/// Whether `self` lies under `other` (same kind, and `other` is a prefix of it).
pub fn is_within(&self, other: &Store) -> bool {
match (self, other) {
(Self::ObjectStore(path), Self::ObjectStore(prefix))
| (Self::Local(path), Self::Local(prefix))
| (Self::InternalStage(path), Self::InternalStage(prefix)) => {
path.starts_with(prefix.as_str())
}
_ => false,
}
}
}
impl std::fmt::Display for Store {
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
Self::ObjectStore(path) | Self::Local(path) => formatter.write_str(path),
Self::InternalStage(name) => write!(formatter, "internal stage {name}"),
}
}
}
/// Files of one format in one store.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct FileSet {
pub store: Store,
pub format: FileFormat,
}
/// What a source can produce.
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub struct ReaderOffer {
/// It can stream Arrow record batches (a database query, Python `yield`, Kafka).
pub arrow: bool,
/// Files that already exist and hold the data.
pub existing_files: Option<FileSet>,
/// Stores a warehouse source can unload Parquet into, in order of preference.
pub unload_to: Vec<Store>,
/// The data's estimated size, when the source knows it.
pub estimated_bytes: Option<u64>,
/// Why the destination cannot load `existing_files`, or the files a warehouse source unloads, exactly by reading them itself.
pub direct_blocked: Option<String>,
/// Why SQLLoad must read `existing_files` itself even where the destination could read files of their kind: `hmac` masking, whose key must never appear in SQL; table checks; the test-only `test_loader_route`; or files on an SFTP server, which no destination reads.
pub needs_loader: Option<String>,
/// `copy_to_internal_stage`: files the destination cannot reach are uploaded unchanged to a store it can, never read by SQLLoad.
pub copy_to_internal_stage: bool,
/// The destination's store paths name `existing_files`, but it cannot read them (the same bucket on another endpoint).
pub out_of_reach: bool,
}
/// What a destination can take.
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub struct WriterAccepts {
/// It can take Arrow record batches directly (Postgres binary `COPY`, ClickHouse insert, a Parquet file writer).
pub arrow: bool,
/// Stores it can read files from itself (`COPY` from a stage, ClickHouse `s3()`, a same-store copy).
pub pulls_from: Vec<Store>,
/// Formats it can read when pulling files.
pub pull_formats: Vec<FileFormat>,
/// Stores where SQLLoad may write files for it to pull, in order of preference (external first).
pub staging: Vec<Store>,
/// How it takes rows when no store is usable, such as Snowpipe Streaming.
pub rows_into: Option<String>,
/// A cloud warehouse's name: a files source reaches it only by its own read, never through SQLLoad, unless `ReaderOffer::needs_loader` says why.
pub reads_files_itself: Option<&'static str>,
/// Where files the destination cannot reach are uploaded unchanged for `copy_to_internal_stage`.
pub copies_files_to: Option<Store>,
}
/// How a warehouse source is read.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
pub enum ReadVia {
/// Unload above the size threshold (or when the size is unknown), query below it.
#[default]
Auto,
Query,
Unload,
}
/// Which kind of stage to stage files in.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
pub enum StagePreference {
/// An external stage (an object store) if one exists, otherwise an internal stage.
#[default]
Auto,
External,
Internal,
}
/// User choices that override the planner's defaults.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
pub struct RouteOverrides {
pub read_via: ReadVia,
pub stage: StagePreference,
}
/// One step of a route.
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum Step {
/// The source streams Arrow batches (a query, Python `yield`, Kafka).
ReadArrow,
/// The warehouse source unloads Parquet into a store.
Unload(Store),
/// Existing files are listed and used as they are.
UseFiles(FileSet),
/// Adapter: files are read into Arrow batches.
FileReader(FileFormat),
/// Adapter: Arrow batches are written as Parquet into a store.
ParquetWriter(Store),
/// The files are copied unchanged into a store the destination reads (`copy_to_internal_stage`); nothing is decoded.
CopyFiles(Store),
/// The destination reads the files itself; no data passes through SQLLoad.
DestinationPulls(FileSet),
/// The destination takes Arrow batches.
DestinationTakesArrow,
/// The rows go in without files, this way.
RowsInto(String),
}
impl std::fmt::Display for Step {
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
Self::ReadArrow => formatter.write_str("read as Arrow"),
Self::Unload(store) => write!(formatter, "unload Parquet to {store}"),
Self::UseFiles(files) => {
write!(formatter, "use {} files in {}", files.format, files.store)
}
Self::FileReader(format) => write!(formatter, "read {format} into Arrow"),
Self::ParquetWriter(store) => write!(formatter, "write Parquet to {store}"),
Self::CopyFiles(store) => write!(formatter, "copy the files unchanged to {store}"),
Self::DestinationPulls(files) => {
write!(
formatter,
"destination loads {} files from {}",
files.format, files.store
)
}
Self::DestinationTakesArrow => formatter.write_str("destination takes Arrow"),
Self::RowsInto(how) => formatter.write_str(how),
}
}
}
/// The steps that move one table's data, and why they were chosen.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Route {
pub steps: Vec<Step>,
pub reason: String,
}