sqlload 0.63.0

SQL data loading tooling.
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
use std::collections::BTreeMap;
use std::sync::Mutex;
use std::time::{Duration, Instant};

use serde::Deserialize;
use serde_json::{Map, Value};
use tokio::sync::mpsc::Sender;

use crate::row_batches::models::FieldSpec;

/// A job a table function built with `sl.http(...)`: run entirely in Rust, its pages parsed like `sl.json(...)` bytes.
#[derive(Debug, Clone, Deserialize)]
#[serde(tag = "kind", rename_all = "snake_case")]
pub enum HttpJob {
    /// Every page of one endpoint.
    Paginate(Box<Request>),
    /// Every page of one endpoint per key of `over`, up to `concurrency` endpoints at once.
    FanOut(Box<FanOut>),
}

impl HttpJob {
    /// The `transform=` the rows of the job's pages go through.
    pub fn transform(&self) -> Option<&PageTransform> {
        match self {
            Self::Paginate(request) => request.transform.as_ref(),
            Self::FanOut(fan_out) => fan_out.request.transform.as_ref(),
        }
    }
}

/// One endpoint and how to page through it.
#[derive(Debug, Clone, Deserialize)]
pub struct Request {
    pub client: ClientSpec,
    /// A path under the connection's `base_url`, with `{name}` placeholders in a fan-out.
    pub path: String,
    /// Query parameters; a null value is left out.
    #[serde(default)]
    pub params: Map<String, Value>,
    #[serde(default)]
    pub method: Method,
    /// The JSON body every `POST` starts from; each page's copy gets the paginator's values when they go in the body.
    #[serde(default)]
    pub json_body: Option<Value>,
    pub paginator: Paginator,
    /// Where the rows are in each response, such as `$.items`; `$` (or none) is the whole response.
    #[serde(default)]
    pub select: Option<String>,
    /// What an HTTP status other than success means, such as `{"404": "skip"}`.
    #[serde(default)]
    pub on_status: BTreeMap<String, StatusAction>,
    /// The only values of each row that load, renamed: `None` loads every key under its own name.
    #[serde(default)]
    pub fields: Option<Vec<FieldSpec>>,
    /// A column holding each row's whole object as JSON.
    #[serde(default)]
    pub keep_record: Option<String>,
    /// For a job used as a fan-out's `over`: the name its picked values fill as keys, when not the selected field's own (such as `order_id` for `$.items[*].id`).
    #[serde(default)]
    pub key: Option<String>,
    /// Each row's list of cells, pivoted into one column per named column before the rows load.
    #[serde(default)]
    pub cells: Option<Cells>,
    /// What a response without the `select` path means.
    #[serde(default)]
    pub select_missing: SelectMissing,
    /// Where a successful response reports an error, such as `$.errors`: a value there that is not null, false or empty fails the table before the page's rows load.
    #[serde(default)]
    pub error_path: Option<String>,
    /// In a fan-out, where each record's key is, checked against the keys its request sent.
    #[serde(default)]
    pub key_path: Option<KeyPath>,
    /// The function each page's rows go through before they load, which the Python crossing calls.
    #[serde(default)]
    pub transform: Option<PageTransform>,
}

/// `transform=` on a job: the key the Python side calls the function by, and the name errors show.
#[derive(Debug, Clone, Deserialize)]
pub struct PageTransform {
    pub function: String,
    pub name: String,
}

/// `fan_out(key_path=...)`: a path to each record's key, from the record or an object around it (`sl.parent`).
#[derive(Debug, Clone, Deserialize)]
pub struct KeyPath {
    pub path: String,
    #[serde(default)]
    pub from: crate::row_batches::models::FieldFrom,
}

/// What a fan-out request's rows take from the request rather than the response: its key's fields, for `sl.key`, and, for `key_path`, the key values it sent (as text).
#[derive(Debug, Clone, Default)]
pub struct RowContext {
    pub key: Map<String, Value>,
    pub sent: Option<Vec<String>>,
}

/// What a response without a request's `select` path means: an error (an API answering an error with a 200 status looks like this), or, chosen for APIs that leave empty lists out, no rows.
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum SelectMissing {
    #[default]
    Fail,
    Empty,
}

/// How each row's list of cells becomes columns: where the cells are, each one's value, and where its column's name comes from.
#[derive(Debug, Clone, Deserialize)]
pub struct Cells {
    /// The row's field holding its cells, such as `$.cells`.
    pub pointer: String,
    /// Where each cell's value is, such as `$.value`; the first present and not null is used.
    pub value: Vec<String>,
    /// Where each cell's column name comes from.
    pub name: CellName,
}

/// Where the cells' column names come from (`sl.cells(name=...)`).
#[derive(Debug, Clone, Deserialize)]
#[serde(tag = "kind", rename_all = "snake_case")]
pub enum CellName {
    /// The columns a response lists at `from`, such as `$.columns`; a page without the list keeps the names before it.
    Listed {
        from: String,
        /// The field of each cell holding its column's id; `None` matches cells to columns by position.
        #[serde(default)]
        key: Option<String>,
        /// The field of each listed column holding its id, matched against each cell's `key`.
        id_key: String,
        /// The field of each listed column holding its name; `None` when each listed column is its name.
        #[serde(default)]
        name_key: Option<String>,
    },
    /// The first row's cells name the columns by position, and that row does not load.
    HeaderRow,
    /// Each cell names its own column at `path`, such as `$.FIELDNAME`, so each row has the columns its cells name.
    Field { path: String },
}

/// The columns a job's cells pivot into, in order, with each one's position by its id as text.
#[derive(Debug, Clone, Default)]
pub struct CellNames {
    pub names: Vec<String>,
    pub positions: std::collections::HashMap<String, usize>,
}

/// The HTTP method of a job's requests.
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Deserialize)]
#[serde(rename_all = "UPPERCASE")]
pub enum Method {
    #[default]
    Get,
    Post,
}

impl Method {
    pub fn as_str(self) -> &'static str {
        match self {
            Self::Get => "GET",
            Self::Post => "POST",
        }
    }
}

/// Where a paginator's values go: the query string, or a path in the JSON body.
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum Location {
    #[default]
    Query,
    Body,
}

/// A page requested ahead (`read_ahead`), with what it fetched.
#[derive(Debug)]
pub struct AheadPage {
    pub call: PageCall,
    pub fetched: Result<Option<Fetched>, crate::python_sources::errors::ReadError>,
}

/// One request of a page: its method, URL and JSON body.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct PageCall {
    pub method: Method,
    pub url: reqwest::Url,
    pub body: Option<Value>,
}

/// A request per key of `over`.
#[derive(Debug, Clone, Deserialize)]
pub struct FanOut {
    #[serde(flatten)]
    pub request: Request,
    pub over: Over,
    pub concurrency: usize,
    /// Keys sent per request, each placeholder of `json_body` filled with the list of the batch's values; `None` sends one key per request.
    #[serde(default)]
    pub batch: Option<usize>,
}

/// The keys a fan-out requests.
#[derive(Debug, Clone, Deserialize)]
#[serde(tag = "kind", rename_all = "snake_case")]
pub enum Over {
    /// Scalars (filling the path's one placeholder) or objects (filling placeholders by name).
    Values { values: Vec<Value> },
    /// The values its `select` (`$.items[*].id`) picks from every page, named after the last field.
    Paginate(Box<Request>),
    /// The values another fan-out's `select` picks from each of its pages, each key keeping the fields of the key above it, so a chain of fan-outs goes as many levels deep as it nests.
    FanOut(Box<FanOut>),
}

/// The connection a job uses and how it calls it.
#[derive(Debug, Clone, Deserialize)]
pub struct ClientSpec {
    pub connection: String,
    #[serde(default)]
    pub auth: Auth,
    /// Requests allowed per second, minute or hour, such as `10/s`, shared by every job of the connection with the same limit.
    #[serde(default)]
    pub rate_limit: Option<String>,
    /// Retries of a request that failed with a transport error, 429 or a 5xx.
    pub retries: u32,
    #[serde(default)]
    pub headers: BTreeMap<String, String>,
}

/// Credentials from the connection's settings, sent only to the connection's own origin.
#[derive(Debug, Clone, Default, Deserialize)]
#[serde(tag = "kind", rename_all = "snake_case")]
pub enum Auth {
    #[default]
    None,
    /// `Authorization: Bearer <token>`.
    Bearer,
    /// The token in a header or a query parameter.
    ApiKey {
        #[serde(default)]
        header: Option<String>,
        #[serde(default)]
        query: Option<String>,
    },
    /// `username` and `password`.
    Basic,
    /// An access token from `token_url` for the connection's `client_id` and `client_secret`.
    Oauth2ClientCredentials(OAuth2Spec),
    /// An access token from `token_url` for the connection's `refresh_token`.
    Oauth2RefreshToken(OAuth2Spec),
    /// An access token from `token_url` for the connection's `username` and `password` (the resource-owner password grant).
    Oauth2Password(OAuth2Spec),
}

/// Where an OAuth2 access token comes from and how the client authenticates there.
#[derive(Debug, Clone, Deserialize)]
pub struct OAuth2Spec {
    pub token_url: String,
    #[serde(default)]
    pub scope: Option<String>,
    #[serde(default)]
    pub client_auth: ClientAuth,
}

/// How the client's id and secret reach the token endpoint.
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum ClientAuth {
    /// HTTP basic auth, which every OAuth2 server must accept.
    #[default]
    Basic,
    /// Form fields `client_id` and `client_secret`.
    Post,
}

/// The form an OAuth2 token request posts, and the access token it last returned.
#[derive(Debug)]
pub struct TokenSource {
    pub token_url: reqwest::Url,
    /// `grant_type` and its fields, plus the client's fields under `client_auth = "post"`.
    pub form: Vec<(String, String)>,
    /// For a grant that starts from something other than a refresh token (the password grant): the `refresh_token` grant's form, used instead of `form` once a token response hands one out.
    pub refresh_form: Option<Vec<(String, String)>>,
    /// `client_id` and `client_secret` for basic client auth, form-encoded as RFC 6749 asks.
    pub basic: Option<(String, String)>,
    /// Shared with a token request under way, which holds it until it has kept what the server sent.
    pub state: std::sync::Arc<tokio::sync::Mutex<TokenState>>,
    /// Every secret of the token flow, including each access token issued, for redaction.
    pub secrets: Mutex<Vec<String>>,
}

/// The current access token, when it was issued and should be replaced, and a rotated refresh token.
#[derive(Debug, Default)]
pub struct TokenState {
    pub access: Option<String>,
    pub issued_at: Option<Instant>,
    pub renew_at: Option<Instant>,
    pub refresh_token: Option<String>,
}

/// The grant an OAuth2 auth uses, with the connection's settings it needs.
#[derive(Debug, Clone)]
pub enum Grant {
    ClientCredentials {
        client_id: String,
        client_secret: String,
    },
    RefreshToken {
        refresh_token: String,
        /// `client_id` and `client_secret`, when the connection has them.
        client: (Option<String>, Option<String>),
    },
    Password {
        username: String,
        password: String,
        /// `client_id` and `client_secret`, when the connection has them.
        client: (Option<String>, Option<String>),
    },
}

/// What a token response issued.
#[derive(Debug, Clone)]
pub struct Issued {
    pub access: String,
    /// When after issue to renew it; `None` keeps it until a 401.
    pub renew_after: Option<Duration>,
    pub refresh_token: Option<String>,
}

/// How the next page is found; always named, never guessed.
#[derive(Debug, Clone, Deserialize)]
#[serde(tag = "kind", rename_all = "snake_case")]
pub enum Paginator {
    /// One request.
    Single,
    /// The next page's URL at `pointer` in the body; ends when it is missing, null or empty.
    NextLink { pointer: String },
    /// The `Link` header's `rel="next"`.
    LinkHeader,
    /// `param` counts rows from 0 by `limit`; ends at a page with fewer rows. In the body, `param` and `limit_param` are paths such as `$.page.offset`.
    Offset {
        param: String,
        limit_param: String,
        limit: u64,
        #[serde(default)]
        location: Location,
        /// Pages requested at once after a full first page, their rows still loaded in page order; `None` reads one at a time.
        #[serde(default)]
        read_ahead: Option<u64>,
    },
    /// `param` counts pages from `start`; ends at an empty page.
    PageNumber {
        param: String,
        start: u64,
        #[serde(default)]
        location: Location,
        /// As `Offset`'s.
        #[serde(default)]
        read_ahead: Option<u64>,
    },
    /// The token at `pointer` in the response goes back as `param`; ends when it is missing, null or empty.
    Cursor {
        pointer: String,
        param: String,
        #[serde(default)]
        location: Location,
    },
}

/// What a listed status does.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum StatusAction {
    /// No rows for that request.
    Skip,
}

/// One response's body, and where its rows are.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct HttpPage {
    pub bytes: Vec<u8>,
    /// The dotted path to the rows, as `sl.json(..., path=...)` takes it.
    pub path: Option<String>,
    /// The request, secrets hidden, for errors about the page.
    pub source: String,
    /// The columns each row loads as, when the job names its fields.
    pub projection: Option<crate::row_batches::models::Projection>,
    /// The page's number within its request, from 1.
    pub number: u64,
    /// The fields of the fan-out key the page was requested for; empty in a paginate or a batched fan-out.
    pub key: Map<String, Value>,
}

/// A parsed `select`: the fields down to the rows and, for keys, the field of each row.
#[derive(Debug, Clone, PartialEq, Eq, Default)]
pub struct Selector {
    pub rows: Vec<String>,
    pub field: Option<String>,
    /// Past the first list (`[*]`), the path on from each of its items to the rows, through more lists (`[*]`), such as `changeHistories`, `[*]` for `$.issueChangeLogs[*].changeHistories[*]`; empty when the rows are the first list's items.
    pub within: Vec<String>,
}

/// A connection resolved for requests: its base URL, headers and credentials.
#[derive(Debug, Clone)]
pub struct Endpoint {
    pub base_url: reqwest::Url,
    pub headers: BTreeMap<String, String>,
    pub credential: Credential,
    /// Values never shown in errors.
    pub secrets: Vec<String>,
}

/// Credentials ready to attach to a request.
#[derive(Debug, Clone)]
pub enum Credential {
    None,
    Header {
        name: String,
        value: String,
    },
    Query {
        name: String,
        value: String,
    },
    /// `Authorization: Bearer <access token>`, the token fetched and renewed for the run.
    OAuth2(std::sync::Arc<TokenSource>),
}

/// Requests spaced `interval` apart, shared across jobs.
#[derive(Debug)]
pub struct Limiter {
    /// The connection whose `rate_limit` this is, named in its load events.
    pub connection: String,
    /// The limit as configured, such as `5/s`.
    pub rate: String,
    pub interval: Duration,
    /// The next free turn, on tokio's clock, which the waits sleep on.
    pub next: Mutex<tokio::time::Instant>,
}

/// What one request needs: the pooled client, the endpoint, the shared rate limit and the retry budget.
#[derive(Debug, Clone)]
pub struct Calls {
    pub http: reqwest::Client,
    pub endpoint: Endpoint,
    pub limiter: Option<std::sync::Arc<Limiter>>,
    pub retries: u32,
    pub on_status: BTreeMap<u16, StatusAction>,
    /// Whether an offset page of this request held its whole limit in this job, which shows the API does not cap its pages lower, so a short first page needs no check; shared by its clones, the keys of a fan-out.
    pub full_page_seen: std::sync::Arc<std::sync::atomic::AtomicBool>,
}

/// What one try at a request came to.
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum Attempt {
    /// The body, or `None` when `on_status` skips the status.
    Done(Option<Fetched>),
    /// A 401 to an OAuth2 access token: renew the token and try once more.
    Refused,
    /// A failure worth retrying, after `after` when the server said.
    Retry {
        reason: String,
        after: Option<Duration>,
    },
}

/// A body fetched, with the `Link` header's next URL.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Fetched {
    pub bytes: Vec<u8>,
    pub next_link: Option<String>,
    pub url: reqwest::Url,
}

/// Where a fan-out's keys come from: listed values, another endpoint's pages, or a fan-out one level up, run with the job's connections and client.
#[derive(Debug)]
pub enum KeyFeed {
    Values(Vec<Value>),
    Pages(Box<(Calls, Request)>),
    FanOut(Box<(FanOut, Map<String, Value>, reqwest::Client)>),
}

/// A fan-out key: the placeholder values it fills, its ancestors' included, and the trail of requests that produced it (such as `/orders order_id=7 → `), which prefixes a multi-level job's errors.
#[derive(Debug, Clone, Default)]
pub struct FanKey {
    pub fields: Map<String, Value>,
    pub trail: String,
}

/// Where a fan-out level's pages go: out as the job's rows, or into keys for the level below.
#[derive(Debug, Clone)]
pub enum Sink {
    Pages(Sender<HttpPage>),
    Keys(Sender<FanKey>),
}

/// The `sl.http` paths a Python table function writes as literals, which discovery reads from its source without calling it, for `plan` to check as load would.
#[derive(Debug, Clone, PartialEq, Eq, Default, Deserialize)]
#[serde(default, deny_unknown_fields)]
pub struct LiteralPaths {
    /// Each `fields=` entry of a job that loads rows, as its column name and path.
    pub fields: Vec<(String, String)>,
    /// Each `key_path=` of a job that loads rows.
    pub key_paths: Vec<String>,
    /// What `plan` can't read as a literal, such as `fields["sku"]` or `key_path`: load checks it.
    pub computed: Vec<String>,
    /// The table function's source can't be read, so load checks every path.
    pub no_source: bool,
    /// Why each `transform=` written as a name can't be looked up: no `@sl.transform` function, or two, registered under it.
    pub transform_problems: Vec<String>,
}

impl LiteralPaths {
    /// The fact `plan` shows when load checks some of the paths: those that are computed, or all of them when the source can't be read.
    pub fn checked_at_load(&self) -> Option<String> {
        if self.no_source {
            return Some(
                "checked at load, as the table function's source can't be read".to_owned(),
            );
        }
        (!self.computed.is_empty()).then(|| {
            format!(
                "checked at load, as they're computed: {}",
                self.computed.join(", ")
            )
        })
    }
}