sqlload 0.16.0

SQL data loading tooling. Pre-release, under development.
use std::collections::BTreeMap;
use std::sync::Mutex;
use std::time::{Duration, Instant};

use serde::Deserialize;
use serde_json::{Map, Value};

/// A job a table function built with `sl.http(...)`: run entirely in Rust, its pages parsed like `sl.json(...)` bytes.
#[derive(Debug, Clone, Deserialize)]
#[serde(tag = "kind", rename_all = "snake_case")]
pub enum HttpJob {
    /// Every page of one endpoint.
    Paginate(Box<Request>),
    /// Every page of one endpoint per key of `over`, up to `concurrency` endpoints at once.
    FanOut(Box<FanOut>),
}

/// One endpoint and how to page through it.
#[derive(Debug, Clone, Deserialize)]
pub struct Request {
    pub client: ClientSpec,
    /// A path under the connection's `base_url`, with `{name}` placeholders in a fan-out.
    pub path: String,
    /// Query parameters; a null value is left out.
    #[serde(default)]
    pub params: Map<String, Value>,
    pub paginator: Paginator,
    /// Where the rows are in each response, such as `$.items`; `$` (or none) is the whole response.
    #[serde(default)]
    pub select: Option<String>,
    /// What an HTTP status other than success means, such as `{"404": "skip"}`.
    #[serde(default)]
    pub on_status: BTreeMap<String, StatusAction>,
}

/// A request per key of `over`.
#[derive(Debug, Clone, Deserialize)]
pub struct FanOut {
    #[serde(flatten)]
    pub request: Request,
    pub over: Over,
    pub concurrency: usize,
}

/// The keys a fan-out requests.
#[derive(Debug, Clone, Deserialize)]
#[serde(tag = "kind", rename_all = "snake_case")]
pub enum Over {
    /// Scalars (filling the path's one placeholder) or objects (filling placeholders by name).
    Values { values: Vec<Value> },
    /// The values its `select` (`$.items[*].id`) picks from every page, named after the last field.
    Paginate(Box<Request>),
}

/// The connection a job uses and how it calls it.
#[derive(Debug, Clone, Deserialize)]
pub struct ClientSpec {
    pub connection: String,
    #[serde(default)]
    pub auth: Auth,
    /// Requests allowed per second, minute or hour, such as `10/s`, shared by every job of the connection with the same limit.
    #[serde(default)]
    pub rate_limit: Option<String>,
    /// Retries of a request that failed with a transport error, 429 or a 5xx.
    pub retries: u32,
    #[serde(default)]
    pub headers: BTreeMap<String, String>,
}

/// Credentials from the connection's settings, sent only to the connection's own origin.
#[derive(Debug, Clone, Default, Deserialize)]
#[serde(tag = "kind", rename_all = "snake_case")]
pub enum Auth {
    #[default]
    None,
    /// `Authorization: Bearer <token>`.
    Bearer,
    /// The token in a header or a query parameter.
    ApiKey {
        #[serde(default)]
        header: Option<String>,
        #[serde(default)]
        query: Option<String>,
    },
    /// `username` and `password`.
    Basic,
}

/// How the next page is found; always named, never guessed.
#[derive(Debug, Clone, Deserialize)]
#[serde(tag = "kind", rename_all = "snake_case")]
pub enum Paginator {
    /// One request.
    Single,
    /// The next page's URL at `pointer` in the body; ends when it is missing, null or empty.
    NextLink { pointer: String },
    /// The `Link` header's `rel="next"`.
    LinkHeader,
    /// `param` counts rows from 0 by `limit`; ends at a page with fewer rows.
    Offset {
        param: String,
        limit_param: String,
        limit: u64,
    },
    /// `param` counts pages from `start`; ends at an empty page.
    PageNumber { param: String, start: u64 },
    /// The token at `pointer` in the body goes back as `param`; ends when it is missing, null or empty.
    Cursor { pointer: String, param: String },
}

/// What a listed status does.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum StatusAction {
    /// No rows for that request.
    Skip,
}

/// One response's body, and where its rows are.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct HttpPage {
    pub bytes: Vec<u8>,
    /// The dotted path to the rows, as `sl.json(..., path=...)` takes it.
    pub path: Option<String>,
    /// The request, secrets hidden, for errors about the page.
    pub source: String,
}

/// A parsed `select`: the fields down to the rows and, for keys, the field of each row.
#[derive(Debug, Clone, PartialEq, Eq, Default)]
pub struct Selector {
    pub rows: Vec<String>,
    pub field: Option<String>,
}

/// A connection resolved for requests: its base URL, headers and credentials.
#[derive(Debug, Clone)]
pub struct Endpoint {
    pub base_url: reqwest::Url,
    pub headers: BTreeMap<String, String>,
    pub credential: Credential,
    /// Values never shown in errors.
    pub secrets: Vec<String>,
}

/// Credentials ready to attach to a request.
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum Credential {
    None,
    Header { name: String, value: String },
    Query { name: String, value: String },
}

/// Requests spaced `interval` apart, shared across jobs.
#[derive(Debug)]
pub struct Limiter {
    pub interval: Duration,
    pub next: Mutex<Instant>,
}

/// What one request needs: the pooled client, the endpoint, the shared rate limit and the retry budget.
#[derive(Debug, Clone)]
pub struct Calls {
    pub http: reqwest::Client,
    pub endpoint: Endpoint,
    pub limiter: Option<std::sync::Arc<Limiter>>,
    pub retries: u32,
    pub on_status: BTreeMap<u16, StatusAction>,
}

/// What one try at a request came to.
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum Attempt {
    /// The body, or `None` when `on_status` skips the status.
    Done(Option<Fetched>),
    /// A failure worth retrying, after `after` when the server said.
    Retry {
        reason: String,
        after: Option<Duration>,
    },
}

/// A body fetched, with the `Link` header's next URL.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Fetched {
    pub bytes: Vec<u8>,
    pub next_link: Option<String>,
    pub url: reqwest::Url,
}

/// Where a fan-out's keys come from: listed values, or another endpoint's pages.
#[derive(Debug)]
pub enum KeyFeed {
    Values(Vec<Value>),
    Pages(Box<(Calls, Request)>),
}