use std::collections::BTreeMap;
use std::sync::Mutex;
use std::time::{Duration, Instant};
use serde::Deserialize;
use serde_json::{Map, Value};
#[derive(Debug, Clone, Deserialize)]
#[serde(tag = "kind", rename_all = "snake_case")]
pub enum HttpJob {
Paginate(Box<Request>),
FanOut(Box<FanOut>),
}
#[derive(Debug, Clone, Deserialize)]
pub struct Request {
pub client: ClientSpec,
pub path: String,
#[serde(default)]
pub params: Map<String, Value>,
#[serde(default)]
pub method: Method,
#[serde(default)]
pub json_body: Option<Value>,
pub paginator: Paginator,
#[serde(default)]
pub select: Option<String>,
#[serde(default)]
pub on_status: BTreeMap<String, StatusAction>,
#[serde(default)]
pub fields: Option<Vec<FieldSpec>>,
#[serde(default)]
pub keep_record: Option<String>,
}
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Deserialize)]
#[serde(rename_all = "UPPERCASE")]
pub enum Method {
#[default]
Get,
Post,
}
impl Method {
pub fn as_str(self) -> &'static str {
match self {
Self::Get => "GET",
Self::Post => "POST",
}
}
}
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum Location {
#[default]
Query,
Body,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct PageCall {
pub method: Method,
pub url: reqwest::Url,
pub body: Option<Value>,
}
#[derive(Debug, Clone, Deserialize)]
pub struct FieldSpec {
pub name: String,
pub path: String,
#[serde(default)]
pub required: bool,
}
#[derive(Debug, Clone, Deserialize)]
pub struct FanOut {
#[serde(flatten)]
pub request: Request,
pub over: Over,
pub concurrency: usize,
}
#[derive(Debug, Clone, Deserialize)]
#[serde(tag = "kind", rename_all = "snake_case")]
pub enum Over {
Values { values: Vec<Value> },
Paginate(Box<Request>),
}
#[derive(Debug, Clone, Deserialize)]
pub struct ClientSpec {
pub connection: String,
#[serde(default)]
pub auth: Auth,
#[serde(default)]
pub rate_limit: Option<String>,
pub retries: u32,
#[serde(default)]
pub headers: BTreeMap<String, String>,
}
#[derive(Debug, Clone, Default, Deserialize)]
#[serde(tag = "kind", rename_all = "snake_case")]
pub enum Auth {
#[default]
None,
Bearer,
ApiKey {
#[serde(default)]
header: Option<String>,
#[serde(default)]
query: Option<String>,
},
Basic,
Oauth2ClientCredentials(OAuth2Spec),
Oauth2RefreshToken(OAuth2Spec),
Oauth2Password(OAuth2Spec),
}
#[derive(Debug, Clone, Deserialize)]
pub struct OAuth2Spec {
pub token_url: String,
#[serde(default)]
pub scope: Option<String>,
#[serde(default)]
pub client_auth: ClientAuth,
}
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum ClientAuth {
#[default]
Basic,
Post,
}
#[derive(Debug)]
pub struct TokenSource {
pub token_url: reqwest::Url,
pub form: Vec<(String, String)>,
pub refresh_form: Option<Vec<(String, String)>>,
pub basic: Option<(String, String)>,
pub state: tokio::sync::Mutex<TokenState>,
pub secrets: Mutex<Vec<String>>,
}
#[derive(Debug, Default)]
pub struct TokenState {
pub access: Option<String>,
pub issued_at: Option<Instant>,
pub renew_at: Option<Instant>,
pub refresh_token: Option<String>,
}
#[derive(Debug, Clone)]
pub enum Grant {
ClientCredentials {
client_id: String,
client_secret: String,
},
RefreshToken {
refresh_token: String,
client: (Option<String>, Option<String>),
},
Password {
username: String,
password: String,
client: (Option<String>, Option<String>),
},
}
#[derive(Debug, Clone)]
pub struct Issued {
pub access: String,
pub renew_after: Option<Duration>,
pub refresh_token: Option<String>,
}
#[derive(Debug, Clone, Deserialize)]
#[serde(tag = "kind", rename_all = "snake_case")]
pub enum Paginator {
Single,
NextLink { pointer: String },
LinkHeader,
Offset {
param: String,
limit_param: String,
limit: u64,
#[serde(default)]
location: Location,
},
PageNumber {
param: String,
start: u64,
#[serde(default)]
location: Location,
},
Cursor {
pointer: String,
param: String,
#[serde(default)]
location: Location,
},
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum StatusAction {
Skip,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct HttpPage {
pub bytes: Vec<u8>,
pub path: Option<String>,
pub source: String,
pub projection: Option<crate::row_batches::models::Projection>,
}
#[derive(Debug, Clone, PartialEq, Eq, Default)]
pub struct Selector {
pub rows: Vec<String>,
pub field: Option<String>,
}
#[derive(Debug, Clone)]
pub struct Endpoint {
pub base_url: reqwest::Url,
pub headers: BTreeMap<String, String>,
pub credential: Credential,
pub secrets: Vec<String>,
}
#[derive(Debug, Clone)]
pub enum Credential {
None,
Header {
name: String,
value: String,
},
Query {
name: String,
value: String,
},
OAuth2(std::sync::Arc<TokenSource>),
}
#[derive(Debug)]
pub struct Limiter {
pub interval: Duration,
pub next: Mutex<Instant>,
}
#[derive(Debug, Clone)]
pub struct Calls {
pub http: reqwest::Client,
pub endpoint: Endpoint,
pub limiter: Option<std::sync::Arc<Limiter>>,
pub retries: u32,
pub on_status: BTreeMap<u16, StatusAction>,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum Attempt {
Done(Option<Fetched>),
Refused,
Retry {
reason: String,
after: Option<Duration>,
},
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Fetched {
pub bytes: Vec<u8>,
pub next_link: Option<String>,
pub url: reqwest::Url,
}
#[derive(Debug)]
pub enum KeyFeed {
Values(Vec<Value>),
Pages(Box<(Calls, Request)>),
}