use std::collections::BTreeMap;
use std::future::Future;
use std::pin::Pin;
use std::sync::Arc;
use serde_json::{Map, Value};
use tokio::sync::Mutex;
use tokio::sync::mpsc::{Receiver, Sender, channel};
use tokio::task::JoinSet;
use crate::http_source::_helpers::cells::cells_problem;
use crate::http_source::_helpers::endpoint::{endpoint, interval, shared_limiter};
use crate::http_source::_helpers::oauth::token_url;
use crate::http_source::_helpers::pages::{paginate, set_at};
use crate::http_source::_helpers::selector::{selected_rows, selector};
use crate::http_source::_helpers::urls::{filled_path, placeholders, redacted};
use crate::python_sources::errors::ReadError;
use crate::row_batches::main::projection::projection;
use crate::row_batches::main::projection_problem::projection_problem;
use crate::row_batches::models::{Projection, ProjectionRules};
use std::time::Duration;
use crate::http::main::install_tls_provider::install_tls_provider;
use crate::http_source::constants::{
BODY_DROPPING_REDIRECTS, JOB_THREADS, MAX_REDIRECTS, PAGES_AHEAD,
};
use crate::http_source::models::{
Auth, Calls, FanKey, FanOut, FieldSpec, HttpJob, HttpPage, KeyFeed, Location, Method, Over,
Paginator, Request, Selector, Sink, StatusAction,
};
pub(crate) fn job_problems(job: &HttpJob, rules: ProjectionRules<'_>) -> Vec<String> {
let rows = match job {
HttpJob::Paginate(request) => request,
HttpJob::FanOut(fan_out) => &fan_out.request,
};
let mut problems = match request_projection(rows) {
Ok(Some(projection)) => projection_problem(&projection, rules).into_iter().collect(),
Ok(None) => Vec::new(),
Err(problem) => vec![problem],
};
problems.extend(match job {
HttpJob::Paginate(request) => {
let mut problems = request_problems(request);
problems.extend(field_problem(request));
problems.extend(placeholders(&request.path).first().map(|name| {
format!(
"paginate path {} has the placeholder {{{name}}}; use fan_out to fill it",
request.path
)
}));
problems
}
HttpJob::FanOut(fan_out) => fan_out_problems(fan_out),
});
problems.extend(rows.key.as_ref().map(|key| {
format!("key={key:?} names the keys a job gives as a fan_out's over; this job loads rows")
}));
problems
}
pub(crate) fn request_projection(request: &Request) -> Result<Option<Projection>, String> {
let fields = request.fields.as_deref().map(field_triples);
projection(fields.as_deref(), request.keep_record.as_deref())
}
fn field_triples(fields: &[FieldSpec]) -> Vec<(String, String, bool)> {
fields
.iter()
.map(|field| (field.name.clone(), field.path.clone(), field.required))
.collect()
}
fn field_problem(request: &Request) -> Option<String> {
let field = match selector(request.select.as_deref()) {
Ok(selected) => selected.field?,
Err(_) => return None,
};
Some(format!(
"select {:?} names a key field ({field}); that form is for fan_out's over, so select the rows instead",
request.select.clone().unwrap_or_default()
))
}
fn request_problems(request: &Request) -> Vec<String> {
let mut problems: Vec<String> = Vec::new();
problems.extend(selector(request.select.as_deref()).err());
problems.extend(paginator_problem(&request.paginator));
problems.extend(body_problems(request));
problems.extend(request.cells.as_ref().and_then(cells_problem));
problems.extend(match &request.client.auth {
Auth::Oauth2ClientCredentials(spec)
| Auth::Oauth2RefreshToken(spec)
| Auth::Oauth2Password(spec) => token_url(spec).err(),
Auth::None | Auth::Bearer | Auth::ApiKey { .. } | Auth::Basic => None,
});
problems.extend(
request
.client
.headers
.iter()
.filter_map(|(name, value)| header_problem(name, value)),
);
problems.extend(
request
.client
.rate_limit
.as_deref()
.and_then(|rate| interval(rate).err()),
);
problems.extend(
request
.on_status
.keys()
.filter_map(|status| match status.parse::<u16>() {
Ok(100..=599) => None,
_ => Some(format!("on_status {status} is not an HTTP status")),
}),
);
problems
}
fn paginator_problem(paginator: &Paginator) -> Option<String> {
match paginator {
Paginator::NextLink { pointer } | Paginator::Cursor { pointer, .. } => {
match selector(Some(pointer)) {
Ok(parsed) if parsed.field.is_none() && !parsed.rows.is_empty() => None,
Ok(_) | Err(_) => Some(format!(
"paginator pointer {pointer:?} is not a path such as \"$.next\" or \"$.meta.next_cursor\""
)),
}
}
Paginator::Offset { limit: 0, .. } => Some("sl.offset limit must be at least 1".to_owned()),
Paginator::Offset { .. }
| Paginator::Single
| Paginator::LinkHeader
| Paginator::PageNumber { .. } => None,
}
}
fn body_problems(request: &Request) -> Vec<String> {
let (paths, location) = match &request.paginator {
Paginator::Offset {
param,
limit_param,
location,
..
} => (vec![param, limit_param], *location),
Paginator::PageNumber {
param, location, ..
}
| Paginator::Cursor {
param, location, ..
} => (vec![param], *location),
Paginator::Single | Paginator::NextLink { .. } | Paginator::LinkHeader => {
(Vec::new(), Location::Query)
}
};
let mut problems: Vec<String> = Vec::new();
if request.method == Method::Get && request.json_body.is_some() {
problems.push("json_body is sent only with method=\"POST\"".to_owned());
}
if location == Location::Query {
return problems;
}
if request.method == Method::Get {
problems.push("a paginator with location=\"body\" needs method=\"POST\"".to_owned());
}
if request
.json_body
.as_ref()
.is_some_and(|body| !body.is_object())
{
problems.push(
"a paginator with location=\"body\" sets fields in json_body, so json_body must be an object".to_owned(),
);
}
problems.extend(paths.iter().filter_map(|path| match selector(Some(path)) {
Ok(parsed) if parsed.field.is_none() && named_fields(&parsed.rows) => None,
Ok(_) | Err(_) => Some(format!(
"body path {path:?} is not a path such as \"$.page.cursor\""
)),
}));
if problems.is_empty() {
let mut body = request
.json_body
.clone()
.unwrap_or_else(|| Value::Object(Map::new()));
problems.extend(
paths
.iter()
.filter_map(|path| set_at(&mut body, path, Value::Null).err()),
);
}
problems
}
fn named_fields(segments: &[String]) -> bool {
!segments.is_empty() && !segments.iter().any(|segment| segment.starts_with('['))
}
fn header_problem(name: &str, value: &str) -> Option<String> {
match (
reqwest::header::HeaderName::from_bytes(name.as_bytes()),
reqwest::header::HeaderValue::from_str(value),
) {
(Ok(_), Ok(_)) => None,
(Err(_), _) | (_, Err(_)) => Some(format!(
"header {name:?} is not a valid HTTP header name and value"
)),
}
}
fn fan_out_problems(fan_out: &FanOut) -> Vec<String> {
let mut problems = request_problems(&fan_out.request);
problems.extend(field_problem(&fan_out.request));
problems.extend(level_problems(fan_out));
problems
}
fn level_problems(fan_out: &FanOut) -> Vec<String> {
let path = &fan_out.request.path;
let wanted = placeholders(path);
let mut problems: Vec<String> = Vec::new();
problems.extend(
(fan_out.concurrency == 0).then(|| "fan_out concurrency must be at least 1".to_owned()),
);
let (picker, kind) = match &fan_out.over {
Over::Values { .. } if wanted.is_empty() => {
problems.push(format!(
"fan-out path {path} has no placeholder, so every key would request the same URL; add one such as {{id}}"
));
return problems;
}
Over::Values { values } => {
problems.extend(
values
.iter()
.find_map(|value| key_problem(path, &wanted, value)),
);
return problems;
}
Over::Paginate(over) => {
problems.extend(request_problems(over));
problems.extend(picker_problem(over));
problems.extend(placeholders(&over.path).first().map(|name| {
format!("fan_out over path {} has the placeholder {{{name}}}; over must be a plain paginate", over.path)
}));
(&**over, "a paginate")
}
Over::FanOut(inner) => {
problems.extend(request_problems(&inner.request));
problems.extend(level_problems(inner));
problems.extend(picker_problem(&inner.request));
(&inner.request, "a fan_out")
}
};
let Some(name) = key_name(picker) else {
problems.push(format!(
"fan_out over {kind} needs a select naming each key's field, such as \"$.items[*].id\", not {:?}",
picker.select.clone().unwrap_or_else(|| "$".to_owned())
));
return problems;
};
let above = match &fan_out.over {
Over::FanOut(inner) => provided_keys(&inner.over, &placeholders(&inner.request.path)),
Over::Values { .. } | Over::Paginate(_) => Vec::new(),
};
problems.extend(above.contains(&name).then(|| {
format!(
"fan_out over picks {name}, which a level above already gives; name it apart with key="
)
}));
let given: Vec<String> = above.into_iter().chain([name]).collect();
problems.extend(
wanted
.iter()
.filter(|name| !given.contains(name))
.map(|name| {
format!(
"fan-out path {path} needs {{{name}}}, but over gives only {}",
given.join(", ")
)
}),
);
problems
}
fn picker_problem(picker: &Request) -> Option<String> {
match (picker.fields.is_some() || picker.keep_record.is_some(), picker.cells.is_some()) {
(true, _) => Some("fan_out's over picks keys with its select; fields and keep_record are for the fan-out's own rows".to_owned()),
(false, true) => Some("fan_out's over picks keys with its select; cells are for the fan-out's own rows".to_owned()),
(false, false) => None,
}
}
fn key_name(picker: &Request) -> Option<String> {
let field = match selector(picker.select.as_deref()) {
Ok(selected) => selected.field,
Err(_) => None,
}?;
Some(picker.key.clone().unwrap_or(field))
}
fn in_every(objects: &[&Map<String, Value>], name: &str) -> bool {
objects.iter().all(|object| object.contains_key(name))
}
fn provided_keys(over: &Over, wanted: &[String]) -> Vec<String> {
match over {
Over::Values { values } => {
let objects: Vec<&Map<String, Value>> =
values.iter().filter_map(Value::as_object).collect();
match objects.split_first() {
Some((first, rest)) if objects.len() == values.len() => first
.keys()
.filter(|name| in_every(rest, name))
.cloned()
.collect(),
Some(_) | None => wanted.to_vec(),
}
}
Over::Paginate(over) => key_name(over).into_iter().collect(),
Over::FanOut(inner) => provided_keys(&inner.over, &placeholders(&inner.request.path))
.into_iter()
.chain(key_name(&inner.request))
.collect(),
}
}
fn key_problem(path: &str, wanted: &[String], value: &Value) -> Option<String> {
let missing = |key: &Map<String, Value>| {
wanted
.iter()
.find(|name| !key.contains_key(name.as_str()))
.cloned()
};
match value {
Value::Object(key) => missing(key).map(|name| {
format!("fan-out path {path} needs {{{name}}}, which the key {value} lacks")
}),
_ if wanted.len() == 1 => None,
_ => Some(format!(
"fan-out path {path} has {} placeholders, so over needs objects naming each, not {value}",
wanted.len()
)),
}
}
pub(crate) fn drive_job(
job: HttpJob,
connections: &Map<String, Value>,
mut deliver: impl FnMut(HttpPage) -> bool,
) -> Result<(), ReadError> {
let runtime = tokio::runtime::Builder::new_multi_thread()
.worker_threads(JOB_THREADS)
.enable_all()
.build()
.map_err(|error| format!("cannot start the sl.http runtime: {error}"))?;
let (sender, mut receiver) = channel(PAGES_AHEAD);
let task = runtime.spawn(run_job(job, connections.clone(), sender));
while let Some(page) = receiver.blocking_recv() {
if !deliver(page) {
task.abort();
break;
}
}
drop(receiver);
match runtime.block_on(task) {
Ok(result) => result,
Err(error) if error.is_cancelled() => Ok(()),
Err(error) => Err(format!("the sl.http job stopped: {error}").into()),
}
}
async fn run_job(
job: HttpJob,
connections: Map<String, Value>,
pages: Sender<HttpPage>,
) -> Result<(), ReadError> {
let posts = match &job {
HttpJob::Paginate(request) => request.method == Method::Post,
HttpJob::FanOut(fan_out) => fan_out_posts(fan_out),
};
let http = job_client(posts)?;
match job {
HttpJob::Paginate(request) => {
let calls = request_calls(&request, &connections, &http)?;
paginate(&calls, &request, &request.path, &pages).await
}
HttpJob::FanOut(fan_out) => {
run_fan_out(*fan_out, &connections, &http, Sink::Pages(pages)).await
}
}
}
fn fan_out_posts(fan_out: &FanOut) -> bool {
fan_out.request.method == Method::Post
|| match &fan_out.over {
Over::Values { .. } => false,
Over::Paginate(over) => over.method == Method::Post,
Over::FanOut(inner) => fan_out_posts(inner),
}
}
fn job_client(posts: bool) -> Result<reqwest::Client, String> {
install_tls_provider();
reqwest::Client::builder()
.redirect(reqwest::redirect::Policy::custom(move |attempt| {
match posts && BODY_DROPPING_REDIRECTS.contains(&attempt.status().as_u16()) {
true => attempt.stop(),
false => same_origin_redirect(attempt),
}
}))
.pool_idle_timeout(Duration::from_secs(90))
.connect_timeout(Duration::from_secs(15))
.timeout(Duration::from_secs(300))
.user_agent(concat!("sqlload/", env!("CARGO_PKG_VERSION")))
.build()
.map_err(|error| format!("cannot start the HTTP client: {error}"))
}
fn same_origin_redirect(attempt: reqwest::redirect::Attempt<'_>) -> reqwest::redirect::Action {
let crossed = attempt
.previous()
.first()
.is_some_and(|first| first.origin() != attempt.url().origin());
match crossed || attempt.previous().len() > MAX_REDIRECTS {
true => attempt.stop(),
false => attempt.follow(),
}
}
fn request_calls(
request: &Request,
connections: &Map<String, Value>,
http: &reqwest::Client,
) -> Result<Calls, String> {
let client = &request.client;
let limiter = client
.rate_limit
.as_deref()
.map(|rate| shared_limiter(&client.connection, rate))
.transpose()?;
let on_status = request
.on_status
.iter()
.map(|(status, action)| {
let code = status
.parse::<u16>()
.map_err(|_| format!("on_status {status} is not an HTTP status"))?;
Ok((code, *action))
})
.collect::<Result<BTreeMap<u16, StatusAction>, String>>()?;
Ok(Calls {
http: http.clone(),
endpoint: endpoint(client, connections)?,
limiter,
retries: client.retries,
on_status,
})
}
fn run_fan_out<'a>(
fan_out: FanOut,
connections: &'a Map<String, Value>,
http: &'a reqwest::Client,
sink: Sink,
) -> LevelRun<'a> {
Box::pin(async move {
let calls = request_calls(&fan_out.request, connections, http)?;
let (key_sender, key_receiver) = channel::<FanKey>(fan_out.concurrency * 2);
let keys = Arc::new(Mutex::new(key_receiver));
let mut tasks: JoinSet<Result<(), ReadError>> = JoinSet::new();
let nested = matches!(sink, Sink::Keys(_)) || matches!(fan_out.over, Over::FanOut(_));
let feed = match fan_out.over.clone() {
Over::Values { values } => KeyFeed::Values(values),
Over::Paginate(over) => {
KeyFeed::Pages(Box::new((request_calls(&over, connections, http)?, *over)))
}
Over::FanOut(inner) => {
KeyFeed::FanOut(Box::new((*inner, connections.clone(), http.clone())))
}
};
let wanted = placeholders(&fan_out.request.path);
tasks.spawn(feed_keys(feed, wanted, key_sender, nested));
let request = Arc::new(fan_out.request);
for _ in 0..fan_out.concurrency {
let (calls, request, keys, sink) =
(calls.clone(), request.clone(), keys.clone(), sink.clone());
tasks.spawn(async move {
loop {
let key = keys.lock().await.recv().await;
let Some(key) = key else { return Ok(()) };
fan_out_key(&calls, &request, &key, &sink)
.await
.map_err(|error| ReadError {
message: match nested {
true => format!(
"{}{}",
redacted(&key.trail, &calls.endpoint.current_secrets()),
error.message
),
false => error.message,
},
retryable: error.retryable,
})?;
}
});
}
while let Some(finished) = tasks.join_next().await {
finished.map_err(|error| format!("a fan-out request stopped: {error}"))??;
}
Ok(())
})
}
type LevelRun<'a> = Pin<Box<dyn Future<Output = Result<(), ReadError>> + Send + 'a>>;
async fn fan_out_key(
calls: &Calls,
request: &Request,
key: &FanKey,
sink: &Sink,
) -> Result<(), ReadError> {
let path = filled_path(&request.path, &key.fields)?;
match sink {
Sink::Pages(pages) => paginate(calls, request, &path, pages).await,
Sink::Keys(keys) => {
let picker = key_picker(request)?;
let (page_sender, page_receiver) = channel::<HttpPage>(PAGES_AHEAD);
let shown = path.clone();
let pager = async move { paginate(calls, request, &path, &page_sender).await };
let secrets = calls.endpoint.current_secrets();
let forward = forward_keys(page_receiver, &picker, (key, &shown, Some(&secrets)), keys);
let (paged, forwarded) = tokio::join!(pager, forward);
paged.and(forwarded)
}
}
}
fn key_picker(request: &Request) -> Result<(Selector, String, String), String> {
let selected = selector(request.select.as_deref())?;
let field = selected.field.clone().unwrap_or_default();
let name = request.key.clone().unwrap_or_else(|| field.clone());
Ok((selected, field, name))
}
async fn feed_keys(
feed: KeyFeed,
wanted: Vec<String>,
keys: Sender<FanKey>,
nested: bool,
) -> Result<(), ReadError> {
let single = |value: Value| -> FanKey {
let fields = match value {
Value::Object(key) => key,
other => wanted
.first()
.map(|name| (name.clone(), other))
.into_iter()
.collect(),
};
FanKey {
fields,
trail: String::new(),
}
};
match feed {
KeyFeed::Values(values) => {
for value in values {
if keys.send(single(value)).await.is_err() {
return Ok(());
}
}
Ok(())
}
KeyFeed::Pages(pages) => {
let (calls, over) = *pages;
let picker = key_picker(&over)?;
let path = over.path.clone();
let secrets = nested.then(|| calls.endpoint.current_secrets());
let (page_sender, page_receiver) = channel::<HttpPage>(PAGES_AHEAD);
let pager = async move { paginate(&calls, &over, &over.path, &page_sender).await };
let root = FanKey::default();
let forward = forward_keys(
page_receiver,
&picker,
(&root, &path, secrets.as_deref()),
&keys,
);
let (paged, forwarded) = tokio::join!(pager, forward);
paged.and(forwarded)
}
KeyFeed::FanOut(level) => {
let (inner, connections, http) = *level;
run_fan_out(inner, &connections, &http, Sink::Keys(keys)).await
}
}
}
async fn forward_keys(
mut pages: Receiver<HttpPage>,
(selected, field, name): &(Selector, String, String),
(parent, path, secrets): (&FanKey, &str, Option<&[String]>),
keys: &Sender<FanKey>,
) -> Result<(), ReadError> {
while let Some(page) = pages.recv().await {
let document: Value = serde_json::from_slice(&page.bytes)
.map_err(|error| format!("the fan-out's over response is not JSON: {error}"))?;
for row in selected_rows(&document, selected) {
let Some(value) = row.get(field).filter(|value| !value.is_null()) else {
continue;
};
let mut fields = parent.fields.clone();
fields.insert(name.clone(), value.clone());
let trail = secrets.map_or_else(String::new, |secrets| {
trail_step(&parent.trail, (path, name, value), secrets)
});
if keys.send(FanKey { fields, trail }).await.is_err() {
return Ok(());
}
}
}
Ok(())
}
fn trail_step(
trail: &str,
(path, name, value): (&str, &str, &Value),
secrets: &[String],
) -> String {
let shown = match value {
Value::String(text) => text.clone(),
other => other.to_string(),
};
format!(
"{trail}{}",
redacted(&format!("{path} {name}={shown} → "), secrets)
)
}