use crate::load_events::main::emit::emit;
use crate::load_events::models::EventKind;
use std::collections::{BTreeMap, HashMap};
use std::sync::{Arc, Mutex, OnceLock};
use std::time::Duration;
use base64::Engine;
use serde_json::{Map, Value};
use crate::http_source::_helpers::oauth::shared_token_source;
use crate::http_source::_helpers::urls::encoded;
use crate::http_source::constants::SECRET_HEADER_WORDS;
use crate::http_source::models::{Auth, ClientSpec, Credential, Endpoint, Grant, Limiter};
pub(crate) fn endpoint(
client: &ClientSpec,
connections: &Map<String, Value>,
) -> Result<Endpoint, String> {
let name = &client.connection;
let settings = connections.get(name).ok_or_else(|| {
format!("sl.http uses connection {name}, which is not in sqlload.toml or is a target's connection")
})?;
let text = |key: &str| settings.get(key).and_then(Value::as_str).map(str::to_owned);
let base =
text("base_url").ok_or_else(|| format!("connection {name} needs base_url for sl.http"))?;
let base_url = reqwest::Url::parse(&base)
.map_err(|error| format!("connection {name}: base_url {base} is not a URL: {error}"))?;
let needed = |key: &str| {
text(key).filter(|value| !value.is_empty()).ok_or_else(|| {
format!("connection {name} needs {key} for the sl.http auth this job uses")
})
};
let present = |key: &str| text(key).filter(|value| !value.is_empty());
let credential = match &client.auth {
Auth::None => Credential::None,
Auth::Bearer => Credential::Header {
name: "Authorization".to_owned(),
value: format!("Bearer {}", needed("token")?),
},
Auth::ApiKey {
header: Some(header),
query: None,
} => Credential::Header {
name: header.clone(),
value: needed("token")?,
},
Auth::ApiKey {
header: None,
query: Some(query),
} => Credential::Query {
name: query.clone(),
value: needed("token")?,
},
Auth::ApiKey { .. } => {
return Err("sl.api_key takes exactly one of header= or query=".to_owned());
}
Auth::Basic => {
let pair = format!("{}:{}", needed("username")?, needed("password")?);
Credential::Header {
name: "Authorization".to_owned(),
value: format!(
"Basic {}",
base64::engine::general_purpose::STANDARD.encode(pair)
),
}
}
Auth::Oauth2ClientCredentials(spec) => {
let grant = Grant::ClientCredentials {
client_id: needed("client_id")?,
client_secret: needed("client_secret")?,
};
Credential::OAuth2(shared_token_source(
name,
&format!("{:?}", client.auth),
spec,
grant,
)?)
}
Auth::Oauth2RefreshToken(spec) => {
let grant = Grant::RefreshToken {
refresh_token: needed("refresh_token")?,
client: (present("client_id"), present("client_secret")),
};
Credential::OAuth2(shared_token_source(
name,
&format!("{:?}", client.auth),
spec,
grant,
)?)
}
Auth::Oauth2Password(spec) => {
let grant = Grant::Password {
username: needed("username")?,
password: needed("password")?,
client: (present("client_id"), present("client_secret")),
};
Credential::OAuth2(shared_token_source(
name,
&format!("{:?}", client.auth),
spec,
grant,
)?)
}
};
let mut headers = settings
.get("headers")
.and_then(Value::as_object)
.map(text_headers)
.unwrap_or_default();
headers.extend(client.headers.clone());
let secrets = secret_texts(
&credential,
&headers,
[
text("token"),
text("password"),
text("client_secret"),
text("refresh_token"),
],
);
Ok(Endpoint {
base_url,
headers,
credential,
secrets,
})
}
fn secret_texts(
credential: &Credential,
headers: &BTreeMap<String, String>,
raw: [Option<String>; 4],
) -> Vec<String> {
let mut secrets: Vec<String> = raw.into_iter().flatten().collect();
let encoded: Vec<String> = secrets.iter().map(|secret| encoded(secret)).collect();
secrets.extend(encoded);
if let Credential::Header { value, .. } | Credential::Query { value, .. } = credential {
secrets.push(value.clone());
secrets.extend(value.split_once(' ').map(|(_, rest)| rest.to_owned()));
}
secrets.extend(
headers
.iter()
.filter(|(name, _)| secret_name(name))
.map(|(_, value)| value.clone()),
);
secrets.retain(|secret| !secret.is_empty());
secrets.sort_by_key(|secret| std::cmp::Reverse(secret.len()));
secrets
}
impl Endpoint {
pub(crate) fn current_secrets(&self) -> Vec<String> {
let mut secrets = self.secrets.clone();
if let Credential::OAuth2(tokens) = &self.credential {
secrets.extend(tokens.secrets());
secrets.sort_by_key(|secret| std::cmp::Reverse(secret.len()));
}
secrets
}
}
fn secret_name(name: &str) -> bool {
let name = name.to_ascii_lowercase();
SECRET_HEADER_WORDS.iter().any(|word| name.contains(word))
}
fn text_headers(headers: &Map<String, Value>) -> BTreeMap<String, String> {
headers
.iter()
.filter_map(|(key, value)| Some((key.clone(), value.as_str()?.to_owned())))
.collect()
}
pub(crate) fn shared_limiter(connection: &str, rate: &str) -> Result<Arc<Limiter>, String> {
static LIMITERS: OnceLock<Mutex<HashMap<String, Arc<Limiter>>>> = OnceLock::new();
let interval = interval(rate)?;
let limiters = LIMITERS.get_or_init(Mutex::default);
let mut limiters = limiters
.lock()
.map_err(|_| "the rate limiters' lock is poisoned".to_owned())?;
let limiter = limiters
.entry(format!("{connection}\u{0}{rate}"))
.or_insert_with(|| {
Arc::new(Limiter {
connection: connection.to_owned(),
rate: rate.to_owned(),
interval,
next: Mutex::new(tokio::time::Instant::now()),
})
});
Ok(limiter.clone())
}
pub(crate) fn interval(rate: &str) -> Result<Duration, String> {
let invalid =
|| format!("rate_limit {rate:?} is not a rate such as \"10/s\", \"600/m\" or \"1000/h\"");
let (count, unit) = rate.split_once('/').ok_or_else(invalid)?;
let count: f64 = count.trim().parse().map_err(|_| invalid())?;
let seconds = match unit.trim() {
"s" => 1.0,
"m" => 60.0,
"h" => 3_600.0,
_ => return Err(invalid()),
};
match count.is_normal() && count.is_sign_positive() {
true => Ok(Duration::from_secs_f64(seconds / count)),
false => Err(invalid()),
}
}
impl Limiter {
pub(crate) async fn wait(&self) {
let wait = match self.next.lock() {
Ok(mut next) => {
let now = tokio::time::Instant::now();
let turn = (*next).max(now);
*next = turn + self.interval;
turn - now
}
Err(_) => Duration::ZERO,
};
if !wait.is_zero() {
emit(
None,
EventKind::RateLimited {
connection: self.connection.clone(),
rate: self.rate.clone(),
wait_ms: u64::try_from(wait.as_millis()).unwrap_or(u64::MAX),
},
);
}
tokio::time::sleep(wait).await;
}
}