use std::collections::HashMap;
use std::net::IpAddr;
use std::sync::Arc;
use std::time::Duration;
use futures_util::StreamExt;
use indexmap::IndexMap;
use moka::future::Cache;
use serde::Deserialize;
use crate::bogon::{bogon_lookup, is_bogon, unmapped};
use crate::database::DatabaseApi;
use crate::entitlement::Entitlement;
use crate::error::{Error, ErrorKind};
use crate::flight::{Flights, restate};
use crate::lookup::Lookup;
use crate::models::{AsnLookupResponse, BatchLookupError, BatchLookupRequest};
use crate::models::{DomainLookupResponse, LookupResponse};
use crate::oauth::{OauthApi, OauthError, authorization_server};
use crate::transport::{Transport, encode_path_segment};
pub const DEFAULT_BASE_URL: &str = "https://api.internetdata.io";
const DEFAULT_CACHE_CAPACITY: u64 = 10_000;
const DEFAULT_CACHE_TTL: Duration = Duration::from_secs(60 * 60);
const DEFAULT_CONCURRENCY: usize = 8;
const DEFAULT_RETRIES: u32 = 2;
const DEFAULT_TIMEOUT: Duration = Duration::from_secs(30);
const DEFAULT_CONNECT_TIMEOUT: Duration = Duration::from_secs(10);
const DEFAULT_READ_TIMEOUT: Duration = Duration::from_secs(30);
const RETRY_BASE_DELAY: Duration = Duration::from_millis(250);
const MAX_RETRY_AFTER: Duration = Duration::from_millis(2_147_483_647);
const BATCH_MAX: usize = 1000;
#[derive(Debug, Clone)]
pub struct Client(Arc<Inner>);
#[derive(Debug)]
struct Inner {
transport: Transport,
cache: Option<Cache<String, Lookup>>,
flights: Flights,
concurrency: usize,
retries: u32,
timeout: Duration,
}
impl Client {
pub fn new() -> Result<Self, Error> {
Self::builder().build()
}
pub fn builder() -> ClientBuilder {
ClientBuilder::default()
}
pub async fn lookup(&self, ip: &str) -> Result<Lookup, Error> {
self.lookup_with(ip, LookupOptions::new()).await
}
pub async fn lookup_with(&self, ip: &str, opts: LookupOptions) -> Result<Lookup, Error> {
let timeout = self.call_timeout(opts.timeout)?;
let retries = opts.retries.unwrap_or(self.0.retries);
if ip.parse::<IpAddr>().is_err() {
return Err(Error::Config(format!(
"'{ip}' is not an IP address: look up an AS with lookup_asn, and a domain with \
lookup_domain"
)));
}
let carried = unmapped(ip);
let ip: &str = &carried;
if is_bogon(ip) {
return Ok(bogon_lookup(ip));
}
let Some(cache) = &self.0.cache else {
return self.serve(ip, retries, timeout).await;
};
loop {
if let Some(hit) = cache.get(ip).await {
return Ok(hit);
}
let (mut pilot, joined) = self.0.flights.board([ip]);
if let Some((_, flight)) = joined.into_iter().next() {
match flight.await {
Ok(landed) => return restate(&landed),
Err(_) => continue,
}
}
let answer = match cache.get(ip).await {
Some(hit) => Ok(hit),
None => {
let answer = self.serve(ip, retries, timeout).await;
if let Ok(lookup) = &answer {
cache.insert(ip.to_owned(), lookup.clone()).await;
}
answer
}
};
pilot.land(ip, &answer);
return answer;
}
}
async fn serve(&self, ip: &str, retries: u32, timeout: Duration) -> Result<Lookup, Error> {
let path = format!("/{}", encode_path_segment(ip));
let answer: LookupResponse =
with_retry(retries, || self.0.transport.get_json(&path, &[], timeout)).await?;
Ok(Lookup::served(answer))
}
pub async fn my_ip(&self) -> Result<Lookup, Error> {
self.my_ip_with(LookupOptions::new()).await
}
pub async fn my_ip_with(&self, opts: LookupOptions) -> Result<Lookup, Error> {
let timeout = self.call_timeout(opts.timeout)?;
let answer: LookupResponse = with_retry(opts.retries.unwrap_or(self.0.retries), || {
self.0.transport.get_json("/myip", &[], timeout)
})
.await?;
Ok(Lookup::served(answer))
}
pub async fn my_entitlement(&self) -> Result<Entitlement, Error> {
self.my_entitlement_with(LookupOptions::new()).await
}
pub async fn my_entitlement_with(&self, opts: LookupOptions) -> Result<Entitlement, Error> {
let timeout = self.call_timeout(opts.timeout)?;
with_retry(opts.retries.unwrap_or(self.0.retries), || {
self.0.transport.get_json("/api/v1/entitlement", &[], timeout)
})
.await
}
pub async fn lookup_batch<I, S>(
&self,
ips: I,
opts: BatchOptions,
) -> IndexMap<String, Result<Lookup, Error>>
where
I: IntoIterator<Item = S>,
S: AsRef<str>,
{
let asked = dedupe(ips);
let wire: Vec<String> = asked.iter().map(|ip| unmapped(ip).into_owned()).collect();
let unique = dedupe(&wire);
let retries = opts.retries.unwrap_or(self.0.retries);
let concurrency = opts.concurrency.unwrap_or(self.0.concurrency);
let refused = match self.call_timeout(opts.timeout) {
Err(err) => Some(err.message()),
Ok(_) if concurrency == 0 => Some("concurrency must be at least 1".to_owned()),
Ok(_) => None,
};
if let Some(message) = refused {
return asked.into_iter().map(|ip| (ip, Err(Error::Config(message.clone())))).collect();
}
let timeout = opts.timeout.unwrap_or(self.0.timeout);
let mut answers: HashMap<String, Result<Lookup, Error>> =
HashMap::with_capacity(unique.len());
let mut pending: Vec<String> = Vec::new();
for ip in &unique {
if is_bogon(ip) {
answers.insert(ip.clone(), Ok(bogon_lookup(ip)));
continue;
}
if let Some(cache) = &self.0.cache {
if let Some(hit) = cache.get(ip).await {
answers.insert(ip.clone(), Ok(hit));
continue;
}
}
pending.push(ip.clone());
}
let (mut pilot, joined) = match &self.0.cache {
Some(_) => self.0.flights.board(pending.iter().map(String::as_str)),
None => self.0.flights.board([]),
};
let mut send: Vec<String> = Vec::with_capacity(pending.len());
for ip in pending {
if joined.iter().any(|(joined_ip, _)| *joined_ip == ip) {
continue;
}
if let Some(cache) = self.0.cache.as_ref().filter(|_| pilot.leads(&ip)) {
if let Some(hit) = cache.get(&ip).await {
let hit = Ok(hit);
pilot.land(&ip, &hit);
answers.insert(ip, hit);
continue;
}
}
send.push(ip);
}
let chunks: Vec<Vec<String>> = send.chunks(BATCH_MAX).map(<[String]>::to_vec).collect();
{
let mut stream = futures_util::stream::iter(chunks)
.map(|chunk| async move { self.lookup_chunk(chunk, retries, timeout).await })
.buffer_unordered(concurrency);
while let Some(chunk_answers) = stream.next().await {
for (ip, answer) in chunk_answers {
pilot.land(&ip, &answer);
answers.insert(ip, answer);
}
}
}
drop(pilot);
for (ip, flight) in joined {
let answer = match flight.await {
Ok(landed) => restate(&landed),
Err(_) => {
let opts = LookupOptions { retries: Some(retries), timeout: Some(timeout) };
self.lookup_with(&ip, opts).await
}
};
answers.insert(ip, answer);
}
let mut uses: HashMap<&str, usize> = HashMap::with_capacity(wire.len());
for ip in &wire {
*uses.entry(ip.as_str()).or_default() += 1;
}
asked
.into_iter()
.zip(&wire)
.map(|(ip, wire)| {
let left = uses.get_mut(wire.as_str()).expect("every address is counted");
*left -= 1;
let answer = if *left == 0 {
answers.remove(wire).expect("every address is answered exactly once")
} else {
restate(answers.get(wire).expect("every address is answered"))
};
(ip, answer)
})
.collect()
}
async fn lookup_chunk(
&self,
chunk: Vec<String>,
retries: u32,
timeout: Duration,
) -> Vec<(String, Result<Lookup, Error>)> {
let request =
BatchLookupRequest { ips: Some(chunk.clone()), ..BatchLookupRequest::default() };
let answered: Result<BatchAnswer, Error> =
with_retry(retries, || self.0.transport.post_json("/batch", &request, timeout)).await;
let mut body = match answered {
Ok(body) => body,
Err(err) => return chunk.into_iter().map(|ip| (ip, Err(err.restated()))).collect(),
};
let mut out = Vec::with_capacity(chunk.len());
for ip in chunk {
let answer = if let Some(served) = body.results.remove(&ip) {
match serde_json::from_value::<LookupResponse>(served) {
Ok(served) => {
let lookup = Lookup::served(served);
if let Some(cache) = &self.0.cache {
cache.insert(ip.clone(), lookup.clone()).await;
}
Ok(lookup)
}
Err(err) => Err(unreadable_entry(&ip, &err)),
}
} else if let Some(failed) = body.errors.remove(&ip) {
match serde_json::from_value::<BatchLookupError>(failed) {
Ok(failed) => Err(Error::from_entry(
u16::try_from(failed.status).unwrap_or(500),
&failed.error,
)),
Err(err) => Err(unreadable_entry(&ip, &err)),
}
} else {
Err(Error::Api {
kind: ErrorKind::ServerError,
message: format!("the batch answer did not include {ip}"),
status: 200,
retry_after: None,
})
};
out.push((ip, answer));
}
out
}
pub async fn lookup_asn(&self, asn: u32) -> Result<AsnLookupResponse, Error> {
self.lookup_asn_with(asn, LookupOptions::new()).await
}
pub async fn lookup_asn_with(
&self,
asn: u32,
opts: LookupOptions,
) -> Result<AsnLookupResponse, Error> {
let timeout = self.call_timeout(opts.timeout)?;
let path = format!("/asn/{asn}");
with_retry(opts.retries.unwrap_or(self.0.retries), || {
self.0.transport.get_json(&path, &[], timeout)
})
.await
}
pub async fn lookup_domain(&self, domain: &str) -> Result<DomainLookupResponse, Error> {
self.lookup_domain_with(domain, LookupOptions::new()).await
}
pub async fn lookup_domain_with(
&self,
domain: &str,
opts: LookupOptions,
) -> Result<DomainLookupResponse, Error> {
let timeout = self.call_timeout(opts.timeout)?;
let path = format!("/domain/{}", encode_path_segment(domain));
with_retry(opts.retries.unwrap_or(self.0.retries), || {
self.0.transport.get_json(&path, &[], timeout)
})
.await
}
pub fn is_bogon(&self, ip: &str) -> bool {
is_bogon(ip)
}
pub fn database(&self) -> DatabaseApi<'_> {
DatabaseApi::new(self)
}
pub fn oauth(&self) -> OauthApi<'_> {
OauthApi::new(self)
}
pub(crate) fn transport(&self) -> &Transport {
&self.0.transport
}
pub(crate) fn retries(&self) -> u32 {
self.0.retries
}
pub(crate) fn timeout(&self) -> Duration {
self.0.timeout
}
pub(crate) fn call_timeout(&self, per_call: Option<Duration>) -> Result<Duration, Error> {
match per_call {
Some(timeout) if timeout.is_zero() => {
Err(Error::Config("timeout must be positive".to_owned()))
}
Some(timeout) => Ok(timeout),
None => Ok(self.0.timeout),
}
}
}
#[derive(Debug, Default)]
pub struct ClientBuilder {
api_key: Option<String>,
base_url: Option<String>,
cache_capacity: Option<u64>,
cache_ttl: Option<Duration>,
cache_off: bool,
concurrency: Option<usize>,
retries: Option<u32>,
timeout: Option<Duration>,
http_client: Option<reqwest::Client>,
}
impl ClientBuilder {
pub fn new() -> Self {
Self::default()
}
pub fn api_key(mut self, key: impl Into<String>) -> Self {
self.api_key = Some(key.into());
self
}
pub fn base_url(mut self, url: impl Into<String>) -> Self {
self.base_url = Some(url.into());
self
}
pub fn cache(mut self, capacity: u64, ttl: Duration) -> Self {
self.cache_capacity = Some(capacity);
self.cache_ttl = Some(ttl);
self.cache_off = false;
self
}
pub fn no_cache(mut self) -> Self {
self.cache_off = true;
self
}
pub fn concurrency(mut self, n: usize) -> Self {
self.concurrency = Some(n);
self
}
pub fn retries(mut self, n: u32) -> Self {
self.retries = Some(n);
self
}
pub fn timeout(mut self, timeout: Duration) -> Self {
self.timeout = Some(timeout);
self
}
pub fn http_client(mut self, client: reqwest::Client) -> Self {
self.http_client = Some(client);
self
}
pub fn build(self) -> Result<Client, Error> {
let base_url = self.base_url.unwrap_or_else(|| DEFAULT_BASE_URL.to_owned());
let parsed = reqwest::Url::parse(&base_url)
.map_err(|e| Error::Config(format!("base url {base_url:?}: {e}")))?;
if !parsed.has_host() {
return Err(Error::Config(format!("base url {base_url:?} needs a scheme and a host")));
}
let concurrency = self.concurrency.unwrap_or(DEFAULT_CONCURRENCY);
if concurrency == 0 {
return Err(Error::Config("concurrency must be at least 1".to_owned()));
}
let capacity = self.cache_capacity.unwrap_or(DEFAULT_CACHE_CAPACITY);
let ttl = self.cache_ttl.unwrap_or(DEFAULT_CACHE_TTL);
if capacity == 0 {
return Err(Error::Config("cache capacity must be at least 1".to_owned()));
}
if ttl.is_zero() {
return Err(Error::Config("cache ttl must be positive".to_owned()));
}
let timeout = self.timeout.unwrap_or(DEFAULT_TIMEOUT);
if timeout.is_zero() {
return Err(Error::Config("timeout must be positive".to_owned()));
}
let api_key = self.api_key.map(|key| key.trim().to_owned()).filter(|key| !key.is_empty());
if api_key.as_deref().is_some_and(|key| key.chars().any(char::is_control)) {
return Err(Error::Config("the API key holds a control character".to_owned()));
}
let http = match self.http_client {
Some(client) => client,
None => reqwest::Client::builder()
.connect_timeout(DEFAULT_CONNECT_TIMEOUT)
.read_timeout(DEFAULT_READ_TIMEOUT)
.user_agent(concat!("internetdata-rust/", env!("CARGO_PKG_VERSION")))
.redirect(reqwest::redirect::Policy::none())
.build()?,
};
let base_url = base_url.trim_end_matches('/').to_owned();
let oauth_base_url = authorization_server(&base_url);
Ok(Client(Arc::new(Inner {
transport: Transport::new(http, base_url, oauth_base_url, api_key),
cache: (!self.cache_off)
.then(|| Cache::builder().max_capacity(capacity).time_to_live(ttl).build()),
flights: Flights::default(),
concurrency,
retries: self.retries.unwrap_or(DEFAULT_RETRIES),
timeout,
})))
}
}
#[derive(Deserialize)]
struct BatchAnswer {
results: HashMap<String, serde_json::Value>,
errors: HashMap<String, serde_json::Value>,
}
fn unreadable_entry(ip: &str, err: &serde_json::Error) -> Error {
Error::Api {
kind: ErrorKind::ServerError,
message: format!("could not decode the batch answer for {ip}: {err}"),
status: 200,
retry_after: None,
}
}
#[derive(Debug, Clone, Default)]
pub struct LookupOptions {
retries: Option<u32>,
timeout: Option<Duration>,
}
impl LookupOptions {
pub fn new() -> Self {
Self::default()
}
pub fn retries(mut self, n: u32) -> Self {
self.retries = Some(n);
self
}
pub fn timeout(mut self, timeout: Duration) -> Self {
self.timeout = Some(timeout);
self
}
}
#[derive(Debug, Clone, Default)]
pub struct BatchOptions {
retries: Option<u32>,
concurrency: Option<usize>,
timeout: Option<Duration>,
}
impl BatchOptions {
pub fn new() -> Self {
Self::default()
}
pub fn retries(mut self, n: u32) -> Self {
self.retries = Some(n);
self
}
pub fn concurrency(mut self, n: usize) -> Self {
self.concurrency = Some(n);
self
}
pub fn timeout(mut self, timeout: Duration) -> Self {
self.timeout = Some(timeout);
self
}
}
pub(crate) async fn with_retry<T, E, F, Fut>(retries: u32, mut attempt: F) -> Result<T, E>
where
E: Retry,
F: FnMut() -> Fut,
Fut: Future<Output = Result<T, E>>,
{
let mut delay = RETRY_BASE_DELAY;
let mut remaining = retries;
loop {
match attempt().await {
Ok(value) => return Ok(value),
Err(err) if remaining == 0 || !err.retryable() => return Err(err),
Err(err) => {
let wait = err.retry_after().filter(|wait| *wait <= MAX_RETRY_AFTER);
tokio::time::sleep(wait.unwrap_or(delay)).await;
delay = delay.saturating_mul(2);
remaining -= 1;
}
}
}
}
pub(crate) trait Retry {
fn retryable(&self) -> bool;
fn retry_after(&self) -> Option<Duration>;
}
impl Retry for Error {
fn retryable(&self) -> bool {
Error::retryable(self)
}
fn retry_after(&self) -> Option<Duration> {
Error::retry_after(self)
}
}
impl Retry for OauthError {
fn retryable(&self) -> bool {
OauthError::retryable(self)
}
fn retry_after(&self) -> Option<Duration> {
OauthError::retry_after(self)
}
}
fn dedupe<I, S>(ips: I) -> Vec<String>
where
I: IntoIterator<Item = S>,
S: AsRef<str>,
{
let mut seen = std::collections::HashSet::new();
ips.into_iter().map(|ip| ip.as_ref().to_owned()).filter(|ip| seen.insert(ip.clone())).collect()
}