mod error;
#[allow(clippy::all)]
pub mod generated;
mod types;
pub use error::{Error, Result};
pub use types::*;
use serde::Deserialize;
use serde_json::{json, Value};
use std::sync::{Arc, Mutex};
use std::time::{Duration, Instant};
const DEFAULT_BASE_URL: &str = "https://api.hydrafetch.com";
const ACCEPTED: u16 = 202;
const CALL_JOB_TIMEOUT: Duration = Duration::from_secs(330);
const CALL_JOB_POLL: Duration = Duration::from_secs(2);
#[derive(Deserialize)]
struct ApiError {
code: Option<String>,
message: Option<String>,
}
#[derive(Deserialize)]
struct Meta {
#[serde(rename = "requestId")]
request_id: Option<String>,
}
#[derive(Deserialize)]
struct Envelope {
success: Option<bool>,
data: Option<Value>,
error: Option<ApiError>,
meta: Option<Meta>,
poll: Option<String>,
}
#[derive(Deserialize)]
struct CallJobError {
code: Option<String>,
message: Option<String>,
status: Option<u16>,
}
#[derive(Deserialize)]
struct CallJob {
#[serde(rename = "jobId")]
job_id: Option<String>,
status: String,
result: Option<Value>,
error: Option<CallJobError>,
}
pub struct ClientBuilder {
api_key: Option<String>,
base_url: String,
timeout: Duration,
max_retries: u32,
job_poll: Duration,
}
impl ClientBuilder {
pub fn api_key(mut self, key: impl Into<String>) -> Self {
self.api_key = Some(key.into());
self
}
pub fn base_url(mut self, url: impl Into<String>) -> Self {
self.base_url = url.into().trim_end_matches('/').to_string();
self
}
pub fn timeout(mut self, d: Duration) -> Self {
self.timeout = d;
self
}
pub fn max_retries(mut self, n: u32) -> Self {
self.max_retries = n;
self
}
pub fn job_poll_interval(mut self, d: Duration) -> Self {
self.job_poll = d;
self
}
pub fn build(self) -> Result<Client> {
let api_key = self
.api_key
.or_else(|| std::env::var("HYDRAFETCH_API_KEY").ok())
.filter(|k| !k.is_empty())
.ok_or(Error::MissingApiKey)?;
let http = reqwest::Client::builder()
.timeout(self.timeout)
.build()
.map_err(Error::Transport)?;
Ok(Client {
api_key,
base_url: self.base_url,
http,
max_retries: self.max_retries,
job_poll: self.job_poll,
last_request_id: Arc::new(Mutex::new(None)),
})
}
}
#[derive(Clone, Debug)]
pub struct Client {
api_key: String,
base_url: String,
http: reqwest::Client,
max_retries: u32,
job_poll: Duration,
last_request_id: Arc<Mutex<Option<String>>>,
}
impl Client {
pub fn builder() -> ClientBuilder {
ClientBuilder {
api_key: None,
base_url: DEFAULT_BASE_URL.to_string(),
timeout: Duration::from_secs(120),
max_retries: 2,
job_poll: CALL_JOB_POLL,
}
}
pub fn new(api_key: impl Into<String>) -> Result<Self> {
Self::builder().api_key(api_key).build()
}
pub fn from_env() -> Result<Self> {
Self::builder().build()
}
async fn request<T: for<'de> Deserialize<'de>>(
&self,
method: reqwest::Method,
path: &str,
body: Option<Value>,
query: &[(&str, String)],
) -> Result<T> {
let url = format!("{}{}", self.base_url, path);
let mut attempt = 0;
loop {
let mut req = self
.http
.request(method.clone(), &url)
.header("X-API-Key", &self.api_key)
.header("Accept", "application/json");
if !query.is_empty() {
req = req.query(query);
}
if let Some(ref payload) = body {
req = req.json(payload);
}
let response = match req.send().await {
Ok(r) => r,
Err(e) if e.is_timeout() => {
return Err(Error::Timeout(format!(
"request to {path} timed out; raise the timeout, or use the crawl and batch job endpoints for long work"
)))
}
Err(e) => {
if attempt >= self.max_retries {
return Err(Error::Transport(e));
}
attempt += 1;
backoff(attempt).await;
continue;
}
};
let status = response.status().as_u16();
if let Some(id) = response
.headers()
.get("x-request-id")
.and_then(|v| v.to_str().ok())
{
if let Ok(mut last) = self.last_request_id.lock() {
*last = Some(id.to_string());
}
}
let text = response.text().await.map_err(Error::Transport)?;
let envelope: Option<Envelope> = serde_json::from_str(&text).ok();
let failed = status >= 400
|| envelope
.as_ref()
.and_then(|e| e.success)
.map(|ok| !ok)
.unwrap_or(false);
if failed {
let (code, message, request_id) = match envelope {
Some(env) => (
env.error.as_ref().and_then(|e| e.code.clone()),
env.error.as_ref().and_then(|e| e.message.clone()),
env.meta.and_then(|m| m.request_id),
),
None => (None, None, None),
};
let err = Error::Api {
code: code.unwrap_or_else(|| format!("HTTP_{status}")),
message: message.unwrap_or_else(|| text.chars().take(300).collect()),
status,
request_id,
};
if !err.is_retryable() || attempt >= self.max_retries {
return Err(err);
}
attempt += 1;
backoff(attempt).await;
continue;
}
if status == ACCEPTED {
if let Some(poll) = envelope.as_ref().and_then(|e| e.poll.clone()) {
let payload = self.follow_call_job(&poll).await?;
return Ok(serde_json::from_value(payload)?);
}
}
let payload = match envelope.and_then(|e| e.data) {
Some(data) => data,
None => serde_json::from_str(&text)?,
};
return Ok(serde_json::from_value(payload)?);
}
}
async fn follow_call_job(&self, poll: &str) -> Result<Value> {
let url = format!("{}{}", self.base_url, poll);
let started = Instant::now();
loop {
tokio::time::sleep(self.job_poll).await;
let response = self
.http
.get(&url)
.header("X-API-Key", &self.api_key)
.header("Accept", "application/json")
.send()
.await
.map_err(Error::Transport)?;
let status = response.status().as_u16();
let text = response.text().await.map_err(Error::Transport)?;
if status >= 400 {
return Err(Error::Api {
code: format!("HTTP_{status}"),
message: text.chars().take(300).collect(),
status,
request_id: None,
});
}
let job: CallJob = serde_json::from_str(&text)?;
match job.status.as_str() {
"completed" => {
let result = job.result.unwrap_or(Value::Null);
return Ok(match result.get("data") {
Some(data) => data.clone(),
None => result,
});
}
"failed" => {
let err = job.error.unwrap_or(CallJobError {
code: None,
message: None,
status: None,
});
return Err(Error::Api {
code: err.code.unwrap_or_else(|| "JOB_FAILED".to_string()),
message: err.message.unwrap_or_else(|| "The job failed.".to_string()),
status: err.status.unwrap_or(500),
request_id: job.job_id,
});
}
_ => {}
}
if started.elapsed() + self.job_poll > CALL_JOB_TIMEOUT {
return Err(Error::Timeout(format!(
"the job did not finish in time; poll {poll} for its result"
)));
}
}
}
fn merge(base: Value, extra: Option<impl serde::Serialize>) -> Value {
let mut body = base;
if let Some(options) = extra {
if let Ok(Value::Object(fields)) = serde_json::to_value(options) {
if let Value::Object(ref mut target) = body {
for (k, v) in fields {
target.insert(k, v);
}
}
}
}
body
}
pub fn last_request_id(&self) -> Option<String> {
self.last_request_id.lock().ok().and_then(|id| id.clone())
}
pub async fn rate_result(&self, input: RateResultInput) -> Result<RateResultResponse> {
if input.request_id.is_none() && input.url.is_none() {
return Err(Error::Api {
code: "VALIDATION_ERROR".to_string(),
message: "pass request_id or url to say which result you are rating".to_string(),
status: 400,
request_id: None,
});
}
let body = serde_json::to_value(input)?;
self.request(reqwest::Method::POST, "/v1/web/feedback", Some(body), &[])
.await
}
pub async fn scrape(&self, url: &str, options: Option<ScrapeOptions>) -> Result<ScrapeResult> {
let body = Self::merge(json!({ "url": url }), options);
self.request(reqwest::Method::POST, "/v1/web/scrape", Some(body), &[])
.await
}
pub async fn markdown(&self, url: &str) -> Result<String> {
let page = self
.scrape(
url,
Some(ScrapeOptions {
formats: Some(vec![Format::Markdown]),
..Default::default()
}),
)
.await?;
Ok(page.markdown.unwrap_or_default())
}
pub async fn map(&self, url: &str, options: Option<MapOptions>) -> Result<MapResult> {
let body = Self::merge(json!({ "url": url }), options);
self.request(reqwest::Method::POST, "/v1/web/map", Some(body), &[])
.await
}
pub async fn search(
&self,
query: &str,
options: Option<SearchOptions>,
) -> Result<SearchResult> {
let body = Self::merge(json!({ "query": query }), options);
self.request(reqwest::Method::POST, "/v1/web/search", Some(body), &[])
.await
}
pub async fn news(&self, query: &str, options: Option<NewsOptions>) -> Result<NewsResult> {
let body = Self::merge(json!({ "query": query }), options);
self.request(reqwest::Method::POST, "/v1/web/news", Some(body), &[])
.await
}
pub async fn search_jobs(
&self,
query: &str,
options: Option<SearchJobsOptions>,
) -> Result<Vec<OpenJob>> {
let body = Self::merge(json!({ "query": query }), options);
self.request(
reqwest::Method::POST,
"/v1/web/index/jobs/search",
Some(body),
&[],
)
.await
}
pub async fn hiring_changes(
&self,
domains: &[&str],
since: &str,
options: Option<HiringChangesOptions>,
) -> Result<Vec<CompanyHiringChanges>> {
let body = Self::merge(json!({ "domains": domains, "since": since }), options);
self.request(
reqwest::Method::POST,
"/v1/web/index/jobs/changes",
Some(body),
&[],
)
.await
}
pub async fn extract(
&self,
urls: &[&str],
options: Option<ExtractOptions>,
) -> Result<ExtractResult> {
let body = Self::merge(json!({ "urls": urls }), options);
self.request(reqwest::Method::POST, "/v1/web/extract", Some(body), &[])
.await
}
pub async fn brand(&self, domain: &str) -> Result<BrandResult> {
self.request(
reqwest::Method::GET,
"/v1/web/brand",
None,
&[("domain", domain.to_string())],
)
.await
}
pub async fn logo(&self, domain: &str, options: Option<LogoOptions>) -> Result<LogoResult> {
let mut query = vec![("domain", domain.to_string())];
if let Some(opts) = options {
if let Some(theme) = opts.theme {
query.push(("theme", theme));
}
if let Some(kind) = opts.kind {
query.push(("type", kind));
}
}
self.request(reqwest::Method::GET, "/v1/web/brand/logo", None, &query)
.await
}
pub async fn styleguide(&self, domain_or_url: &str) -> Result<StyleguideResult> {
self.styleguide_with(domain_or_url, StyleguideOptions::default())
.await
}
pub async fn styleguide_with(
&self,
domain_or_url: &str,
options: StyleguideOptions,
) -> Result<StyleguideResult> {
let lower = domain_or_url.to_ascii_lowercase();
let key = if lower.starts_with("http://") || lower.starts_with("https://") {
"url"
} else {
"domain"
};
let mut query = vec![(key, domain_or_url.to_string())];
if let Some(max_age) = options.max_age {
query.push(("maxAge", max_age.to_string()));
}
self.request(reqwest::Method::GET, "/v1/web/styleguide", None, &query)
.await
}
pub async fn screenshot(
&self,
url: &str,
options: Option<ScreenshotOptions>,
) -> Result<ScreenshotResult> {
let body = Self::merge(json!({ "url": url }), options);
self.request(reqwest::Method::POST, "/v1/web/screenshot", Some(body), &[])
.await
}
pub async fn images(&self, url: &str) -> Result<ImagesResult> {
self.request(
reqwest::Method::POST,
"/v1/web/images",
Some(json!({ "url": url })),
&[],
)
.await
}
pub async fn start_crawl(&self, url: &str, options: Option<CrawlOptions>) -> Result<String> {
let body = Self::merge(json!({ "url": url }), options);
let payload: Value = self
.request(reqwest::Method::POST, "/v1/web/crawl", Some(body), &[])
.await?;
job_id(&payload, "crawlId")
}
pub async fn start_batch(
&self,
urls: &[&str],
options: Option<BatchOptions>,
) -> Result<String> {
let body = Self::merge(json!({ "urls": urls }), options);
let payload: Value = self
.request(reqwest::Method::POST, "/v1/web/batch", Some(body), &[])
.await?;
job_id(&payload, "batchId")
}
pub async fn crawl_status(&self, id: &str) -> Result<Job> {
self.request(
reqwest::Method::GET,
&format!("/v1/web/crawl/{id}"),
None,
&[],
)
.await
}
pub async fn batch_status(&self, id: &str) -> Result<Job> {
self.request(
reqwest::Method::GET,
&format!("/v1/web/batch/{id}"),
None,
&[],
)
.await
}
pub async fn crawl(
&self,
url: &str,
options: Option<CrawlOptions>,
wait: Option<WaitOptions>,
) -> Result<Job> {
let id = self.start_crawl(url, options).await?;
self.poll(JobKind::Crawl, &id, wait.unwrap_or_default())
.await
}
pub async fn batch(
&self,
urls: &[&str],
options: Option<BatchOptions>,
wait: Option<WaitOptions>,
) -> Result<Job> {
let id = self.start_batch(urls, options).await?;
self.poll(JobKind::Batch, &id, wait.unwrap_or_default())
.await
}
async fn poll(&self, kind: JobKind, id: &str, wait: WaitOptions) -> Result<Job> {
let deadline = Instant::now() + wait.timeout;
loop {
let job = match kind {
JobKind::Crawl => self.crawl_status(id).await?,
JobKind::Batch => self.batch_status(id).await?,
};
if job.is_done() {
return Ok(job);
}
if Instant::now() + wait.poll_interval > deadline {
return Err(Error::Timeout(format!(
"job did not finish within {:?}; it is still running server-side, so poll its status or pass a webhook",
wait.timeout
)));
}
tokio::time::sleep(wait.poll_interval).await;
}
}
}
#[derive(Clone, Copy)]
enum JobKind {
Crawl,
Batch,
}
fn job_id(payload: &Value, key: &str) -> Result<String> {
for candidate in [key, "id"] {
if let Some(value) = payload.get(candidate).and_then(|v| v.as_str()) {
if !value.is_empty() {
return Ok(value.to_string());
}
}
}
Err(Error::Api {
code: "NO_JOB_ID".into(),
message: "job accepted but returned no id".into(),
status: 502,
request_id: None,
})
}
async fn backoff(attempt: u32) {
tokio::time::sleep(Duration::from_millis(250 * 2u64.pow(attempt.min(5)))).await;
}