use chrono::{DateTime, Utc};
use serde::{Deserialize, Serialize};
use serde_json::{Map, Value, json};
use crate::authentification::CredentialsOrApiKey;
use crate::billing::PageQuery;
use crate::error::Result;
use crate::http::{client, ok};
use crate::marmot::browser::{js_number, js_string};
use crate::parrot::generative::Text;
use crate::termite::os::Wake;
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
#[serde(rename_all = "lowercase")]
pub enum ScraperStatus {
Building,
Active,
Off,
}
#[derive(Debug, Clone, PartialEq, Serialize)]
#[serde(rename_all = "camelCase")]
pub struct Scraper {
pub dataset_id: String,
pub workspace_id: String,
pub status: ScraperStatus,
pub store: bool,
pub key: Vec<String>,
pub instructions: String,
pub live_snapshot_id: Option<String>,
pub last_rows_at: Option<DateTime<Utc>>,
pub rows_total: u64,
pub last_error: String,
pub created_at: Option<DateTime<Utc>>,
pub updated_at: Option<DateTime<Utc>>,
}
fn date(value: &Value) -> Option<DateTime<Utc>> {
match value {
Value::Null | Value::Bool(false) => None,
Value::String(text) if text.is_empty() => None,
Value::Number(number) if number.as_f64() == Some(0.0) => None,
value => crate::tickets::parse_date(value),
}
}
impl Scraper {
pub fn from_json(json: &Value) -> Scraper {
Scraper {
dataset_id: js_string(json.get("datasetId"), "undefined"),
workspace_id: js_string(json.get("workspaceId"), "undefined"),
status: match json["status"].as_str() {
Some("active") => ScraperStatus::Active,
Some("off") => ScraperStatus::Off,
_ => ScraperStatus::Building,
},
store: match &json["store"] {
Value::Bool(store) => *store,
Value::Null => false,
Value::Number(number) => number.as_f64().is_some_and(|number| number != 0.0),
Value::String(text) => !text.is_empty(),
_ => true,
},
key: json["key"]
.as_array()
.map(|key| key.iter().map(|field| js_string(Some(field), "")).collect())
.unwrap_or_default(),
instructions: js_string(json.get("instructions"), ""),
live_snapshot_id: json["liveSnapshotId"].as_str().map(str::to_string),
last_rows_at: date(&json["lastRowsAt"]),
rows_total: js_number(Some(json.get("rowsTotal").unwrap_or(&Value::from(0)))) as u64,
last_error: js_string(json.get("lastError"), ""),
created_at: date(&json["createdAt"]),
updated_at: date(&json["updatedAt"]),
}
}
}
#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
pub struct ScraperSettings {
#[serde(default, skip_serializing_if = "Option::is_none")]
pub store: Option<bool>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub key: Option<Vec<String>>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub instructions: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub text: Option<Text>,
}
async fn post(
authorization: &CredentialsOrApiKey,
route: &str,
body: Value,
) -> Result<reqwest::Response> {
let response = client()
.post(format!("{}{route}", crate::get_url()))
.header(
"Authorization",
match authorization {
CredentialsOrApiKey::Credentials(credentials) => {
format!("Token {}", credentials.token)
}
CredentialsOrApiKey::ApiKey(key) => format!("ApiKey {}", key.id),
},
)
.json(&body)
.send()
.await?;
ok(response).await
}
fn with_settings(dataset_id: &str, settings: &ScraperSettings) -> Result<Value> {
let mut body = Map::new();
body.insert("datasetId".to_string(), Value::from(dataset_id));
if let Value::Object(fields) = serde_json::to_value(settings)? {
body.extend(fields);
}
Ok(Value::Object(body))
}
pub async fn get_scraper(
authorization: &CredentialsOrApiKey,
dataset_id: &str,
) -> Result<Option<Scraper>> {
let response = post(
authorization,
"/marmot/dataset/scraper/get",
json!({ "datasetId": dataset_id }),
)
.await?;
let json: Value = response.json().await?;
Ok(if json.is_null() || json == false {
None
} else {
Some(Scraper::from_json(&json))
})
}
pub async fn create_scraper(
authorization: &CredentialsOrApiKey,
dataset_id: &str,
settings: &ScraperSettings,
) -> Result<Scraper> {
let response = post(
authorization,
"/marmot/dataset/scraper/create",
with_settings(dataset_id, settings)?,
)
.await?;
Ok(Scraper::from_json(&response.json().await?))
}
pub async fn update_scraper(
authorization: &CredentialsOrApiKey,
dataset_id: &str,
settings: &ScraperSettings,
) -> Result<Scraper> {
let response = post(
authorization,
"/marmot/dataset/scraper/update",
with_settings(dataset_id, settings)?,
)
.await?;
Ok(Scraper::from_json(&response.json().await?))
}
pub async fn rebuild_scraper(
authorization: &CredentialsOrApiKey,
dataset_id: &str,
note: Option<&str>,
) -> Result<()> {
let mut body = Map::new();
body.insert("datasetId".to_string(), Value::from(dataset_id));
if let Some(note) = note {
body.insert("note".to_string(), Value::from(note));
}
post(
authorization,
"/marmot/dataset/scraper/rebuild",
Value::Object(body),
)
.await?;
Ok(())
}
pub async fn remove_scraper(authorization: &CredentialsOrApiKey, dataset_id: &str) -> Result<()> {
post(
authorization,
"/marmot/dataset/scraper/remove",
json!({ "datasetId": dataset_id }),
)
.await?;
Ok(())
}
pub async fn list_wakes(
authorization: &CredentialsOrApiKey,
workspace_id: &str,
query: PageQuery,
) -> Result<Vec<Wake>> {
let response = post(
authorization,
"/marmot/workspace/wakes",
json!({ "workspaceId": workspace_id, "cursor": query.cursor, "max": query.max }),
)
.await?;
let json: Value = response.json().await?;
Ok(json
.as_array()
.into_iter()
.flatten()
.map(Wake::from_json)
.collect())
}