1use chrono::{DateTime, Utc};
6use serde::{Deserialize, Serialize};
7use serde_json::{Map, Value, json};
8
9use crate::authentification::CredentialsOrApiKey;
10use crate::billing::PageQuery;
11use crate::error::Result;
12use crate::http::{client, ok};
13use crate::marmot::browser::{js_number, js_string};
14use crate::parrot::generative::Text;
15use crate::termite::os::Wake;
16
17#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
18#[serde(rename_all = "lowercase")]
19pub enum ScraperStatus {
20 Building,
21 Active,
22 Off,
23}
24
25#[derive(Debug, Clone, PartialEq, Serialize)]
27#[serde(rename_all = "camelCase")]
28pub struct Scraper {
29 pub dataset_id: String,
30 pub workspace_id: String,
31 pub status: ScraperStatus,
33 pub store: bool,
35 pub key: Vec<String>,
37 pub instructions: String,
38 pub live_snapshot_id: Option<String>,
40 pub last_rows_at: Option<DateTime<Utc>>,
41 pub rows_total: u64,
42 pub last_error: String,
44 pub created_at: Option<DateTime<Utc>>,
45 pub updated_at: Option<DateTime<Utc>>,
46}
47
48fn date(value: &Value) -> Option<DateTime<Utc>> {
49 match value {
50 Value::Null | Value::Bool(false) => None,
51 Value::String(text) if text.is_empty() => None,
52 Value::Number(number) if number.as_f64() == Some(0.0) => None,
53 value => crate::tickets::parse_date(value),
54 }
55}
56
57impl Scraper {
58 pub fn from_json(json: &Value) -> Scraper {
59 Scraper {
60 dataset_id: js_string(json.get("datasetId"), "undefined"),
61 workspace_id: js_string(json.get("workspaceId"), "undefined"),
62 status: match json["status"].as_str() {
63 Some("active") => ScraperStatus::Active,
64 Some("off") => ScraperStatus::Off,
65 _ => ScraperStatus::Building,
66 },
67 store: match &json["store"] {
68 Value::Bool(store) => *store,
69 Value::Null => false,
70 Value::Number(number) => number.as_f64().is_some_and(|number| number != 0.0),
71 Value::String(text) => !text.is_empty(),
72 _ => true,
73 },
74 key: json["key"]
75 .as_array()
76 .map(|key| key.iter().map(|field| js_string(Some(field), "")).collect())
77 .unwrap_or_default(),
78 instructions: js_string(json.get("instructions"), ""),
79 live_snapshot_id: json["liveSnapshotId"].as_str().map(str::to_string),
80 last_rows_at: date(&json["lastRowsAt"]),
81 rows_total: js_number(Some(json.get("rowsTotal").unwrap_or(&Value::from(0)))) as u64,
82 last_error: js_string(json.get("lastError"), ""),
83 created_at: date(&json["createdAt"]),
84 updated_at: date(&json["updatedAt"]),
85 }
86 }
87}
88
89#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
90pub struct ScraperSettings {
91 #[serde(default, skip_serializing_if = "Option::is_none")]
92 pub store: Option<bool>,
93 #[serde(default, skip_serializing_if = "Option::is_none")]
94 pub key: Option<Vec<String>>,
95 #[serde(default, skip_serializing_if = "Option::is_none")]
97 pub instructions: Option<String>,
98 #[serde(default, skip_serializing_if = "Option::is_none")]
100 pub text: Option<Text>,
101}
102
103async fn post(
104 authorization: &CredentialsOrApiKey,
105 route: &str,
106 body: Value,
107) -> Result<reqwest::Response> {
108 let response = client()
109 .post(format!("{}{route}", crate::get_url()))
110 .header(
111 "Authorization",
112 match authorization {
113 CredentialsOrApiKey::Credentials(credentials) => {
114 format!("Token {}", credentials.token)
115 }
116 CredentialsOrApiKey::ApiKey(key) => format!("ApiKey {}", key.id),
117 },
118 )
119 .json(&body)
120 .send()
121 .await?;
122 ok(response).await
123}
124
125fn with_settings(dataset_id: &str, settings: &ScraperSettings) -> Result<Value> {
126 let mut body = Map::new();
127 body.insert("datasetId".to_string(), Value::from(dataset_id));
128 if let Value::Object(fields) = serde_json::to_value(settings)? {
129 body.extend(fields);
130 }
131 Ok(Value::Object(body))
132}
133
134pub async fn get_scraper(
136 authorization: &CredentialsOrApiKey,
137 dataset_id: &str,
138) -> Result<Option<Scraper>> {
139 let response = post(
140 authorization,
141 "/marmot/dataset/scraper/get",
142 json!({ "datasetId": dataset_id }),
143 )
144 .await?;
145 let json: Value = response.json().await?;
146 Ok(if json.is_null() || json == false {
147 None
148 } else {
149 Some(Scraper::from_json(&json))
150 })
151}
152
153pub async fn create_scraper(
157 authorization: &CredentialsOrApiKey,
158 dataset_id: &str,
159 settings: &ScraperSettings,
160) -> Result<Scraper> {
161 let response = post(
162 authorization,
163 "/marmot/dataset/scraper/create",
164 with_settings(dataset_id, settings)?,
165 )
166 .await?;
167 Ok(Scraper::from_json(&response.json().await?))
168}
169
170pub async fn update_scraper(
172 authorization: &CredentialsOrApiKey,
173 dataset_id: &str,
174 settings: &ScraperSettings,
175) -> Result<Scraper> {
176 let response = post(
177 authorization,
178 "/marmot/dataset/scraper/update",
179 with_settings(dataset_id, settings)?,
180 )
181 .await?;
182 Ok(Scraper::from_json(&response.json().await?))
183}
184
185pub async fn rebuild_scraper(
187 authorization: &CredentialsOrApiKey,
188 dataset_id: &str,
189 note: Option<&str>,
190) -> Result<()> {
191 let mut body = Map::new();
192 body.insert("datasetId".to_string(), Value::from(dataset_id));
193 if let Some(note) = note {
194 body.insert("note".to_string(), Value::from(note));
195 }
196 post(
197 authorization,
198 "/marmot/dataset/scraper/rebuild",
199 Value::Object(body),
200 )
201 .await?;
202 Ok(())
203}
204
205pub async fn remove_scraper(authorization: &CredentialsOrApiKey, dataset_id: &str) -> Result<()> {
207 post(
208 authorization,
209 "/marmot/dataset/scraper/remove",
210 json!({ "datasetId": dataset_id }),
211 )
212 .await?;
213 Ok(())
214}
215
216pub async fn list_wakes(
218 authorization: &CredentialsOrApiKey,
219 workspace_id: &str,
220 query: PageQuery,
221) -> Result<Vec<Wake>> {
222 let response = post(
223 authorization,
224 "/marmot/workspace/wakes",
225 json!({ "workspaceId": workspace_id, "cursor": query.cursor, "max": query.max }),
226 )
227 .await?;
228 let json: Value = response.json().await?;
229 Ok(json
230 .as_array()
231 .into_iter()
232 .flatten()
233 .map(Wake::from_json)
234 .collect())
235}