Skip to main content

fauna_api/marmot/
scraper.rs

1//! Keep updated: a workspace's one Termite repository, where its machine agent
2//! keeps a scraper per kept dataset running nonstop. Scrapers push rows into
3//! their dataset as they scrape; with `store` on, the rows merge into it.
4
5use chrono::{DateTime, Utc};
6use serde::{Deserialize, Serialize};
7use serde_json::{Map, Value, json};
8
9use crate::authentification::CredentialsOrApiKey;
10use crate::billing::PageQuery;
11use crate::error::Result;
12use crate::http::{client, ok};
13use crate::marmot::browser::{js_number, js_string};
14use crate::parrot::generative::Text;
15use crate::termite::os::Wake;
16
17#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
18#[serde(rename_all = "lowercase")]
19pub enum ScraperStatus {
20    Building,
21    Active,
22    Off,
23}
24
25/// A dataset kept updated by its workspace's scraper agent.
26#[derive(Debug, Clone, PartialEq, Serialize)]
27#[serde(rename_all = "camelCase")]
28pub struct Scraper {
29    pub dataset_id: String,
30    pub workspace_id: String,
31    /// `building` until its first rows arrive, then `active`. `off` when it stopped and needs turning on again.
32    pub status: ScraperStatus,
33    /// Pushed rows merge into the dataset's live snapshot, so its data includes them.
34    pub store: bool,
35    /// Fields a row is known by: a pushed row with the same key replaces the old one. None means rows append.
36    pub key: Vec<String>,
37    pub instructions: String,
38    /// The snapshot pushed rows merge into, once there is one.
39    pub live_snapshot_id: Option<String>,
40    pub last_rows_at: Option<DateTime<Utc>>,
41    pub rows_total: u64,
42    /// The last thing that went wrong, or ''.
43    pub last_error: String,
44    pub created_at: Option<DateTime<Utc>>,
45    pub updated_at: Option<DateTime<Utc>>,
46}
47
48fn date(value: &Value) -> Option<DateTime<Utc>> {
49    match value {
50        Value::Null | Value::Bool(false) => None,
51        Value::String(text) if text.is_empty() => None,
52        Value::Number(number) if number.as_f64() == Some(0.0) => None,
53        value => crate::tickets::parse_date(value),
54    }
55}
56
57impl Scraper {
58    pub fn from_json(json: &Value) -> Scraper {
59        Scraper {
60            dataset_id: js_string(json.get("datasetId"), "undefined"),
61            workspace_id: js_string(json.get("workspaceId"), "undefined"),
62            status: match json["status"].as_str() {
63                Some("active") => ScraperStatus::Active,
64                Some("off") => ScraperStatus::Off,
65                _ => ScraperStatus::Building,
66            },
67            store: match &json["store"] {
68                Value::Bool(store) => *store,
69                Value::Null => false,
70                Value::Number(number) => number.as_f64().is_some_and(|number| number != 0.0),
71                Value::String(text) => !text.is_empty(),
72                _ => true,
73            },
74            key: json["key"]
75                .as_array()
76                .map(|key| key.iter().map(|field| js_string(Some(field), "")).collect())
77                .unwrap_or_default(),
78            instructions: js_string(json.get("instructions"), ""),
79            live_snapshot_id: json["liveSnapshotId"].as_str().map(str::to_string),
80            last_rows_at: date(&json["lastRowsAt"]),
81            rows_total: js_number(Some(json.get("rowsTotal").unwrap_or(&Value::from(0)))) as u64,
82            last_error: js_string(json.get("lastError"), ""),
83            created_at: date(&json["createdAt"]),
84            updated_at: date(&json["updatedAt"]),
85        }
86    }
87}
88
89#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
90pub struct ScraperSettings {
91    #[serde(default, skip_serializing_if = "Option::is_none")]
92    pub store: Option<bool>,
93    #[serde(default, skip_serializing_if = "Option::is_none")]
94    pub key: Option<Vec<String>>,
95    /// What the agent should know about this dataset.
96    #[serde(default, skip_serializing_if = "Option::is_none")]
97    pub instructions: Option<String>,
98    /// The model the workspace's agent thinks with.
99    #[serde(default, skip_serializing_if = "Option::is_none")]
100    pub text: Option<Text>,
101}
102
103async fn post(
104    authorization: &CredentialsOrApiKey,
105    route: &str,
106    body: Value,
107) -> Result<reqwest::Response> {
108    let response = client()
109        .post(format!("{}{route}", crate::get_url()))
110        .header(
111            "Authorization",
112            match authorization {
113                CredentialsOrApiKey::Credentials(credentials) => {
114                    format!("Token {}", credentials.token)
115                }
116                CredentialsOrApiKey::ApiKey(key) => format!("ApiKey {}", key.id),
117            },
118        )
119        .json(&body)
120        .send()
121        .await?;
122    ok(response).await
123}
124
125fn with_settings(dataset_id: &str, settings: &ScraperSettings) -> Result<Value> {
126    let mut body = Map::new();
127    body.insert("datasetId".to_string(), Value::from(dataset_id));
128    if let Value::Object(fields) = serde_json::to_value(settings)? {
129        body.extend(fields);
130    }
131    Ok(Value::Object(body))
132}
133
134/// Whether a dataset is kept updated, and how it's going. None when it isn't.
135pub async fn get_scraper(
136    authorization: &CredentialsOrApiKey,
137    dataset_id: &str,
138) -> Result<Option<Scraper>> {
139    let response = post(
140        authorization,
141        "/marmot/dataset/scraper/get",
142        json!({ "datasetId": dataset_id }),
143    )
144    .await?;
145    let json: Value = response.json().await?;
146    Ok(if json.is_null() || json == false {
147        None
148    } else {
149        Some(Scraper::from_json(&json))
150    })
151}
152
153/// Keep a dataset updated. The workspace's machine agent writes its scraper, and
154/// the workspace's machine runs it nonstop; the workspace's owner pays for both.
155/// You become the person the agent runs as.
156pub async fn create_scraper(
157    authorization: &CredentialsOrApiKey,
158    dataset_id: &str,
159    settings: &ScraperSettings,
160) -> Result<Scraper> {
161    let response = post(
162        authorization,
163        "/marmot/dataset/scraper/create",
164        with_settings(dataset_id, settings)?,
165    )
166    .await?;
167    Ok(Scraper::from_json(&response.json().await?))
168}
169
170/// Change a kept dataset's settings. Key and instruction changes go to the agent.
171pub async fn update_scraper(
172    authorization: &CredentialsOrApiKey,
173    dataset_id: &str,
174    settings: &ScraperSettings,
175) -> Result<Scraper> {
176    let response = post(
177        authorization,
178        "/marmot/dataset/scraper/update",
179        with_settings(dataset_id, settings)?,
180    )
181    .await?;
182    Ok(Scraper::from_json(&response.json().await?))
183}
184
185/// Ask the agent to go over a kept dataset's scraper again, with a note for it.
186pub async fn rebuild_scraper(
187    authorization: &CredentialsOrApiKey,
188    dataset_id: &str,
189    note: Option<&str>,
190) -> Result<()> {
191    let mut body = Map::new();
192    body.insert("datasetId".to_string(), Value::from(dataset_id));
193    if let Some(note) = note {
194        body.insert("note".to_string(), Value::from(note));
195    }
196    post(
197        authorization,
198        "/marmot/dataset/scraper/rebuild",
199        Value::Object(body),
200    )
201    .await?;
202    Ok(())
203}
204
205/// Stop keeping a dataset updated. Its scraper goes; the machine stops with the workspace's last one.
206pub async fn remove_scraper(authorization: &CredentialsOrApiKey, dataset_id: &str) -> Result<()> {
207    post(
208        authorization,
209        "/marmot/dataset/scraper/remove",
210        json!({ "datasetId": dataset_id }),
211    )
212    .await?;
213    Ok(())
214}
215
216/// The times a workspace's scraper agent woke, newest first.
217pub async fn list_wakes(
218    authorization: &CredentialsOrApiKey,
219    workspace_id: &str,
220    query: PageQuery,
221) -> Result<Vec<Wake>> {
222    let response = post(
223        authorization,
224        "/marmot/workspace/wakes",
225        json!({ "workspaceId": workspace_id, "cursor": query.cursor, "max": query.max }),
226    )
227    .await?;
228    let json: Value = response.json().await?;
229    Ok(json
230        .as_array()
231        .into_iter()
232        .flatten()
233        .map(Wake::from_json)
234        .collect())
235}