Skip to main content

fauna_api/marmot/
research.rs

1use std::ops::Deref;
2use std::sync::{Arc, Mutex, MutexGuard, PoisonError};
3use std::time::Duration;
4
5use chrono::{DateTime, Utc};
6use futures_util::stream::{self, BoxStream};
7use futures_util::{SinkExt, StreamExt};
8use serde::{Deserialize, Serialize};
9use serde_json::{Map, Value, json};
10use tokio::sync::{mpsc, oneshot};
11use tokio_tungstenite::tungstenite::Message;
12
13use super::scraper::{
14    Scraper, ScraperSettings, create_scraper, get_scraper, rebuild_scraper, remove_scraper,
15    update_scraper,
16};
17use super::workspace::PartialWorkspace;
18use crate::authentification::CredentialsOrApiKey;
19use crate::error::{Error, Result};
20use crate::groups::Group;
21use crate::http::{client, encode, ok, websocket};
22use crate::parrot::format::Format;
23use crate::parrot::generative::{Embedding, Text};
24
25pub use super::media::{DocumentMode, ProxyMode};
26pub use super::workspace::WorkspaceFile;
27
28/// `{ minimum, maximum }` in the TypeScript SDK: how many items a collection holds.
29#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
30pub struct ArrayBounds {
31    pub minimum: u64,
32    pub maximum: u64,
33}
34
35/// `string | string[]` in the TypeScript SDK.
36#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
37#[serde(untagged)]
38pub enum StringOrStrings {
39    String(String),
40    Strings(Vec<String>),
41}
42
43/// `Format | SchemaLeaf | Record<string, SchemaLeaf>` in the TypeScript SDK: a
44/// question, a single wrapped leaf, or named children in order.
45#[allow(clippy::large_enum_variant)]
46pub enum FormatOrSchemaLeafOrChildren {
47    Format(Format),
48    SchemaLeaf(Box<SchemaLeaf>),
49    Children(Vec<(String, SchemaLeaf)>),
50}
51
52/// One field of a dataset's schema. Cloning copies the questions and their answers.
53pub struct SchemaLeaf {
54    pub value: FormatOrSchemaLeafOrChildren,
55    pub prompt: Option<String>,
56    pub sources: Option<u64>,
57    pub optional: Option<bool>,
58    pub array: Option<ArrayBounds>,
59
60    pub max_iterations: Option<u64>,
61    pub filters: Option<StringOrStrings>,
62}
63
64impl Clone for SchemaLeaf {
65    fn clone(&self) -> Self {
66        deserialize_leaf(&serialize_leaf(self)).expect("a leaf reads its own JSON")
67    }
68}
69
70fn insert(fields: &mut Map<String, Value>, key: &str, value: Option<Value>) {
71    if let Some(value) = value {
72        fields.insert(key.to_string(), value);
73    }
74}
75
76fn serialize_leaf(leaf: &SchemaLeaf) -> Value {
77    let mut fields = Map::new();
78    insert(&mut fields, "prompt", leaf.prompt.clone().map(Value::from));
79    insert(&mut fields, "sources", leaf.sources.map(Value::from));
80    insert(&mut fields, "optional", leaf.optional.map(Value::from));
81    insert(&mut fields, "array", leaf.array.map(|array| json!(array)));
82    insert(
83        &mut fields,
84        "maxIterations",
85        leaf.max_iterations.map(Value::from),
86    );
87    insert(
88        &mut fields,
89        "filters",
90        leaf.filters.as_ref().map(|filters| json!(filters)),
91    );
92    let value = match &leaf.value {
93        FormatOrSchemaLeafOrChildren::Format(format) => json!({ "question": format.to_json() }),
94        FormatOrSchemaLeafOrChildren::SchemaLeaf(leaf) => json!({ "schema": serialize_leaf(leaf) }),
95        FormatOrSchemaLeafOrChildren::Children(children) => {
96            let children: Map<String, Value> = children
97                .iter()
98                .map(|(key, leaf)| (key.clone(), serialize_leaf(leaf)))
99                .collect();
100            json!({ "children": children })
101        }
102    };
103    fields.insert("value".to_string(), value);
104    Value::Object(fields)
105}
106
107fn deserialize_leaf(data: &Value) -> Result<SchemaLeaf> {
108    let array = match &data["array"] {
109        Value::Null => None,
110        array => {
111            let bound = |key: &str| {
112                array[key]
113                    .as_f64()
114                    .filter(|bound| bound.fract() == 0.0 && *bound >= 0.0)
115            };
116            let Some(minimum) = bound("minimum") else {
117                return Err(Error::Message(format!(
118                    "SchemaLeaf array.minimum must be a non-negative integer, got {}",
119                    array["minimum"]
120                )));
121            };
122            let Some(maximum) = bound("maximum") else {
123                return Err(Error::Message(format!(
124                    "SchemaLeaf array.maximum must be a non-negative integer, got {}",
125                    array["maximum"]
126                )));
127            };
128            if minimum > maximum {
129                return Err(Error::Message(format!(
130                    "SchemaLeaf array.minimum ({minimum}) cannot exceed array.maximum ({maximum})"
131                )));
132            }
133            Some(ArrayBounds {
134                minimum: minimum as u64,
135                maximum: maximum as u64,
136            })
137        }
138    };
139    let value = &data["value"];
140    let value = if value.get("question").is_some() {
141        FormatOrSchemaLeafOrChildren::Format(Format::from_json(&value["question"])?)
142    } else if let Some(children) = value.get("children") {
143        FormatOrSchemaLeafOrChildren::Children(
144            children
145                .as_object()
146                .into_iter()
147                .flatten()
148                .map(|(key, leaf)| Ok((key.clone(), deserialize_leaf(leaf)?)))
149                .collect::<Result<Vec<_>>>()?,
150        )
151    } else {
152        FormatOrSchemaLeafOrChildren::SchemaLeaf(Box::new(deserialize_leaf(&value["schema"])?))
153    };
154    Ok(SchemaLeaf {
155        value,
156        prompt: data["prompt"].as_str().map(str::to_string),
157        sources: data["sources"].as_u64(),
158        optional: data["optional"].as_bool(),
159        array,
160        max_iterations: data["maxIterations"].as_u64(),
161        filters: serde_json::from_value(data["filters"].clone()).ok(),
162    })
163}
164
165/// A dataset's schema: one JSON object, or CSV columns.
166#[derive(Clone)]
167#[allow(clippy::large_enum_variant)]
168pub enum Schema {
169    Json(SchemaJSON),
170    Csv(SchemaCSV),
171}
172
173impl Schema {
174    pub fn r#type(&self) -> &'static str {
175        match self {
176            Schema::Json(_) => "json",
177            Schema::Csv(_) => "csv",
178        }
179    }
180
181    pub fn serialize(&self) -> Value {
182        match self {
183            Schema::Json(schema) => schema.serialize(),
184            Schema::Csv(schema) => schema.serialize(),
185        }
186    }
187
188    pub fn deserialize(data: &Value) -> Result<Schema> {
189        match data["type"].as_str() {
190            Some("json") => Ok(Schema::Json(SchemaJSON::new(deserialize_leaf(
191                &data["schema"],
192            )?))),
193            Some("csv") => {
194                let columns = data["schema"]
195                    .as_object()
196                    .into_iter()
197                    .flatten()
198                    .map(|(key, leaf)| Ok((key.clone(), deserialize_leaf(leaf)?)))
199                    .collect::<Result<Vec<_>>>()?;
200                let bound = |key: &str| data[key].as_f64().unwrap_or(f64::NAN);
201                Ok(Schema::Csv(SchemaCSV::new(
202                    columns,
203                    bound("minimum"),
204                    bound("maximum"),
205                    data["prompt"].as_str(),
206                )?))
207            }
208            kind => Err(Error::Message(format!(
209                "Invalid schema type: {}",
210                kind.unwrap_or("undefined")
211            ))),
212        }
213    }
214}
215
216#[derive(Clone)]
217pub struct SchemaJSON {
218    pub schema: SchemaLeaf,
219}
220
221impl SchemaJSON {
222    pub fn new(schema: SchemaLeaf) -> SchemaJSON {
223        SchemaJSON { schema }
224    }
225
226    pub fn r#type(&self) -> &'static str {
227        "json"
228    }
229
230    pub fn serialize(&self) -> Value {
231        json!({ "type": "json", "schema": serialize_leaf(&self.schema) })
232    }
233}
234
235#[derive(Clone)]
236pub struct SchemaCSV {
237    // Columns are keyed by name (mirrors how JSON object scopes are keyed),
238    // so the key becomes the output column header and extraction scope path.
239    pub schema: Vec<(String, SchemaLeaf)>,
240    pub minimum: u64,
241    pub maximum: u64,
242    pub prompt: String,
243}
244
245impl SchemaCSV {
246    /// Every column must be a question; `minimum` and `maximum` are row counts.
247    /// `prompt` defaults to empty.
248    pub fn new(
249        schema: Vec<(String, SchemaLeaf)>,
250        minimum: f64,
251        maximum: f64,
252        prompt: Option<&str>,
253    ) -> Result<SchemaCSV> {
254        if schema
255            .iter()
256            .any(|(_, leaf)| !matches!(leaf.value, FormatOrSchemaLeafOrChildren::Format(_)))
257        {
258            return Err(Error::Message(
259                "Nested schema are not supported for CSV".to_string(),
260            ));
261        }
262        let whole = |count: f64| count.fract() == 0.0 && count >= 0.0;
263        if !whole(minimum) {
264            return Err(Error::Message(format!(
265                "SchemaCSV minimum must be a non-negative integer, got {minimum}"
266            )));
267        }
268        if !whole(maximum) {
269            return Err(Error::Message(format!(
270                "SchemaCSV maximum must be a non-negative integer, got {maximum}"
271            )));
272        }
273        if minimum > maximum {
274            return Err(Error::Message(format!(
275                "SchemaCSV minimum ({minimum}) cannot exceed maximum ({maximum})"
276            )));
277        }
278        Ok(SchemaCSV {
279            schema,
280            minimum: minimum as u64,
281            maximum: maximum as u64,
282            prompt: prompt.unwrap_or_default().to_string(),
283        })
284    }
285
286    pub fn r#type(&self) -> &'static str {
287        "csv"
288    }
289
290    pub fn serialize(&self) -> Value {
291        let schema: Map<String, Value> = self
292            .schema
293            .iter()
294            .map(|(key, leaf)| (key.clone(), serialize_leaf(leaf)))
295            .collect();
296        json!({
297            "type": "csv",
298            "minimum": self.minimum,
299            "maximum": self.maximum,
300            "prompt": self.prompt,
301            "schema": schema,
302        })
303    }
304}
305
306fn format_signature(value: &FormatOrSchemaLeafOrChildren) -> String {
307    match value {
308        FormatOrSchemaLeafOrChildren::Format(format) => format!(
309            "format:{}",
310            format.to_json()["type"].as_str().unwrap_or("unknown")
311        ),
312        _ => "composite".to_string(),
313    }
314}
315
316fn set(out: &mut Vec<(String, String)>, path: String, signature: String) {
317    match out.iter_mut().find(|(known, _)| *known == path) {
318        Some(entry) => entry.1 = signature,
319        None => out.push((path, signature)),
320    }
321}
322
323fn flatten_leaf_paths(leaf: &SchemaLeaf, path: &str, out: &mut Vec<(String, String)>) {
324    if leaf.array.is_some() {
325        // A collection: descend into the inner leaf with an `[]` marker so
326        // every leaf below it is addressed per-element (e.g. "awards[].year").
327        if let FormatOrSchemaLeafOrChildren::SchemaLeaf(inner) = &leaf.value {
328            flatten_leaf_paths(inner, &format!("{path}[]"), out);
329        }
330        return;
331    }
332    match &leaf.value {
333        FormatOrSchemaLeafOrChildren::Format(_) => {
334            let path = if path.is_empty() { "value" } else { path };
335            set(out, path.to_string(), format_signature(&leaf.value));
336        }
337        // Transparent single-child wrapper (e.g. an array's inner leaf).
338        FormatOrSchemaLeafOrChildren::SchemaLeaf(inner) => flatten_leaf_paths(inner, path, out),
339        FormatOrSchemaLeafOrChildren::Children(children) => {
340            for (key, child) in children {
341                let child_path = if path.is_empty() {
342                    key.clone()
343                } else {
344                    format!("{path}.{key}")
345                };
346                flatten_leaf_paths(child, &child_path, out);
347            }
348        }
349    }
350}
351
352/// Flatten a schema to `leaf path → type signature`, in schema order.
353///   - CSV: each column key is a path, signature is its Format type.
354///   - JSON: object keys descend with `.`, array leaves append `[]`, Format
355///     leaves terminate. e.g. "name", "address.city", "awards[].year".
356pub fn schema_paths(schema: &Schema) -> Vec<(String, String)> {
357    let mut out = Vec::new();
358    match schema {
359        Schema::Csv(schema) => {
360            for (key, leaf) in &schema.schema {
361                set(&mut out, key.clone(), format_signature(&leaf.value));
362            }
363        }
364        Schema::Json(schema) => flatten_leaf_paths(&schema.schema, "", &mut out),
365    }
366    out
367}
368
369/// `{ added, dropped }` in the TypeScript SDK.
370#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
371pub struct SchemaDiff {
372    pub added: Vec<String>,
373    pub dropped: Vec<String>,
374}
375
376/// Diff two schema versions for a dataset migration. A path whose type
377/// signature changed shows up in BOTH lists (drop the old, add the new); we
378/// deliberately do not attempt rename/retype detection. Renames and tree-shape
379/// changes therefore fall out naturally as drop + add.
380pub fn diff_schema(baseline: &Schema, target: &Schema) -> SchemaDiff {
381    let a = schema_paths(baseline);
382    let b = schema_paths(target);
383    let lookup = |paths: &[(String, String)], path: &str| {
384        paths
385            .iter()
386            .find(|(known, _)| known == path)
387            .map(|(_, signature)| signature.clone())
388    };
389    SchemaDiff {
390        added: b
391            .iter()
392            .filter(|(path, signature)| lookup(&a, path).as_ref() != Some(signature))
393            .map(|(path, _)| path.clone())
394            .collect(),
395        dropped: a
396            .iter()
397            .filter(|(path, signature)| lookup(&b, path).as_ref() != Some(signature))
398            .map(|(path, _)| path.clone())
399            .collect(),
400    }
401}
402
403/// When a research run stops, by its `type`; the typed ones carry their `max`.
404#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
405pub struct Termination {
406    pub r#type: String,
407    #[serde(flatten)]
408    pub fields: Map<String, Value>,
409}
410
411#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
412pub struct TokenTermination {
413    #[serde(flatten)]
414    pub termination: Termination,
415    pub max: f64,
416}
417
418impl Deref for TokenTermination {
419    type Target = Termination;
420    fn deref(&self) -> &Termination {
421        &self.termination
422    }
423}
424
425#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
426pub struct TimeTermination {
427    #[serde(flatten)]
428    pub termination: Termination,
429    pub max: f64,
430}
431
432impl Deref for TimeTermination {
433    type Target = Termination;
434    fn deref(&self) -> &Termination {
435        &self.termination
436    }
437}
438
439#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
440pub struct CostTermination {
441    #[serde(flatten)]
442    pub termination: Termination,
443    pub max: f64,
444}
445
446impl Deref for CostTermination {
447    type Target = Termination;
448    fn deref(&self) -> &Termination {
449        &self.termination
450    }
451}
452
453/// How `/data` writes rows: CSV with a header row, or a JSON array.
454#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Hash, Serialize, Deserialize)]
455#[serde(rename_all = "lowercase")]
456pub enum DataFormat {
457    Csv,
458    #[default]
459    Json,
460}
461
462#[derive(Debug, Clone, Default, PartialEq, Eq)]
463pub struct DataOptions {
464    pub format: Option<DataFormat>,
465    /// A snapshot id. The dataset's current data when absent.
466    pub snapshot: Option<String>,
467    /// Add each row's sources.
468    pub sources: Option<bool>,
469}
470
471/// A row pushed into a dataset, numbered by `seq`, which rises within the dataset.
472#[derive(Debug, Clone, PartialEq, Serialize)]
473pub struct FeedRow {
474    pub seq: u64,
475    pub value: Value,
476    pub sources: Value,
477    pub created: DateTime<Utc>,
478}
479
480impl FeedRow {
481    pub fn from_json(json: &Value) -> FeedRow {
482        FeedRow {
483            seq: json["seq"].as_f64().unwrap_or_default() as u64,
484            value: json["value"].clone(),
485            sources: json["sources"].clone(),
486            created: crate::tickets::parse_date(&json["created"]).unwrap_or_default(),
487        }
488    }
489}
490
491/// `{ index, reason }` in the TypeScript SDK: a pushed row left out, and why.
492#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
493pub struct InvalidRow {
494    pub index: u64,
495    pub reason: String,
496}
497
498/// What a push stored, and the rows it left out with why.
499#[derive(Debug, Clone, Default, PartialEq, Serialize)]
500pub struct PushResult {
501    pub added: u64,
502    pub first: Option<f64>,
503    pub last: Option<f64>,
504    pub invalid: Vec<InvalidRow>,
505}
506
507impl PushResult {
508    pub fn from_json(json: &Value) -> PushResult {
509        PushResult {
510            added: json["added"].as_f64().unwrap_or_default() as u64,
511            first: json["first"].as_f64(),
512            last: json["last"].as_f64(),
513            invalid: json["invalid"]
514                .as_array()
515                .into_iter()
516                .flatten()
517                .map(|row| InvalidRow {
518                    index: row["index"].as_f64().unwrap_or_default() as u64,
519                    reason: match &row["reason"] {
520                        Value::String(reason) => reason.clone(),
521                        reason => reason.to_string(),
522                    },
523                })
524                .collect(),
525        }
526    }
527}
528
529/// A push nothing in was let through: why, and each row's reason.
530#[derive(Debug, Clone, PartialEq, Eq)]
531pub struct PushRefused {
532    pub status: u16,
533    pub message: String,
534    pub invalid: Vec<InvalidRow>,
535}
536
537const FEED_PAGE: u64 = 500;
538
539fn dataset_data_url(dataset_id: &str, options: &DataOptions) -> String {
540    let mut query = format!(
541        "format={}",
542        match options.format.unwrap_or_default() {
543            DataFormat::Csv => "csv",
544            DataFormat::Json => "json",
545        }
546    );
547    if let Some(snapshot) = &options.snapshot {
548        query.push_str(&format!("&snapshot={}", encode(snapshot)));
549    }
550    if options.sources == Some(true) {
551        query.push_str("&sources=true");
552    }
553    format!(
554        "{}/marmot/dataset/{dataset_id}/data?{query}",
555        crate::get_url()
556    )
557}
558
559fn authorized(
560    request: reqwest::RequestBuilder,
561    authorization: &CredentialsOrApiKey,
562    group: Option<&Group>,
563) -> reqwest::RequestBuilder {
564    let request = request.header(
565        "Authorization",
566        match authorization {
567            CredentialsOrApiKey::Credentials(credentials) => format!("Token {}", credentials.token),
568            CredentialsOrApiKey::ApiKey(key) => format!("ApiKey {}", key.id),
569        },
570    );
571    match group {
572        Some(group) => request.header("Group", &group.id),
573        None => request,
574    }
575}
576
577async fn data_body(response: reqwest::Response, csv: bool) -> Result<Value> {
578    let response = ok(response).await?;
579    Ok(if csv {
580        Value::String(response.text().await?)
581    } else {
582        response.json().await?
583    })
584}
585
586/// What `Snapshot::download` gives: its rows as JSON or text, or with
587/// `assets` its archive's bytes.
588#[derive(Debug, Clone, PartialEq)]
589pub enum SnapshotData {
590    Json(Value),
591    Text(String),
592    Bytes(Vec<u8>),
593}
594
595/// The `options` argument of `Snapshot::data`.
596#[derive(Debug, Clone, Default)]
597pub struct SnapshotDataOptions {
598    pub format: Option<DataFormat>,
599    pub sources: Option<bool>,
600}
601
602#[derive(Default)]
603struct SnapshotCache {
604    schema: Option<Schema>,
605    data: Option<SnapshotData>,
606}
607
608#[derive(Clone)]
609pub struct Snapshot {
610    pub id: String,
611    pub created_at: DateTime<Utc>,
612
613    pub valid: bool,
614
615    authorization: CredentialsOrApiKey,
616    workspace_id: String,
617    dataset_id: String,
618    group: Option<Group>,
619    cache: Arc<Mutex<SnapshotCache>>,
620}
621
622impl Snapshot {
623    pub fn new(
624        authorization: &CredentialsOrApiKey,
625        workspace_id: &str,
626        dataset_id: &str,
627        id: &str,
628        created_at: DateTime<Utc>,
629        valid: bool,
630        group: Option<&Group>,
631    ) -> Snapshot {
632        Snapshot {
633            id: id.to_string(),
634            created_at,
635            valid,
636            authorization: authorization.clone(),
637            workspace_id: workspace_id.to_string(),
638            dataset_id: dataset_id.to_string(),
639            group: group.cloned(),
640            cache: Arc::new(Mutex::new(SnapshotCache::default())),
641        }
642    }
643
644    fn cache(&self) -> MutexGuard<'_, SnapshotCache> {
645        self.cache.lock().unwrap_or_else(PoisonError::into_inner)
646    }
647
648    fn url(&self, tail: &str) -> String {
649        format!(
650            "{}/marmot/snapshot/{}/{}/{}{tail}",
651            crate::get_url(),
652            self.workspace_id,
653            self.dataset_id,
654            self.id
655        )
656    }
657
658    async fn get(&self, tail: &str) -> Result<reqwest::Response> {
659        let response = authorized(
660            client().get(self.url(tail)),
661            &self.authorization,
662            self.group.as_ref(),
663        )
664        .send()
665        .await?;
666        ok(response).await
667    }
668
669    pub async fn download(&self, assets: bool, revalidate: bool) -> Result<SnapshotData> {
670        if !assets
671            && !revalidate
672            && let Some(data) = self.cache().data.clone()
673        {
674            return Ok(data);
675        }
676        let response = self.get(&format!("/?assets={assets}")).await?;
677        if assets {
678            return Ok(SnapshotData::Bytes(response.bytes().await?.to_vec()));
679        }
680        let json = response
681            .headers()
682            .get("content-type")
683            .and_then(|value| value.to_str().ok())
684            .is_some_and(|value| value.contains("application/json"));
685        let data = if json {
686            SnapshotData::Json(response.json().await?)
687        } else {
688            SnapshotData::Text(response.text().await?)
689        };
690        self.cache().data = Some(data.clone());
691        Ok(data)
692    }
693
694    /// This snapshot's rows as CSV text or, by default, a JSON array.
695    pub async fn data(&self, options: SnapshotDataOptions) -> Result<Value> {
696        let csv = options.format == Some(DataFormat::Csv);
697        let url = dataset_data_url(
698            &self.dataset_id,
699            &DataOptions {
700                format: options.format,
701                snapshot: Some(self.id.clone()),
702                sources: options.sources,
703            },
704        );
705        let response = authorized(client().get(url), &self.authorization, None)
706            .send()
707            .await?;
708        data_body(response, csv).await
709    }
710
711    pub async fn download_schema(&self, revalidate: bool) -> Result<Schema> {
712        if !revalidate && let Some(schema) = self.cache().schema.clone() {
713            return Ok(schema);
714        }
715        let json: Value = self.get("/schema").await?.json().await?;
716        let schema = Schema::deserialize(&json)?;
717        self.cache().schema = Some(schema.clone());
718        Ok(schema)
719    }
720
721    pub async fn download_log(&self) -> Result<Vec<u8>> {
722        Ok(self.get("/log").await?.bytes().await?.to_vec())
723    }
724
725    pub async fn download_embeddings(&self) -> Result<Vec<u8>> {
726        Ok(self.get("/embeddings").await?.bytes().await?.to_vec())
727    }
728
729    pub async fn delete(&self) -> Result<()> {
730        let response = authorized(
731            client().delete(self.url("")),
732            &self.authorization,
733            self.group.as_ref(),
734        )
735        .send()
736        .await?;
737        ok(response).await?;
738        Ok(())
739    }
740}
741
742/// A page a research run sent to the workspace's computer to be rendered or investigated.
743/// Watch it in the machine's room: `termite::os::MachineRoom::connect` with
744/// `MachineTarget::Id { id: machine }`.
745#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
746pub struct MachineFrame {
747    pub url: String,
748    pub machine: u64,
749}
750
751/// `'user' | 'tokens' | 'time' | 'cost'` in the TypeScript SDK.
752#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
753#[serde(rename_all = "lowercase")]
754pub enum CancelReason {
755    User,
756    Tokens,
757    Time,
758    Cost,
759}
760
761/// `{ value, source }` in the TypeScript SDK: an answer a field couldn't settle on.
762#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
763pub struct Candidate {
764    pub value: Value,
765    pub source: String,
766}
767
768/// A research run's live timeline. Every method defaults to a no-op, like the
769/// optional members of the TypeScript `ResearchCallbacks` object.
770pub trait ResearchCallbacks: Send + Sync {
771    fn on_connection(&self, _ongoing: bool) {}
772    fn on_query(&self, _query: &str) {}
773    fn on_source(&self, _url: &str, _title: &str, _content_type: &str) {}
774    fn on_answer(
775        &self,
776        _field: &str,
777        _value: &Value,
778        _converged: bool,
779        _answers_count: u64,
780        _sources: Option<&[String]>,
781    ) {
782    }
783    /// A row/entity was verified and committed at `scope`. `row` is the full
784    /// committed object (CSV: column→value; JSON: the scope's object), which is
785    /// what the live "building" preview tabulates.
786    fn on_nomination(
787        &self,
788        _scope: &str,
789        _row: &Value,
790        _committed_count: u64,
791        _sources: Option<&[String]>,
792    ) {
793    }
794    fn on_supervisor_query(&self, _question: &str) {}
795    fn on_supervisor_search(&self, _query: &str) {}
796    fn on_supervisor_answer(&self, _question: &str, _answer: &str) {}
797    fn on_supervisor_refused(&self, _question: &str) {}
798    /// A page went to the workspace's computer to be rendered or investigated.
799    fn on_machine(&self, _frame: &MachineFrame) {}
800    fn on_compress(&self) {}
801    fn on_info(&self, _message: &str) {}
802    fn on_done(&self, _snapshot: &Snapshot) {}
803    fn on_cancelled(&self, _reason: CancelReason) {}
804    fn on_convergence_failed(&self, _field: &str, _reason: &str, _candidates: &[Candidate]) {}
805    fn on_error(&self, _message: &str) {}
806}
807
808impl ResearchCallbacks for () {}
809
810#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
811pub struct ResearchOptions {
812    pub text: Option<Text>,
813    pub embedding: Option<Embedding>,
814}
815
816/// `rows: any[] | string` in the TypeScript SDK: row objects, or CSV text with a header row.
817#[derive(Debug, Clone, PartialEq)]
818pub enum RowsOrCsv {
819    Rows(Vec<Value>),
820    Csv(String),
821}
822
823/// The `options` argument of `add_rows`.
824#[derive(Debug, Clone, Default)]
825pub struct AddRowsOptions {
826    pub source: Option<String>,
827}
828
829/// `{ since, max }` in the TypeScript SDK: rows after `since`, at most `max`.
830#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
831pub struct FeedQuery {
832    pub since: u64,
833    pub max: u64,
834}
835
836fn text(value: &Value) -> String {
837    match value {
838        Value::Null => String::new(),
839        Value::String(text) => text.clone(),
840        value => value.to_string(),
841    }
842}
843
844#[derive(Clone)]
845pub struct Dataset {
846    pub id: String,
847    pub workspace_id: String,
848    pub name: String,
849    pub schema: Schema,
850    pub termination: Vec<Termination>,
851    pub content_types: Vec<String>,
852    pub proxy: ProxyMode,
853    pub document_modes: Vec<DocumentMode>,
854    snapshots: Arc<Mutex<Vec<Snapshot>>>,
855
856    authorization: CredentialsOrApiKey,
857    group: Option<Group>,
858    ws: Arc<Mutex<Option<mpsc::UnboundedSender<Message>>>>,
859}
860
861impl Dataset {
862    pub fn snapshots(&self) -> Vec<Snapshot> {
863        self.snapshots
864            .lock()
865            .unwrap_or_else(PoisonError::into_inner)
866            .clone()
867    }
868
869    pub async fn research(
870        &self,
871        callbacks: &dyn ResearchCallbacks,
872        options: ResearchOptions,
873    ) -> Result<()> {
874        self.run_socket(Vec::new(), callbacks, options, false)
875            .await
876            .map(|_| ())
877    }
878
879    /// Attach to an *already-running* research/migration session for this
880    /// dataset and replay its live timeline from the start, then keep streaming.
881    ///
882    /// Unlike `research`, this NEVER starts a new run: if nothing is in flight
883    /// it fails, so callers can handle it on load (e.g. to passively re-bind a
884    /// run after a restart). It otherwise resolves on the same terminal events
885    /// as `research` (done / cancelled / convergence_failed / error).
886    pub async fn reconnect(
887        &self,
888        callbacks: &dyn ResearchCallbacks,
889        options: ResearchOptions,
890    ) -> Result<()> {
891        self.run_socket(Vec::new(), callbacks, options, true)
892            .await
893            .map(|_| ())
894    }
895
896    async fn run_socket(
897        &self,
898        extra: Vec<(&str, String)>,
899        callbacks: &dyn ResearchCallbacks,
900        options: ResearchOptions,
901        reconnect: bool,
902    ) -> Result<Option<Snapshot>> {
903        let mut params = Map::new();
904        params.insert(
905            "Authorization".to_string(),
906            Value::from(match &self.authorization {
907                CredentialsOrApiKey::Credentials(credentials) => {
908                    format!("Token {}", credentials.token)
909                }
910                CredentialsOrApiKey::ApiKey(key) => format!("ApiKey {}", key.id),
911            }),
912        );
913        if let Some(group) = &self.group {
914            params.insert("Group".to_string(), Value::from(group.id.as_str()));
915        }
916        params.insert(
917            "Workspace".to_string(),
918            Value::from(self.workspace_id.as_str()),
919        );
920        params.insert("Dataset".to_string(), Value::from(self.id.as_str()));
921        for (key, value) in extra {
922            params.insert(key.to_string(), Value::from(value));
923        }
924        if let Some(model) = options.text {
925            params.insert("Text".to_string(), Value::from(model.as_str()));
926        }
927        if let Some(model) = options.embedding {
928            params.insert("Embedding".to_string(), Value::from(model.as_str()));
929        }
930        // Attach-only: bind to an in-flight run and replay its log, but never
931        // start a new one. The server answers `inactive` if nothing is running.
932        if reconnect {
933            params.insert("Attach".to_string(), Value::from("true"));
934        }
935        let url = format!(
936            "{}/marmot/research?authorization={}",
937            websocket(&crate::get_url()),
938            encode(&Value::Object(params).to_string())
939        );
940
941        let socket = match tokio_tungstenite::connect_async(url.as_str()).await {
942            Ok((socket, _)) => socket,
943            Err(_) if reconnect => {
944                return Err(Error::Message(
945                    "Failed to connect to research run".to_string(),
946                ));
947            }
948            Err(_) => {
949                callbacks.on_error("WebSocket connection failed");
950                return Ok(None);
951            }
952        };
953        let (mut write, mut read) = socket.split();
954        let (outbox, mut queue) = mpsc::unbounded_channel::<Message>();
955        tokio::spawn(async move {
956            while let Some(message) = queue.recv().await {
957                if write.send(message).await.is_err() {
958                    break;
959                }
960            }
961            let _ = write.close().await;
962        });
963        *self.ws.lock().unwrap_or_else(PoisonError::into_inner) = Some(outbox);
964        let close = || {
965            if let Some(outbox) = self
966                .ws
967                .lock()
968                .unwrap_or_else(PoisonError::into_inner)
969                .take()
970            {
971                let _ = outbox.send(Message::Close(None));
972            }
973        };
974
975        let mut attached = false;
976        let strings =
977            |value: &Value| -> Option<Vec<String>> { serde_json::from_value(value.clone()).ok() };
978        loop {
979            let packet = match read.next().await {
980                Some(Ok(Message::Text(raw))) => match serde_json::from_str::<Value>(raw.as_str()) {
981                    Ok(packet) => packet,
982                    Err(_) => continue,
983                },
984                Some(Ok(Message::Close(_))) | None => {
985                    self.ws
986                        .lock()
987                        .unwrap_or_else(PoisonError::into_inner)
988                        .take();
989                    if reconnect && !attached {
990                        return Err(Error::Message("No active research run".to_string()));
991                    }
992                    return Ok(None);
993                }
994                Some(Err(_)) => {
995                    self.ws
996                        .lock()
997                        .unwrap_or_else(PoisonError::into_inner)
998                        .take();
999                    // A connection-level failure before we attached to a run leaves a
1000                    // reconnect caller with nothing to resume.
1001                    if reconnect && !attached {
1002                        return Err(Error::Message(
1003                            "Failed to connect to research run".to_string(),
1004                        ));
1005                    }
1006                    callbacks.on_error("WebSocket connection failed");
1007                    return Ok(None);
1008                }
1009                Some(Ok(_)) => continue,
1010            };
1011            match packet["event"].as_str().unwrap_or_default() {
1012                "connection" => {
1013                    attached = packet["ongoing"] == true;
1014                    callbacks.on_connection(attached);
1015                }
1016                "inactive" => {
1017                    close();
1018                    return Err(Error::Message("No active research run".to_string()));
1019                }
1020                "query" => callbacks.on_query(&text(&packet["query"])),
1021                "source" => callbacks.on_source(
1022                    &text(&packet["url"]),
1023                    &text(&packet["title"]),
1024                    &text(&packet["contentType"]),
1025                ),
1026                "answer" => callbacks.on_answer(
1027                    &text(&packet["field"]),
1028                    &packet["value"],
1029                    packet["converged"] == true,
1030                    packet["answersCount"].as_u64().unwrap_or_default(),
1031                    strings(&packet["sources"]).as_deref(),
1032                ),
1033                "nomination_accepted" => callbacks.on_nomination(
1034                    &text(&packet["scope"]),
1035                    &packet["row"],
1036                    packet["committedCount"].as_u64().unwrap_or_default(),
1037                    strings(&packet["sources"]).as_deref(),
1038                ),
1039                "supervisor_query" => callbacks.on_supervisor_query(&text(&packet["question"])),
1040                "supervisor_search" => callbacks.on_supervisor_search(&text(&packet["query"])),
1041                "supervisor_answer" => callbacks
1042                    .on_supervisor_answer(&text(&packet["question"]), &text(&packet["answer"])),
1043                "supervisor_refused" => callbacks.on_supervisor_refused(&text(&packet["question"])),
1044                "machine" => {
1045                    if let Ok(frame) = serde_json::from_value::<MachineFrame>(packet.clone()) {
1046                        callbacks.on_machine(&frame);
1047                    }
1048                }
1049                "compress" => callbacks.on_compress(),
1050                "info" => callbacks.on_info(&text(&packet["message"])),
1051                "done" => {
1052                    let snapshot = Snapshot::new(
1053                        &self.authorization,
1054                        &self.workspace_id,
1055                        &self.id,
1056                        &text(&packet["snapshotId"]),
1057                        Utc::now(),
1058                        packet["valid"].as_bool().unwrap_or(true),
1059                        self.group.as_ref(),
1060                    );
1061                    self.snapshots
1062                        .lock()
1063                        .unwrap_or_else(PoisonError::into_inner)
1064                        .push(snapshot.clone());
1065                    callbacks.on_done(&snapshot);
1066                    close();
1067                    return Ok(Some(snapshot));
1068                }
1069                "cancelled" => {
1070                    callbacks.on_cancelled(
1071                        serde_json::from_value(packet["reason"].clone())
1072                            .unwrap_or(CancelReason::User),
1073                    );
1074                    close();
1075                    return Ok(None);
1076                }
1077                "convergence_failed" => {
1078                    let candidates: Vec<Candidate> =
1079                        serde_json::from_value(packet["candidates"].clone()).unwrap_or_default();
1080                    callbacks.on_convergence_failed(
1081                        &text(&packet["field"]),
1082                        &text(&packet["reason"]),
1083                        &candidates,
1084                    );
1085                    close();
1086                    return Ok(None);
1087                }
1088                "error" => {
1089                    callbacks.on_error(&text(&packet["message"]));
1090                    close();
1091                    return Ok(None);
1092                }
1093                _ => {}
1094            }
1095        }
1096    }
1097
1098    /// Send the cancel signal but do NOT close the socket. The server finishes
1099    /// the in-flight operation, breaks out of its loop, and emits a 'cancelled'
1100    /// event before closing the socket itself.
1101    pub async fn cancel(&self) -> Result<()> {
1102        if let Some(outbox) = self
1103            .ws
1104            .lock()
1105            .unwrap_or_else(PoisonError::into_inner)
1106            .as_ref()
1107        {
1108            let _ = outbox.send(Message::text(json!({ "event": "cancel" }).to_string()));
1109        }
1110        Ok(())
1111    }
1112
1113    async fn update(&self, field: &str, value: Value) -> Result<()> {
1114        let response = authorized(
1115            client().post(format!("{}/marmot/dataset/update", crate::get_url())),
1116            &self.authorization,
1117            self.group.as_ref(),
1118        )
1119        .json(&json!({ "workspaceId": self.workspace_id, "id": self.id, field: value }))
1120        .send()
1121        .await?;
1122        ok(response).await?;
1123        Ok(())
1124    }
1125
1126    pub async fn rename(&mut self, name: &str) -> Result<()> {
1127        self.update("name", Value::from(name)).await?;
1128        self.name = name.to_string();
1129        Ok(())
1130    }
1131
1132    /// Persist a new schema WITHOUT producing a snapshot. Use for edits that
1133    /// don't change the set of columns (prompt / sources / optional tweaks);
1134    /// for column add/drop use `migrate` instead so existing data is reconciled.
1135    pub async fn save_schema(&mut self, schema: Schema) -> Result<()> {
1136        self.update("schema", schema.serialize()).await?;
1137        self.schema = schema;
1138        Ok(())
1139    }
1140
1141    /// Migrate the dataset to a new schema, producing a fresh snapshot. Runs
1142    /// over the research socket so the caller sees the same live timeline.
1143    ///
1144    ///   - `seed_snapshot_id` = a snapshot id: the migration starts from that
1145    ///     snapshot's rows, drops removed columns, and backfills added columns
1146    ///     (researching only the new fields per row, reusing each row's stored
1147    ///     source URLs first).
1148    ///   - `seed_snapshot_id` = None: a from-scratch research run with the new
1149    ///     schema (no backfill).
1150    ///
1151    /// The diff between the seed snapshot's schema and `schema` is recomputed
1152    /// authoritatively on the server. On completion the dataset's schema is
1153    /// updated to `schema`.
1154    pub async fn migrate(
1155        &mut self,
1156        schema: Schema,
1157        seed_snapshot_id: Option<&str>,
1158        callbacks: &dyn ResearchCallbacks,
1159        options: ResearchOptions,
1160    ) -> Result<()> {
1161        let migrate =
1162            json!({ "seedSnapshotId": seed_snapshot_id, "schema": schema.serialize() }).to_string();
1163        let done = self
1164            .run_socket(vec![("Migrate", migrate)], callbacks, options, false)
1165            .await?;
1166        if done.is_some() {
1167            self.schema = schema;
1168        }
1169        Ok(())
1170    }
1171
1172    pub async fn set_termination(&mut self, termination: Vec<Termination>) -> Result<()> {
1173        self.update("termination", serde_json::to_value(&termination)?)
1174            .await?;
1175        self.termination = termination;
1176        Ok(())
1177    }
1178
1179    pub async fn set_content_types(&mut self, content_types: Vec<String>) -> Result<()> {
1180        self.update("contentTypes", json!(content_types)).await?;
1181        self.content_types = content_types;
1182        Ok(())
1183    }
1184
1185    pub async fn set_proxy(&mut self, proxy: ProxyMode) -> Result<()> {
1186        self.update("proxy", serde_json::to_value(proxy)?).await?;
1187        self.proxy = proxy;
1188        Ok(())
1189    }
1190
1191    pub async fn set_document_modes(&mut self, document_modes: Vec<DocumentMode>) -> Result<()> {
1192        self.update("documentModes", serde_json::to_value(&document_modes)?)
1193            .await?;
1194        self.document_modes = document_modes;
1195        Ok(())
1196    }
1197
1198    pub async fn remove(&self) -> Result<()> {
1199        let response = authorized(
1200            client().post(format!("{}/marmot/dataset/remove", crate::get_url())),
1201            &self.authorization,
1202            self.group.as_ref(),
1203        )
1204        .json(&json!({ "workspaceId": self.workspace_id, "id": self.id }))
1205        .send()
1206        .await?;
1207        ok(response).await?;
1208        Ok(())
1209    }
1210
1211    /// The dataset's data as CSV text or, by default, a JSON array: its live
1212    /// snapshot when new rows are stored into it, else its newest valid snapshot.
1213    pub async fn data(&self, options: DataOptions) -> Result<Value> {
1214        let csv = options.format == Some(DataFormat::Csv);
1215        let response = authorized(
1216            client().get(dataset_data_url(&self.id, &options)),
1217            &self.authorization,
1218            None,
1219        )
1220        .send()
1221        .await?;
1222        data_body(response, csv).await
1223    }
1224
1225    /// The `/data` address, for pipelines and curl. Calls to it still need your authorization.
1226    pub fn data_url(&self, options: &DataOptions) -> String {
1227        dataset_data_url(&self.id, options)
1228    }
1229
1230    /// Push rows into the dataset: objects shaped like its rows, or CSV text with a
1231    /// header row. Rows that don't fit come back in `invalid`; when none fit, it
1232    /// fails with `Error::PushRefused` listing them.
1233    pub async fn add_rows(&self, rows: RowsOrCsv, options: AddRowsOptions) -> Result<PushResult> {
1234        let mut url = format!("{}/marmot/dataset/{}/rows", crate::get_url(), self.id);
1235        if let Some(source) = options.source.filter(|source| !source.is_empty()) {
1236            url.push_str(&format!("?source={}", encode(&source)));
1237        }
1238        let request = authorized(client().post(url), &self.authorization, None);
1239        let request = match rows {
1240            RowsOrCsv::Csv(csv) => request.header("Content-Type", "text/csv").body(csv),
1241            RowsOrCsv::Rows(rows) => request.json(&rows),
1242        };
1243        let response = request.send().await?;
1244        if !response.status().is_success() {
1245            let status = response.status().as_u16();
1246            let text = response.text().await.unwrap_or_default();
1247            let body: Value = serde_json::from_str(&text).unwrap_or(Value::Null);
1248            return Err(Error::PushRefused(PushRefused {
1249                status,
1250                message: body["message"].as_str().map(str::to_string).unwrap_or(text),
1251                invalid: PushResult::from_json(&body).invalid,
1252            }));
1253        }
1254        Ok(PushResult::from_json(&response.json().await?))
1255    }
1256
1257    /// Rows pushed after `since`, oldest first, at most `max`. The next page starts at the last `seq` here.
1258    pub async fn updates(&self, query: FeedQuery) -> Result<Vec<FeedRow>> {
1259        let response = authorized(
1260            client().get(format!(
1261                "{}/marmot/dataset/{}/updates?since={}&max={}",
1262                crate::get_url(),
1263                self.id,
1264                query.since,
1265                query.max
1266            )),
1267            &self.authorization,
1268            None,
1269        )
1270        .send()
1271        .await?;
1272        let json: Value = ok(response).await?.json().await?;
1273        Ok(json
1274            .as_array()
1275            .into_iter()
1276            .flatten()
1277            .map(FeedRow::from_json)
1278            .collect())
1279    }
1280
1281    /// Every row pushed after `since`, then each new one as it lands. It opens the
1282    /// live socket first, catches up through `updates`, and skips rows it already
1283    /// gave, so nothing is missed or repeated. It reconnects by itself until the
1284    /// stream is dropped.
1285    pub fn live(&self, since: Option<u64>) -> BoxStream<'static, FeedRow> {
1286        let (rows, received) = mpsc::unbounded_channel::<FeedRow>();
1287        let dataset = self.clone();
1288        tokio::spawn(async move {
1289            let mut since = since.unwrap_or(0);
1290            while !rows.is_closed() {
1291                dataset.live_once(&mut since, &rows).await;
1292                if rows.is_closed() {
1293                    break;
1294                }
1295                tokio::time::sleep(Duration::from_secs(2)).await;
1296            }
1297        });
1298        stream::unfold(received, |mut received| async move {
1299            received.recv().await.map(|row| (row, received))
1300        })
1301        .boxed()
1302    }
1303
1304    async fn live_once(&self, since: &mut u64, rows: &mpsc::UnboundedSender<FeedRow>) {
1305        let params = json!({
1306            "Authorization": match &self.authorization {
1307                CredentialsOrApiKey::Credentials(credentials) => format!("Token {}", credentials.token),
1308                CredentialsOrApiKey::ApiKey(key) => format!("ApiKey {}", key.id),
1309            },
1310            "Dataset": self.id,
1311        });
1312        let url = format!(
1313            "{}/marmot/dataset/live?authorization={}",
1314            websocket(&crate::get_url()),
1315            encode(&params.to_string())
1316        );
1317        let Ok((socket, _)) = tokio_tungstenite::connect_async(url.as_str()).await else {
1318            return;
1319        };
1320        let (mut write, mut read) = socket.split();
1321        let (held, mut pending) = mpsc::unbounded_channel::<FeedRow>();
1322        let (ready, is_ready) = oneshot::channel::<()>();
1323        let reader = tokio::spawn(async move {
1324            let mut ready = Some(ready);
1325            while let Some(Ok(message)) = read.next().await {
1326                let Message::Text(raw) = message else {
1327                    if matches!(message, Message::Close(_)) {
1328                        break;
1329                    }
1330                    continue;
1331                };
1332                let Ok(packet) = serde_json::from_str::<Value>(raw.as_str()) else {
1333                    continue;
1334                };
1335                if packet["event"] == "ready" {
1336                    if let Some(ready) = ready.take() {
1337                        let _ = ready.send(());
1338                    }
1339                } else if packet["event"] == "rows" {
1340                    for row in packet["rows"].as_array().into_iter().flatten() {
1341                        let _ = held.send(FeedRow::from_json(row));
1342                    }
1343                }
1344            }
1345        });
1346
1347        if is_ready.await.is_ok() {
1348            loop {
1349                let Ok(page) = self
1350                    .updates(FeedQuery {
1351                        since: *since,
1352                        max: FEED_PAGE,
1353                    })
1354                    .await
1355                else {
1356                    break;
1357                };
1358                let count = page.len() as u64;
1359                for row in page {
1360                    if row.seq <= *since {
1361                        continue;
1362                    }
1363                    *since = row.seq;
1364                    if rows.send(row).is_err() {
1365                        reader.abort();
1366                        let _ = write.close().await;
1367                        return;
1368                    }
1369                }
1370                if count < FEED_PAGE {
1371                    break;
1372                }
1373            }
1374            loop {
1375                tokio::select! {
1376                    row = pending.recv() => {
1377                        let Some(row) = row else {
1378                            break;
1379                        };
1380                        if row.seq <= *since {
1381                            continue;
1382                        }
1383                        *since = row.seq;
1384                        if rows.send(row).is_err() {
1385                            break;
1386                        }
1387                    }
1388                    _ = rows.closed() => break,
1389                }
1390            }
1391        }
1392        reader.abort();
1393        let _ = write.close().await;
1394    }
1395
1396    /// Whether this dataset is kept updated, and how it's going. None when it isn't.
1397    pub async fn scraper(&self) -> Result<Option<Scraper>> {
1398        get_scraper(&self.authorization, &self.id).await
1399    }
1400
1401    /// Keep this dataset updated by its workspace's scraper agent. You become the person the agent runs as.
1402    pub async fn keep_updated(&self, settings: &ScraperSettings) -> Result<Scraper> {
1403        create_scraper(&self.authorization, &self.id, settings).await
1404    }
1405
1406    pub async fn update_scraper(&self, settings: &ScraperSettings) -> Result<Scraper> {
1407        update_scraper(&self.authorization, &self.id, settings).await
1408    }
1409
1410    /// Ask the agent to go over this dataset's scraper again, with a note for it.
1411    pub async fn rebuild_scraper(&self, note: Option<&str>) -> Result<()> {
1412        rebuild_scraper(&self.authorization, &self.id, note).await
1413    }
1414
1415    /// Stop keeping this dataset updated.
1416    pub async fn stop_keeping_updated(&self) -> Result<()> {
1417        remove_scraper(&self.authorization, &self.id).await
1418    }
1419}
1420
1421/// The `opts` argument of the calls that take only a group.
1422#[derive(Debug, Clone, Default)]
1423pub struct GroupOptions {
1424    pub group: Option<Group>,
1425}
1426
1427pub async fn list_datasets(
1428    authorization: &CredentialsOrApiKey,
1429    workspace_id: &str,
1430    opts: GroupOptions,
1431) -> Result<Vec<Dataset>> {
1432    let response = authorized(
1433        client().post(format!("{}/marmot/dataset/list", crate::get_url())),
1434        authorization,
1435        opts.group.as_ref(),
1436    )
1437    .json(&json!({ "workspaceId": workspace_id }))
1438    .send()
1439    .await?;
1440    let data: Value = ok(response).await?.json().await?;
1441    data.as_array()
1442        .into_iter()
1443        .flatten()
1444        .map(|d| {
1445            let id = text(&d["id"]);
1446            let snapshots = d["snapshots"]
1447                .as_array()
1448                .into_iter()
1449                .flatten()
1450                .map(|s| {
1451                    Snapshot::new(
1452                        authorization,
1453                        workspace_id,
1454                        &id,
1455                        &text(&s["id"]),
1456                        crate::tickets::parse_date(&s["createdAt"]).unwrap_or_default(),
1457                        s["valid"].as_bool().unwrap_or(true),
1458                        opts.group.as_ref(),
1459                    )
1460                })
1461                .collect();
1462            Ok(Dataset {
1463                workspace_id: workspace_id.to_string(),
1464                name: text(&d["name"]),
1465                schema: Schema::deserialize(&d["schema"])?,
1466                termination: serde_json::from_value(d["termination"].clone()).unwrap_or_default(),
1467                content_types: serde_json::from_value(d["contentTypes"].clone())
1468                    .unwrap_or_default(),
1469                proxy: serde_json::from_value(d["proxy"].clone()).unwrap_or(ProxyMode::Datacenter),
1470                document_modes: serde_json::from_value(d["documentModes"].clone())
1471                    .unwrap_or_else(|_| vec![DocumentMode::Extraction]),
1472                snapshots: Arc::new(Mutex::new(snapshots)),
1473                authorization: authorization.clone(),
1474                group: opts.group.clone(),
1475                ws: Arc::new(Mutex::new(None)),
1476                id,
1477            })
1478        })
1479        .collect()
1480}
1481
1482/// The `options` argument of `create_dataset`.
1483#[derive(Clone)]
1484pub struct CreateDatasetOptions {
1485    pub name: String,
1486    pub schema: Schema,
1487    pub termination: Option<Vec<Termination>>,
1488    pub content_types: Option<Vec<String>>,
1489    pub proxy: Option<ProxyMode>,
1490    pub document_modes: Option<Vec<DocumentMode>>,
1491}
1492
1493pub async fn create_dataset(
1494    authorization: &CredentialsOrApiKey,
1495    workspace_id: &str,
1496    options: &CreateDatasetOptions,
1497    opts: GroupOptions,
1498) -> Result<String> {
1499    let response = authorized(
1500        client().post(format!("{}/marmot/dataset/create", crate::get_url())),
1501        authorization,
1502        opts.group.as_ref(),
1503    )
1504    .json(&json!({
1505        "workspaceId": workspace_id,
1506        "name": options.name,
1507        "schema": options.schema.serialize(),
1508        "termination": options.termination.clone().unwrap_or_default(),
1509        "contentTypes": options.content_types.clone().unwrap_or_default(),
1510        "proxy": options.proxy.unwrap_or(ProxyMode::Datacenter),
1511        "documentModes": options.document_modes.clone().unwrap_or_else(|| vec![DocumentMode::Extraction]),
1512    }))
1513    .send()
1514    .await?;
1515    Ok(ok(response).await?.text().await?)
1516}
1517
1518#[derive(Debug, Clone)]
1519pub struct Workspace {
1520    pub id: String,
1521    pub name: String,
1522    pub description: String,
1523    pub created_at: i64,
1524    pub datasets: u64,
1525    pub files: u64,
1526
1527    authorization: CredentialsOrApiKey,
1528    group: Option<Group>,
1529}
1530
1531impl Workspace {
1532    fn opts(&self) -> GroupOptions {
1533        GroupOptions {
1534            group: self.group.clone(),
1535        }
1536    }
1537
1538    pub async fn list_datasets(&self) -> Result<Vec<Dataset>> {
1539        list_datasets(&self.authorization, &self.id, self.opts()).await
1540    }
1541
1542    pub async fn create_dataset(&self, options: &CreateDatasetOptions) -> Result<String> {
1543        create_dataset(&self.authorization, &self.id, options, self.opts()).await
1544    }
1545
1546    pub async fn rename(&mut self, name: &str) -> Result<()> {
1547        update_workspace(
1548            &self.authorization,
1549            &self.id,
1550            &PartialWorkspace {
1551                name: Some(name.to_string()),
1552                ..Default::default()
1553            },
1554            self.opts(),
1555        )
1556        .await?;
1557        self.name = name.to_string();
1558        Ok(())
1559    }
1560
1561    pub async fn set_description(&mut self, description: &str) -> Result<()> {
1562        update_workspace(
1563            &self.authorization,
1564            &self.id,
1565            &PartialWorkspace {
1566                description: Some(description.to_string()),
1567                ..Default::default()
1568            },
1569            self.opts(),
1570        )
1571        .await?;
1572        self.description = description.to_string();
1573        Ok(())
1574    }
1575
1576    async fn post(&self, route: &str, body: Value) -> Result<reqwest::Response> {
1577        let response = authorized(
1578            client().post(format!("{}{route}", crate::get_url())),
1579            &self.authorization,
1580            self.group.as_ref(),
1581        )
1582        .json(&body)
1583        .send()
1584        .await?;
1585        ok(response).await
1586    }
1587
1588    pub async fn remove(&self) -> Result<()> {
1589        self.post("/marmot/workspace/remove", json!({ "id": self.id }))
1590            .await?;
1591        Ok(())
1592    }
1593
1594    pub async fn get_files(&self) -> Result<Vec<WorkspaceFile>> {
1595        Ok(get_workspace(&self.authorization, &self.id, self.opts())
1596            .await?
1597            .files)
1598    }
1599
1600    pub async fn upload_file(
1601        &self,
1602        name: &str,
1603        content_type: &str,
1604        data: &[u8],
1605    ) -> Result<WorkspaceFile> {
1606        use base64::Engine;
1607        let data = base64::engine::general_purpose::STANDARD.encode(data);
1608        Ok(self
1609            .post(
1610                "/marmot/workspace/file/upload",
1611                json!({ "workspaceId": self.id, "name": name, "contentType": content_type, "data": data }),
1612            )
1613            .await?
1614            .json()
1615            .await?)
1616    }
1617
1618    pub async fn download_file(&self, file: &WorkspaceFile) -> Result<Vec<u8>> {
1619        Ok(self
1620            .post(
1621                "/marmot/workspace/file/get",
1622                json!({ "key": file.key, "contentType": file.content_type, "workspaceId": self.id }),
1623            )
1624            .await?
1625            .bytes()
1626            .await?
1627            .to_vec())
1628    }
1629
1630    pub async fn remove_file(&self, key: &str) -> Result<()> {
1631        self.post(
1632            "/marmot/workspace/file/remove",
1633            json!({ "workspaceId": self.id, "key": key }),
1634        )
1635        .await?;
1636        Ok(())
1637    }
1638}
1639
1640pub async fn list_workspaces(
1641    authorization: &CredentialsOrApiKey,
1642    opts: GroupOptions,
1643) -> Result<Vec<Workspace>> {
1644    let response = authorized(
1645        client().post(format!("{}/marmot/workspace/list", crate::get_url())),
1646        authorization,
1647        opts.group.as_ref(),
1648    )
1649    .header("Content-Type", "application/json")
1650    .send()
1651    .await?;
1652    let data: Value = ok(response).await?.json().await?;
1653    Ok(data
1654        .as_array()
1655        .into_iter()
1656        .flatten()
1657        .map(|w| Workspace {
1658            id: text(&w["id"]),
1659            name: text(&w["name"]),
1660            description: text(&w["description"]),
1661            created_at: w["createdAt"].as_f64().unwrap_or_default() as i64,
1662            datasets: w["datasets"].as_f64().unwrap_or_default() as u64,
1663            files: w["files"].as_f64().unwrap_or_default() as u64,
1664            authorization: authorization.clone(),
1665            group: opts.group.clone(),
1666        })
1667        .collect())
1668}
1669
1670/// `{ id, name, description, createdAt, files }` in the TypeScript SDK.
1671#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
1672#[serde(rename_all = "camelCase")]
1673pub struct WorkspaceDetail {
1674    pub id: String,
1675    pub name: String,
1676    #[serde(default)]
1677    pub description: String,
1678    #[serde(default)]
1679    pub created_at: i64,
1680    #[serde(default)]
1681    pub files: Vec<WorkspaceFile>,
1682}
1683
1684pub async fn get_workspace(
1685    authorization: &CredentialsOrApiKey,
1686    id: &str,
1687    opts: GroupOptions,
1688) -> Result<WorkspaceDetail> {
1689    let response = authorized(
1690        client().post(format!("{}/marmot/workspace/get", crate::get_url())),
1691        authorization,
1692        opts.group.as_ref(),
1693    )
1694    .json(&json!({ "id": id }))
1695    .send()
1696    .await?;
1697    Ok(ok(response).await?.json().await?)
1698}
1699
1700/// `description` defaults to empty.
1701pub async fn create_workspace(
1702    authorization: &CredentialsOrApiKey,
1703    name: &str,
1704    description: Option<&str>,
1705    opts: GroupOptions,
1706) -> Result<String> {
1707    let response = authorized(
1708        client().post(format!("{}/marmot/workspace/create", crate::get_url())),
1709        authorization,
1710        opts.group.as_ref(),
1711    )
1712    .json(&json!({ "name": name, "description": description.unwrap_or_default() }))
1713    .send()
1714    .await?;
1715    Ok(ok(response).await?.text().await?)
1716}
1717
1718pub async fn update_workspace(
1719    authorization: &CredentialsOrApiKey,
1720    id: &str,
1721    update: &PartialWorkspace,
1722    opts: GroupOptions,
1723) -> Result<()> {
1724    let mut body = Map::new();
1725    body.insert("id".to_string(), Value::from(id));
1726    if let Value::Object(fields) = serde_json::to_value(update)? {
1727        body.extend(fields);
1728    }
1729    let response = authorized(
1730        client().post(format!("{}/marmot/workspace/update", crate::get_url())),
1731        authorization,
1732        opts.group.as_ref(),
1733    )
1734    .json(&body)
1735    .send()
1736    .await?;
1737    ok(response).await?;
1738    Ok(())
1739}