Skip to main content

fauna_api/marmot/
research.rs

1use std::ops::Deref;
2use std::sync::{Arc, Mutex, MutexGuard, PoisonError};
3use std::time::Duration;
4
5use chrono::{DateTime, Utc};
6use futures_util::stream::{self, BoxStream};
7use futures_util::{SinkExt, StreamExt};
8use serde::{Deserialize, Serialize};
9use serde_json::{Map, Value, json};
10use tokio::sync::{mpsc, oneshot};
11use tokio_tungstenite::tungstenite::Message;
12
13use super::scraper::{
14    Scraper, ScraperSettings, create_scraper, get_scraper, rebuild_scraper, remove_scraper,
15    update_scraper,
16};
17use super::workspace::PartialWorkspace;
18use crate::authentification::CredentialsOrApiKey;
19use crate::error::{Error, Result};
20use crate::groups::Group;
21use crate::http::{client, encode, ok, websocket};
22use crate::parrot::format::Format;
23use crate::parrot::generative::{Embedding, Text};
24use crate::shared::server::Presence;
25
26pub use super::browser::BrowserTool;
27pub use super::media::{DocumentMode, ProxyMode};
28pub use super::workspace::WorkspaceFile;
29
30/// `{ minimum, maximum }` in the TypeScript SDK: how many items a collection holds.
31#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
32pub struct ArrayBounds {
33    pub minimum: u64,
34    pub maximum: u64,
35}
36
37/// `string | string[]` in the TypeScript SDK.
38#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
39#[serde(untagged)]
40pub enum StringOrStrings {
41    String(String),
42    Strings(Vec<String>),
43}
44
45/// `Format | SchemaLeaf | Record<string, SchemaLeaf>` in the TypeScript SDK: a
46/// question, a single wrapped leaf, or named children in order.
47#[allow(clippy::large_enum_variant)]
48pub enum FormatOrSchemaLeafOrChildren {
49    Format(Format),
50    SchemaLeaf(Box<SchemaLeaf>),
51    Children(Vec<(String, SchemaLeaf)>),
52}
53
54/// One field of a dataset's schema. Cloning copies the questions and their answers.
55pub struct SchemaLeaf {
56    pub value: FormatOrSchemaLeafOrChildren,
57    pub prompt: Option<String>,
58    pub sources: Option<u64>,
59    pub optional: Option<bool>,
60    pub array: Option<ArrayBounds>,
61
62    pub max_iterations: Option<u64>,
63    pub filters: Option<StringOrStrings>,
64}
65
66impl Clone for SchemaLeaf {
67    fn clone(&self) -> Self {
68        deserialize_leaf(&serialize_leaf(self)).expect("a leaf reads its own JSON")
69    }
70}
71
72fn insert(fields: &mut Map<String, Value>, key: &str, value: Option<Value>) {
73    if let Some(value) = value {
74        fields.insert(key.to_string(), value);
75    }
76}
77
78fn serialize_leaf(leaf: &SchemaLeaf) -> Value {
79    let mut fields = Map::new();
80    insert(&mut fields, "prompt", leaf.prompt.clone().map(Value::from));
81    insert(&mut fields, "sources", leaf.sources.map(Value::from));
82    insert(&mut fields, "optional", leaf.optional.map(Value::from));
83    insert(&mut fields, "array", leaf.array.map(|array| json!(array)));
84    insert(
85        &mut fields,
86        "maxIterations",
87        leaf.max_iterations.map(Value::from),
88    );
89    insert(
90        &mut fields,
91        "filters",
92        leaf.filters.as_ref().map(|filters| json!(filters)),
93    );
94    let value = match &leaf.value {
95        FormatOrSchemaLeafOrChildren::Format(format) => json!({ "question": format.to_json() }),
96        FormatOrSchemaLeafOrChildren::SchemaLeaf(leaf) => json!({ "schema": serialize_leaf(leaf) }),
97        FormatOrSchemaLeafOrChildren::Children(children) => {
98            let children: Map<String, Value> = children
99                .iter()
100                .map(|(key, leaf)| (key.clone(), serialize_leaf(leaf)))
101                .collect();
102            json!({ "children": children })
103        }
104    };
105    fields.insert("value".to_string(), value);
106    Value::Object(fields)
107}
108
109fn deserialize_leaf(data: &Value) -> Result<SchemaLeaf> {
110    let array = match &data["array"] {
111        Value::Null => None,
112        array => {
113            let bound = |key: &str| {
114                array[key]
115                    .as_f64()
116                    .filter(|bound| bound.fract() == 0.0 && *bound >= 0.0)
117            };
118            let Some(minimum) = bound("minimum") else {
119                return Err(Error::Message(format!(
120                    "SchemaLeaf array.minimum must be a non-negative integer, got {}",
121                    array["minimum"]
122                )));
123            };
124            let Some(maximum) = bound("maximum") else {
125                return Err(Error::Message(format!(
126                    "SchemaLeaf array.maximum must be a non-negative integer, got {}",
127                    array["maximum"]
128                )));
129            };
130            if minimum > maximum {
131                return Err(Error::Message(format!(
132                    "SchemaLeaf array.minimum ({minimum}) cannot exceed array.maximum ({maximum})"
133                )));
134            }
135            Some(ArrayBounds {
136                minimum: minimum as u64,
137                maximum: maximum as u64,
138            })
139        }
140    };
141    let value = &data["value"];
142    let value = if value.get("question").is_some() {
143        FormatOrSchemaLeafOrChildren::Format(Format::from_json(&value["question"])?)
144    } else if let Some(children) = value.get("children") {
145        FormatOrSchemaLeafOrChildren::Children(
146            children
147                .as_object()
148                .into_iter()
149                .flatten()
150                .map(|(key, leaf)| Ok((key.clone(), deserialize_leaf(leaf)?)))
151                .collect::<Result<Vec<_>>>()?,
152        )
153    } else {
154        FormatOrSchemaLeafOrChildren::SchemaLeaf(Box::new(deserialize_leaf(&value["schema"])?))
155    };
156    Ok(SchemaLeaf {
157        value,
158        prompt: data["prompt"].as_str().map(str::to_string),
159        sources: data["sources"].as_u64(),
160        optional: data["optional"].as_bool(),
161        array,
162        max_iterations: data["maxIterations"].as_u64(),
163        filters: serde_json::from_value(data["filters"].clone()).ok(),
164    })
165}
166
167/// A dataset's schema: one JSON object, or CSV columns.
168#[derive(Clone)]
169#[allow(clippy::large_enum_variant)]
170pub enum Schema {
171    Json(SchemaJSON),
172    Csv(SchemaCSV),
173}
174
175impl Schema {
176    pub fn r#type(&self) -> &'static str {
177        match self {
178            Schema::Json(_) => "json",
179            Schema::Csv(_) => "csv",
180        }
181    }
182
183    pub fn serialize(&self) -> Value {
184        match self {
185            Schema::Json(schema) => schema.serialize(),
186            Schema::Csv(schema) => schema.serialize(),
187        }
188    }
189
190    pub fn deserialize(data: &Value) -> Result<Schema> {
191        match data["type"].as_str() {
192            Some("json") => Ok(Schema::Json(SchemaJSON::new(deserialize_leaf(
193                &data["schema"],
194            )?))),
195            Some("csv") => {
196                let columns = data["schema"]
197                    .as_object()
198                    .into_iter()
199                    .flatten()
200                    .map(|(key, leaf)| Ok((key.clone(), deserialize_leaf(leaf)?)))
201                    .collect::<Result<Vec<_>>>()?;
202                let bound = |key: &str| data[key].as_f64().unwrap_or(f64::NAN);
203                Ok(Schema::Csv(SchemaCSV::new(
204                    columns,
205                    bound("minimum"),
206                    bound("maximum"),
207                    data["prompt"].as_str(),
208                )?))
209            }
210            kind => Err(Error::Message(format!(
211                "Invalid schema type: {}",
212                kind.unwrap_or("undefined")
213            ))),
214        }
215    }
216}
217
218#[derive(Clone)]
219pub struct SchemaJSON {
220    pub schema: SchemaLeaf,
221}
222
223impl SchemaJSON {
224    pub fn new(schema: SchemaLeaf) -> SchemaJSON {
225        SchemaJSON { schema }
226    }
227
228    pub fn r#type(&self) -> &'static str {
229        "json"
230    }
231
232    pub fn serialize(&self) -> Value {
233        json!({ "type": "json", "schema": serialize_leaf(&self.schema) })
234    }
235}
236
237#[derive(Clone)]
238pub struct SchemaCSV {
239    // Columns are keyed by name (mirrors how JSON object scopes are keyed),
240    // so the key becomes the output column header and extraction scope path.
241    pub schema: Vec<(String, SchemaLeaf)>,
242    pub minimum: u64,
243    pub maximum: u64,
244    pub prompt: String,
245}
246
247impl SchemaCSV {
248    /// Every column must be a question; `minimum` and `maximum` are row counts.
249    /// `prompt` defaults to empty.
250    pub fn new(
251        schema: Vec<(String, SchemaLeaf)>,
252        minimum: f64,
253        maximum: f64,
254        prompt: Option<&str>,
255    ) -> Result<SchemaCSV> {
256        if schema
257            .iter()
258            .any(|(_, leaf)| !matches!(leaf.value, FormatOrSchemaLeafOrChildren::Format(_)))
259        {
260            return Err(Error::Message(
261                "Nested schema are not supported for CSV".to_string(),
262            ));
263        }
264        let whole = |count: f64| count.fract() == 0.0 && count >= 0.0;
265        if !whole(minimum) {
266            return Err(Error::Message(format!(
267                "SchemaCSV minimum must be a non-negative integer, got {minimum}"
268            )));
269        }
270        if !whole(maximum) {
271            return Err(Error::Message(format!(
272                "SchemaCSV maximum must be a non-negative integer, got {maximum}"
273            )));
274        }
275        if minimum > maximum {
276            return Err(Error::Message(format!(
277                "SchemaCSV minimum ({minimum}) cannot exceed maximum ({maximum})"
278            )));
279        }
280        Ok(SchemaCSV {
281            schema,
282            minimum: minimum as u64,
283            maximum: maximum as u64,
284            prompt: prompt.unwrap_or_default().to_string(),
285        })
286    }
287
288    pub fn r#type(&self) -> &'static str {
289        "csv"
290    }
291
292    pub fn serialize(&self) -> Value {
293        let schema: Map<String, Value> = self
294            .schema
295            .iter()
296            .map(|(key, leaf)| (key.clone(), serialize_leaf(leaf)))
297            .collect();
298        json!({
299            "type": "csv",
300            "minimum": self.minimum,
301            "maximum": self.maximum,
302            "prompt": self.prompt,
303            "schema": schema,
304        })
305    }
306}
307
308fn format_signature(value: &FormatOrSchemaLeafOrChildren) -> String {
309    match value {
310        FormatOrSchemaLeafOrChildren::Format(format) => format!(
311            "format:{}",
312            format.to_json()["type"].as_str().unwrap_or("unknown")
313        ),
314        _ => "composite".to_string(),
315    }
316}
317
318fn set(out: &mut Vec<(String, String)>, path: String, signature: String) {
319    match out.iter_mut().find(|(known, _)| *known == path) {
320        Some(entry) => entry.1 = signature,
321        None => out.push((path, signature)),
322    }
323}
324
325fn flatten_leaf_paths(leaf: &SchemaLeaf, path: &str, out: &mut Vec<(String, String)>) {
326    if leaf.array.is_some() {
327        // A collection: descend into the inner leaf with an `[]` marker so
328        // every leaf below it is addressed per-element (e.g. "awards[].year").
329        if let FormatOrSchemaLeafOrChildren::SchemaLeaf(inner) = &leaf.value {
330            flatten_leaf_paths(inner, &format!("{path}[]"), out);
331        }
332        return;
333    }
334    match &leaf.value {
335        FormatOrSchemaLeafOrChildren::Format(_) => {
336            let path = if path.is_empty() { "value" } else { path };
337            set(out, path.to_string(), format_signature(&leaf.value));
338        }
339        // Transparent single-child wrapper (e.g. an array's inner leaf).
340        FormatOrSchemaLeafOrChildren::SchemaLeaf(inner) => flatten_leaf_paths(inner, path, out),
341        FormatOrSchemaLeafOrChildren::Children(children) => {
342            for (key, child) in children {
343                let child_path = if path.is_empty() {
344                    key.clone()
345                } else {
346                    format!("{path}.{key}")
347                };
348                flatten_leaf_paths(child, &child_path, out);
349            }
350        }
351    }
352}
353
354/// Flatten a schema to `leaf path → type signature`, in schema order.
355///   - CSV: each column key is a path, signature is its Format type.
356///   - JSON: object keys descend with `.`, array leaves append `[]`, Format
357///     leaves terminate. e.g. "name", "address.city", "awards[].year".
358pub fn schema_paths(schema: &Schema) -> Vec<(String, String)> {
359    let mut out = Vec::new();
360    match schema {
361        Schema::Csv(schema) => {
362            for (key, leaf) in &schema.schema {
363                set(&mut out, key.clone(), format_signature(&leaf.value));
364            }
365        }
366        Schema::Json(schema) => flatten_leaf_paths(&schema.schema, "", &mut out),
367    }
368    out
369}
370
371/// `{ added, dropped }` in the TypeScript SDK.
372#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
373pub struct SchemaDiff {
374    pub added: Vec<String>,
375    pub dropped: Vec<String>,
376}
377
378/// Diff two schema versions for a dataset migration. A path whose type
379/// signature changed shows up in BOTH lists (drop the old, add the new); we
380/// deliberately do not attempt rename/retype detection. Renames and tree-shape
381/// changes therefore fall out naturally as drop + add.
382pub fn diff_schema(baseline: &Schema, target: &Schema) -> SchemaDiff {
383    let a = schema_paths(baseline);
384    let b = schema_paths(target);
385    let lookup = |paths: &[(String, String)], path: &str| {
386        paths
387            .iter()
388            .find(|(known, _)| known == path)
389            .map(|(_, signature)| signature.clone())
390    };
391    SchemaDiff {
392        added: b
393            .iter()
394            .filter(|(path, signature)| lookup(&a, path).as_ref() != Some(signature))
395            .map(|(path, _)| path.clone())
396            .collect(),
397        dropped: a
398            .iter()
399            .filter(|(path, signature)| lookup(&b, path).as_ref() != Some(signature))
400            .map(|(path, _)| path.clone())
401            .collect(),
402    }
403}
404
405/// When a research run stops, by its `type`; the typed ones carry their `max`.
406#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
407pub struct Termination {
408    pub r#type: String,
409    #[serde(flatten)]
410    pub fields: Map<String, Value>,
411}
412
413#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
414pub struct TokenTermination {
415    #[serde(flatten)]
416    pub termination: Termination,
417    pub max: f64,
418}
419
420impl Deref for TokenTermination {
421    type Target = Termination;
422    fn deref(&self) -> &Termination {
423        &self.termination
424    }
425}
426
427#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
428pub struct TimeTermination {
429    #[serde(flatten)]
430    pub termination: Termination,
431    pub max: f64,
432}
433
434impl Deref for TimeTermination {
435    type Target = Termination;
436    fn deref(&self) -> &Termination {
437        &self.termination
438    }
439}
440
441#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
442pub struct CostTermination {
443    #[serde(flatten)]
444    pub termination: Termination,
445    pub max: f64,
446}
447
448impl Deref for CostTermination {
449    type Target = Termination;
450    fn deref(&self) -> &Termination {
451        &self.termination
452    }
453}
454
455/// How `/data` writes rows: CSV with a header row, or a JSON array.
456#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Hash, Serialize, Deserialize)]
457#[serde(rename_all = "lowercase")]
458pub enum DataFormat {
459    Csv,
460    #[default]
461    Json,
462}
463
464#[derive(Debug, Clone, Default, PartialEq, Eq)]
465pub struct DataOptions {
466    pub format: Option<DataFormat>,
467    /// A snapshot id. The dataset's current data when absent.
468    pub snapshot: Option<String>,
469    /// Add each row's sources.
470    pub sources: Option<bool>,
471}
472
473/// A row pushed into a dataset, numbered by `seq`, which rises within the dataset.
474#[derive(Debug, Clone, PartialEq, Serialize)]
475pub struct FeedRow {
476    pub seq: u64,
477    pub value: Value,
478    pub sources: Value,
479    pub created: DateTime<Utc>,
480}
481
482impl FeedRow {
483    pub fn from_json(json: &Value) -> FeedRow {
484        FeedRow {
485            seq: json["seq"].as_f64().unwrap_or_default() as u64,
486            value: json["value"].clone(),
487            sources: json["sources"].clone(),
488            created: crate::tickets::parse_date(&json["created"]).unwrap_or_default(),
489        }
490    }
491}
492
493/// `{ index, reason }` in the TypeScript SDK: a pushed row left out, and why.
494#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
495pub struct InvalidRow {
496    pub index: u64,
497    pub reason: String,
498}
499
500/// What a push stored, and the rows it left out with why.
501#[derive(Debug, Clone, Default, PartialEq, Serialize)]
502pub struct PushResult {
503    pub added: u64,
504    pub first: Option<f64>,
505    pub last: Option<f64>,
506    pub invalid: Vec<InvalidRow>,
507}
508
509impl PushResult {
510    pub fn from_json(json: &Value) -> PushResult {
511        PushResult {
512            added: json["added"].as_f64().unwrap_or_default() as u64,
513            first: json["first"].as_f64(),
514            last: json["last"].as_f64(),
515            invalid: json["invalid"]
516                .as_array()
517                .into_iter()
518                .flatten()
519                .map(|row| InvalidRow {
520                    index: row["index"].as_f64().unwrap_or_default() as u64,
521                    reason: match &row["reason"] {
522                        Value::String(reason) => reason.clone(),
523                        reason => reason.to_string(),
524                    },
525                })
526                .collect(),
527        }
528    }
529}
530
531/// A push nothing in was let through: why, and each row's reason.
532#[derive(Debug, Clone, PartialEq, Eq)]
533pub struct PushRefused {
534    pub status: u16,
535    pub message: String,
536    pub invalid: Vec<InvalidRow>,
537}
538
539const FEED_PAGE: u64 = 500;
540
541fn dataset_data_url(dataset_id: &str, options: &DataOptions) -> String {
542    let mut query = format!(
543        "format={}",
544        match options.format.unwrap_or_default() {
545            DataFormat::Csv => "csv",
546            DataFormat::Json => "json",
547        }
548    );
549    if let Some(snapshot) = &options.snapshot {
550        query.push_str(&format!("&snapshot={}", encode(snapshot)));
551    }
552    if options.sources == Some(true) {
553        query.push_str("&sources=true");
554    }
555    format!(
556        "{}/marmot/dataset/{dataset_id}/data?{query}",
557        crate::get_url()
558    )
559}
560
561fn authorized(
562    request: reqwest::RequestBuilder,
563    authorization: &CredentialsOrApiKey,
564    group: Option<&Group>,
565) -> reqwest::RequestBuilder {
566    let request = request.header(
567        "Authorization",
568        match authorization {
569            CredentialsOrApiKey::Credentials(credentials) => format!("Token {}", credentials.token),
570            CredentialsOrApiKey::ApiKey(key) => format!("ApiKey {}", key.id),
571        },
572    );
573    match group {
574        Some(group) => request.header("Group", &group.id),
575        None => request,
576    }
577}
578
579async fn data_body(response: reqwest::Response, csv: bool) -> Result<Value> {
580    let response = ok(response).await?;
581    Ok(if csv {
582        Value::String(response.text().await?)
583    } else {
584        response.json().await?
585    })
586}
587
588/// What `Snapshot::download` gives: its rows as JSON or text, or with
589/// `assets` its archive's bytes.
590#[derive(Debug, Clone, PartialEq)]
591pub enum SnapshotData {
592    Json(Value),
593    Text(String),
594    Bytes(Vec<u8>),
595}
596
597/// The `options` argument of `Snapshot::data`.
598#[derive(Debug, Clone, Default)]
599pub struct SnapshotDataOptions {
600    pub format: Option<DataFormat>,
601    pub sources: Option<bool>,
602}
603
604#[derive(Default)]
605struct SnapshotCache {
606    schema: Option<Schema>,
607    data: Option<SnapshotData>,
608}
609
610#[derive(Clone)]
611pub struct Snapshot {
612    pub id: String,
613    pub created_at: DateTime<Utc>,
614
615    pub valid: bool,
616
617    authorization: CredentialsOrApiKey,
618    workspace_id: String,
619    dataset_id: String,
620    group: Option<Group>,
621    cache: Arc<Mutex<SnapshotCache>>,
622}
623
624impl Snapshot {
625    pub fn new(
626        authorization: &CredentialsOrApiKey,
627        workspace_id: &str,
628        dataset_id: &str,
629        id: &str,
630        created_at: DateTime<Utc>,
631        valid: bool,
632        group: Option<&Group>,
633    ) -> Snapshot {
634        Snapshot {
635            id: id.to_string(),
636            created_at,
637            valid,
638            authorization: authorization.clone(),
639            workspace_id: workspace_id.to_string(),
640            dataset_id: dataset_id.to_string(),
641            group: group.cloned(),
642            cache: Arc::new(Mutex::new(SnapshotCache::default())),
643        }
644    }
645
646    fn cache(&self) -> MutexGuard<'_, SnapshotCache> {
647        self.cache.lock().unwrap_or_else(PoisonError::into_inner)
648    }
649
650    fn url(&self, tail: &str) -> String {
651        format!(
652            "{}/marmot/snapshot/{}/{}/{}{tail}",
653            crate::get_url(),
654            self.workspace_id,
655            self.dataset_id,
656            self.id
657        )
658    }
659
660    async fn get(&self, tail: &str) -> Result<reqwest::Response> {
661        let response = authorized(
662            client().get(self.url(tail)),
663            &self.authorization,
664            self.group.as_ref(),
665        )
666        .send()
667        .await?;
668        ok(response).await
669    }
670
671    pub async fn download(&self, assets: bool, revalidate: bool) -> Result<SnapshotData> {
672        if !assets
673            && !revalidate
674            && let Some(data) = self.cache().data.clone()
675        {
676            return Ok(data);
677        }
678        let response = self.get(&format!("/?assets={assets}")).await?;
679        if assets {
680            return Ok(SnapshotData::Bytes(response.bytes().await?.to_vec()));
681        }
682        let json = response
683            .headers()
684            .get("content-type")
685            .and_then(|value| value.to_str().ok())
686            .is_some_and(|value| value.contains("application/json"));
687        let data = if json {
688            SnapshotData::Json(response.json().await?)
689        } else {
690            SnapshotData::Text(response.text().await?)
691        };
692        self.cache().data = Some(data.clone());
693        Ok(data)
694    }
695
696    /// This snapshot's rows as CSV text or, by default, a JSON array.
697    pub async fn data(&self, options: SnapshotDataOptions) -> Result<Value> {
698        let csv = options.format == Some(DataFormat::Csv);
699        let url = dataset_data_url(
700            &self.dataset_id,
701            &DataOptions {
702                format: options.format,
703                snapshot: Some(self.id.clone()),
704                sources: options.sources,
705            },
706        );
707        let response = authorized(client().get(url), &self.authorization, None)
708            .send()
709            .await?;
710        data_body(response, csv).await
711    }
712
713    pub async fn download_schema(&self, revalidate: bool) -> Result<Schema> {
714        if !revalidate && let Some(schema) = self.cache().schema.clone() {
715            return Ok(schema);
716        }
717        let json: Value = self.get("/schema").await?.json().await?;
718        let schema = Schema::deserialize(&json)?;
719        self.cache().schema = Some(schema.clone());
720        Ok(schema)
721    }
722
723    pub async fn download_log(&self) -> Result<Vec<u8>> {
724        Ok(self.get("/log").await?.bytes().await?.to_vec())
725    }
726
727    pub async fn download_embeddings(&self) -> Result<Vec<u8>> {
728        Ok(self.get("/embeddings").await?.bytes().await?.to_vec())
729    }
730
731    pub async fn delete(&self) -> Result<()> {
732        let response = authorized(
733            client().delete(self.url("")),
734            &self.authorization,
735            self.group.as_ref(),
736        )
737        .send()
738        .await?;
739        ok(response).await?;
740        Ok(())
741    }
742}
743
744/// `'render' | 'virtualized'` in the TypeScript SDK: how a research page is driven.
745#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
746#[serde(rename_all = "lowercase")]
747pub enum BrowserMode {
748    Render,
749    Virtualized,
750}
751
752/// A browser room a research run opened for a page. Join with `code` to watch live HTML.
753#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
754pub struct BrowserFrame {
755    pub url: String,
756    pub reference: String,
757    pub code: String,
758    pub mode: BrowserMode,
759}
760
761/// `'user' | 'tokens' | 'time' | 'cost'` in the TypeScript SDK.
762#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
763#[serde(rename_all = "lowercase")]
764pub enum CancelReason {
765    User,
766    Tokens,
767    Time,
768    Cost,
769}
770
771/// `{ value, source }` in the TypeScript SDK: an answer a field couldn't settle on.
772#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
773pub struct Candidate {
774    pub value: Value,
775    pub source: String,
776}
777
778/// A research run's live timeline. Every method defaults to a no-op, like the
779/// optional members of the TypeScript `ResearchCallbacks` object.
780pub trait ResearchCallbacks: Send + Sync {
781    fn on_connection(&self, _ongoing: bool) {}
782    fn on_query(&self, _query: &str) {}
783    fn on_source(&self, _url: &str, _title: &str, _content_type: &str) {}
784    fn on_answer(
785        &self,
786        _field: &str,
787        _value: &Value,
788        _converged: bool,
789        _answers_count: u64,
790        _sources: Option<&[String]>,
791    ) {
792    }
793    /// A row/entity was verified and committed at `scope`. `row` is the full
794    /// committed object (CSV: column→value; JSON: the scope's object), which is
795    /// what the live "building" preview tabulates.
796    fn on_nomination(
797        &self,
798        _scope: &str,
799        _row: &Value,
800        _committed_count: u64,
801        _sources: Option<&[String]>,
802    ) {
803    }
804    fn on_supervisor_query(&self, _question: &str) {}
805    fn on_supervisor_search(&self, _query: &str) {}
806    fn on_supervisor_answer(&self, _question: &str, _answer: &str) {}
807    fn on_supervisor_refused(&self, _question: &str) {}
808    /// A browser room was opened for a page. Join with `code` to watch live HTML.
809    fn on_browser(&self, _frame: &BrowserFrame) {}
810    fn on_browser_step(&self, _url: &str, _step: u64, _action: &BrowserTool) {}
811    fn on_compress(&self) {}
812    fn on_info(&self, _message: &str) {}
813    fn on_done(&self, _snapshot: &Snapshot) {}
814    fn on_cancelled(&self, _reason: CancelReason) {}
815    fn on_convergence_failed(&self, _field: &str, _reason: &str, _candidates: &[Candidate]) {}
816    fn on_error(&self, _message: &str) {}
817}
818
819impl ResearchCallbacks for () {}
820
821#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
822pub struct ResearchOptions {
823    pub text: Option<Text>,
824    pub embedding: Option<Embedding>,
825}
826
827/// `rows: any[] | string` in the TypeScript SDK: row objects, or CSV text with a header row.
828#[derive(Debug, Clone, PartialEq)]
829pub enum RowsOrCsv {
830    Rows(Vec<Value>),
831    Csv(String),
832}
833
834/// The `options` argument of `add_rows`.
835#[derive(Debug, Clone, Default)]
836pub struct AddRowsOptions {
837    pub source: Option<String>,
838}
839
840/// `{ since, max }` in the TypeScript SDK: rows after `since`, at most `max`.
841#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
842pub struct FeedQuery {
843    pub since: u64,
844    pub max: u64,
845}
846
847fn text(value: &Value) -> String {
848    match value {
849        Value::Null => String::new(),
850        Value::String(text) => text.clone(),
851        value => value.to_string(),
852    }
853}
854
855#[derive(Clone)]
856pub struct Dataset {
857    pub id: String,
858    pub workspace_id: String,
859    pub name: String,
860    pub schema: Schema,
861    pub termination: Vec<Termination>,
862    pub content_types: Vec<String>,
863    pub proxy: ProxyMode,
864    pub document_modes: Vec<DocumentMode>,
865    snapshots: Arc<Mutex<Vec<Snapshot>>>,
866
867    authorization: CredentialsOrApiKey,
868    group: Option<Group>,
869    ws: Arc<Mutex<Option<mpsc::UnboundedSender<Message>>>>,
870}
871
872impl Dataset {
873    pub fn snapshots(&self) -> Vec<Snapshot> {
874        self.snapshots
875            .lock()
876            .unwrap_or_else(PoisonError::into_inner)
877            .clone()
878    }
879
880    pub async fn research(
881        &self,
882        callbacks: &dyn ResearchCallbacks,
883        options: ResearchOptions,
884    ) -> Result<()> {
885        self.run_socket(Vec::new(), callbacks, options, false)
886            .await
887            .map(|_| ())
888    }
889
890    /// Join a live BrowserRoom opened by a research run.
891    pub async fn join_browser(
892        &self,
893        access_code: &str,
894    ) -> Result<(super::browser::BrowserRoom, Presence)> {
895        super::browser::BrowserRoom::join(&self.authorization, access_code, Default::default())
896            .await
897    }
898
899    /// Attach to an *already-running* research/migration session for this
900    /// dataset and replay its live timeline from the start, then keep streaming.
901    ///
902    /// Unlike `research`, this NEVER starts a new run: if nothing is in flight
903    /// it fails, so callers can handle it on load (e.g. to passively re-bind a
904    /// run after a restart). It otherwise resolves on the same terminal events
905    /// as `research` (done / cancelled / convergence_failed / error).
906    pub async fn reconnect(
907        &self,
908        callbacks: &dyn ResearchCallbacks,
909        options: ResearchOptions,
910    ) -> Result<()> {
911        self.run_socket(Vec::new(), callbacks, options, true)
912            .await
913            .map(|_| ())
914    }
915
916    async fn run_socket(
917        &self,
918        extra: Vec<(&str, String)>,
919        callbacks: &dyn ResearchCallbacks,
920        options: ResearchOptions,
921        reconnect: bool,
922    ) -> Result<Option<Snapshot>> {
923        let mut params = Map::new();
924        params.insert(
925            "Authorization".to_string(),
926            Value::from(match &self.authorization {
927                CredentialsOrApiKey::Credentials(credentials) => {
928                    format!("Token {}", credentials.token)
929                }
930                CredentialsOrApiKey::ApiKey(key) => format!("ApiKey {}", key.id),
931            }),
932        );
933        if let Some(group) = &self.group {
934            params.insert("Group".to_string(), Value::from(group.id.as_str()));
935        }
936        params.insert(
937            "Workspace".to_string(),
938            Value::from(self.workspace_id.as_str()),
939        );
940        params.insert("Dataset".to_string(), Value::from(self.id.as_str()));
941        for (key, value) in extra {
942            params.insert(key.to_string(), Value::from(value));
943        }
944        if let Some(model) = options.text {
945            params.insert("Text".to_string(), Value::from(model.as_str()));
946        }
947        if let Some(model) = options.embedding {
948            params.insert("Embedding".to_string(), Value::from(model.as_str()));
949        }
950        // Attach-only: bind to an in-flight run and replay its log, but never
951        // start a new one. The server answers `inactive` if nothing is running.
952        if reconnect {
953            params.insert("Attach".to_string(), Value::from("true"));
954        }
955        let url = format!(
956            "{}/marmot/research?authorization={}",
957            websocket(&crate::get_url()),
958            encode(&Value::Object(params).to_string())
959        );
960
961        let socket = match tokio_tungstenite::connect_async(url.as_str()).await {
962            Ok((socket, _)) => socket,
963            Err(_) if reconnect => {
964                return Err(Error::Message(
965                    "Failed to connect to research run".to_string(),
966                ));
967            }
968            Err(_) => {
969                callbacks.on_error("WebSocket connection failed");
970                return Ok(None);
971            }
972        };
973        let (mut write, mut read) = socket.split();
974        let (outbox, mut queue) = mpsc::unbounded_channel::<Message>();
975        tokio::spawn(async move {
976            while let Some(message) = queue.recv().await {
977                if write.send(message).await.is_err() {
978                    break;
979                }
980            }
981            let _ = write.close().await;
982        });
983        *self.ws.lock().unwrap_or_else(PoisonError::into_inner) = Some(outbox);
984        let close = || {
985            if let Some(outbox) = self
986                .ws
987                .lock()
988                .unwrap_or_else(PoisonError::into_inner)
989                .take()
990            {
991                let _ = outbox.send(Message::Close(None));
992            }
993        };
994
995        let mut attached = false;
996        let strings =
997            |value: &Value| -> Option<Vec<String>> { serde_json::from_value(value.clone()).ok() };
998        loop {
999            let packet = match read.next().await {
1000                Some(Ok(Message::Text(raw))) => match serde_json::from_str::<Value>(raw.as_str()) {
1001                    Ok(packet) => packet,
1002                    Err(_) => continue,
1003                },
1004                Some(Ok(Message::Close(_))) | None => {
1005                    self.ws
1006                        .lock()
1007                        .unwrap_or_else(PoisonError::into_inner)
1008                        .take();
1009                    if reconnect && !attached {
1010                        return Err(Error::Message("No active research run".to_string()));
1011                    }
1012                    return Ok(None);
1013                }
1014                Some(Err(_)) => {
1015                    self.ws
1016                        .lock()
1017                        .unwrap_or_else(PoisonError::into_inner)
1018                        .take();
1019                    // A connection-level failure before we attached to a run leaves a
1020                    // reconnect caller with nothing to resume.
1021                    if reconnect && !attached {
1022                        return Err(Error::Message(
1023                            "Failed to connect to research run".to_string(),
1024                        ));
1025                    }
1026                    callbacks.on_error("WebSocket connection failed");
1027                    return Ok(None);
1028                }
1029                Some(Ok(_)) => continue,
1030            };
1031            match packet["event"].as_str().unwrap_or_default() {
1032                "connection" => {
1033                    attached = packet["ongoing"] == true;
1034                    callbacks.on_connection(attached);
1035                }
1036                "inactive" => {
1037                    close();
1038                    return Err(Error::Message("No active research run".to_string()));
1039                }
1040                "query" => callbacks.on_query(&text(&packet["query"])),
1041                "source" => callbacks.on_source(
1042                    &text(&packet["url"]),
1043                    &text(&packet["title"]),
1044                    &text(&packet["contentType"]),
1045                ),
1046                "answer" => callbacks.on_answer(
1047                    &text(&packet["field"]),
1048                    &packet["value"],
1049                    packet["converged"] == true,
1050                    packet["answersCount"].as_u64().unwrap_or_default(),
1051                    strings(&packet["sources"]).as_deref(),
1052                ),
1053                "nomination_accepted" => callbacks.on_nomination(
1054                    &text(&packet["scope"]),
1055                    &packet["row"],
1056                    packet["committedCount"].as_u64().unwrap_or_default(),
1057                    strings(&packet["sources"]).as_deref(),
1058                ),
1059                "supervisor_query" => callbacks.on_supervisor_query(&text(&packet["question"])),
1060                "supervisor_search" => callbacks.on_supervisor_search(&text(&packet["query"])),
1061                "supervisor_answer" => callbacks
1062                    .on_supervisor_answer(&text(&packet["question"]), &text(&packet["answer"])),
1063                "supervisor_refused" => callbacks.on_supervisor_refused(&text(&packet["question"])),
1064                "browser" => {
1065                    if let Ok(frame) = serde_json::from_value::<BrowserFrame>(packet.clone()) {
1066                        callbacks.on_browser(&frame);
1067                    }
1068                }
1069                "browser_step" => {
1070                    if let Ok(action) =
1071                        serde_json::from_value::<BrowserTool>(packet["action"].clone())
1072                    {
1073                        callbacks.on_browser_step(
1074                            &text(&packet["url"]),
1075                            packet["step"].as_u64().unwrap_or_default(),
1076                            &action,
1077                        );
1078                    }
1079                }
1080                "compress" => callbacks.on_compress(),
1081                "info" => callbacks.on_info(&text(&packet["message"])),
1082                "done" => {
1083                    let snapshot = Snapshot::new(
1084                        &self.authorization,
1085                        &self.workspace_id,
1086                        &self.id,
1087                        &text(&packet["snapshotId"]),
1088                        Utc::now(),
1089                        packet["valid"].as_bool().unwrap_or(true),
1090                        self.group.as_ref(),
1091                    );
1092                    self.snapshots
1093                        .lock()
1094                        .unwrap_or_else(PoisonError::into_inner)
1095                        .push(snapshot.clone());
1096                    callbacks.on_done(&snapshot);
1097                    close();
1098                    return Ok(Some(snapshot));
1099                }
1100                "cancelled" => {
1101                    callbacks.on_cancelled(
1102                        serde_json::from_value(packet["reason"].clone())
1103                            .unwrap_or(CancelReason::User),
1104                    );
1105                    close();
1106                    return Ok(None);
1107                }
1108                "convergence_failed" => {
1109                    let candidates: Vec<Candidate> =
1110                        serde_json::from_value(packet["candidates"].clone()).unwrap_or_default();
1111                    callbacks.on_convergence_failed(
1112                        &text(&packet["field"]),
1113                        &text(&packet["reason"]),
1114                        &candidates,
1115                    );
1116                    close();
1117                    return Ok(None);
1118                }
1119                "error" => {
1120                    callbacks.on_error(&text(&packet["message"]));
1121                    close();
1122                    return Ok(None);
1123                }
1124                _ => {}
1125            }
1126        }
1127    }
1128
1129    /// Send the cancel signal but do NOT close the socket. The server finishes
1130    /// the in-flight operation, breaks out of its loop, and emits a 'cancelled'
1131    /// event before closing the socket itself.
1132    pub async fn cancel(&self) -> Result<()> {
1133        if let Some(outbox) = self
1134            .ws
1135            .lock()
1136            .unwrap_or_else(PoisonError::into_inner)
1137            .as_ref()
1138        {
1139            let _ = outbox.send(Message::text(json!({ "event": "cancel" }).to_string()));
1140        }
1141        Ok(())
1142    }
1143
1144    async fn update(&self, field: &str, value: Value) -> Result<()> {
1145        let response = authorized(
1146            client().post(format!("{}/marmot/dataset/update", crate::get_url())),
1147            &self.authorization,
1148            self.group.as_ref(),
1149        )
1150        .json(&json!({ "workspaceId": self.workspace_id, "id": self.id, field: value }))
1151        .send()
1152        .await?;
1153        ok(response).await?;
1154        Ok(())
1155    }
1156
1157    pub async fn rename(&mut self, name: &str) -> Result<()> {
1158        self.update("name", Value::from(name)).await?;
1159        self.name = name.to_string();
1160        Ok(())
1161    }
1162
1163    /// Persist a new schema WITHOUT producing a snapshot. Use for edits that
1164    /// don't change the set of columns (prompt / sources / optional tweaks);
1165    /// for column add/drop use `migrate` instead so existing data is reconciled.
1166    pub async fn save_schema(&mut self, schema: Schema) -> Result<()> {
1167        self.update("schema", schema.serialize()).await?;
1168        self.schema = schema;
1169        Ok(())
1170    }
1171
1172    /// Migrate the dataset to a new schema, producing a fresh snapshot. Runs
1173    /// over the research socket so the caller sees the same live timeline.
1174    ///
1175    ///   - `seed_snapshot_id` = a snapshot id: the migration starts from that
1176    ///     snapshot's rows, drops removed columns, and backfills added columns
1177    ///     (researching only the new fields per row, reusing each row's stored
1178    ///     source URLs first).
1179    ///   - `seed_snapshot_id` = None: a from-scratch research run with the new
1180    ///     schema (no backfill).
1181    ///
1182    /// The diff between the seed snapshot's schema and `schema` is recomputed
1183    /// authoritatively on the server. On completion the dataset's schema is
1184    /// updated to `schema`.
1185    pub async fn migrate(
1186        &mut self,
1187        schema: Schema,
1188        seed_snapshot_id: Option<&str>,
1189        callbacks: &dyn ResearchCallbacks,
1190        options: ResearchOptions,
1191    ) -> Result<()> {
1192        let migrate =
1193            json!({ "seedSnapshotId": seed_snapshot_id, "schema": schema.serialize() }).to_string();
1194        let done = self
1195            .run_socket(vec![("Migrate", migrate)], callbacks, options, false)
1196            .await?;
1197        if done.is_some() {
1198            self.schema = schema;
1199        }
1200        Ok(())
1201    }
1202
1203    pub async fn set_termination(&mut self, termination: Vec<Termination>) -> Result<()> {
1204        self.update("termination", serde_json::to_value(&termination)?)
1205            .await?;
1206        self.termination = termination;
1207        Ok(())
1208    }
1209
1210    pub async fn set_content_types(&mut self, content_types: Vec<String>) -> Result<()> {
1211        self.update("contentTypes", json!(content_types)).await?;
1212        self.content_types = content_types;
1213        Ok(())
1214    }
1215
1216    pub async fn set_proxy(&mut self, proxy: ProxyMode) -> Result<()> {
1217        self.update("proxy", serde_json::to_value(proxy)?).await?;
1218        self.proxy = proxy;
1219        Ok(())
1220    }
1221
1222    pub async fn set_document_modes(&mut self, document_modes: Vec<DocumentMode>) -> Result<()> {
1223        self.update("documentModes", serde_json::to_value(&document_modes)?)
1224            .await?;
1225        self.document_modes = document_modes;
1226        Ok(())
1227    }
1228
1229    pub async fn remove(&self) -> Result<()> {
1230        let response = authorized(
1231            client().post(format!("{}/marmot/dataset/remove", crate::get_url())),
1232            &self.authorization,
1233            self.group.as_ref(),
1234        )
1235        .json(&json!({ "workspaceId": self.workspace_id, "id": self.id }))
1236        .send()
1237        .await?;
1238        ok(response).await?;
1239        Ok(())
1240    }
1241
1242    /// The dataset's data as CSV text or, by default, a JSON array: its live
1243    /// snapshot when new rows are stored into it, else its newest valid snapshot.
1244    pub async fn data(&self, options: DataOptions) -> Result<Value> {
1245        let csv = options.format == Some(DataFormat::Csv);
1246        let response = authorized(
1247            client().get(dataset_data_url(&self.id, &options)),
1248            &self.authorization,
1249            None,
1250        )
1251        .send()
1252        .await?;
1253        data_body(response, csv).await
1254    }
1255
1256    /// The `/data` address, for pipelines and curl. Calls to it still need your authorization.
1257    pub fn data_url(&self, options: &DataOptions) -> String {
1258        dataset_data_url(&self.id, options)
1259    }
1260
1261    /// Push rows into the dataset: objects shaped like its rows, or CSV text with a
1262    /// header row. Rows that don't fit come back in `invalid`; when none fit, it
1263    /// fails with `Error::PushRefused` listing them.
1264    pub async fn add_rows(&self, rows: RowsOrCsv, options: AddRowsOptions) -> Result<PushResult> {
1265        let mut url = format!("{}/marmot/dataset/{}/rows", crate::get_url(), self.id);
1266        if let Some(source) = options.source.filter(|source| !source.is_empty()) {
1267            url.push_str(&format!("?source={}", encode(&source)));
1268        }
1269        let request = authorized(client().post(url), &self.authorization, None);
1270        let request = match rows {
1271            RowsOrCsv::Csv(csv) => request.header("Content-Type", "text/csv").body(csv),
1272            RowsOrCsv::Rows(rows) => request.json(&rows),
1273        };
1274        let response = request.send().await?;
1275        if !response.status().is_success() {
1276            let status = response.status().as_u16();
1277            let text = response.text().await.unwrap_or_default();
1278            let body: Value = serde_json::from_str(&text).unwrap_or(Value::Null);
1279            return Err(Error::PushRefused(PushRefused {
1280                status,
1281                message: body["message"].as_str().map(str::to_string).unwrap_or(text),
1282                invalid: PushResult::from_json(&body).invalid,
1283            }));
1284        }
1285        Ok(PushResult::from_json(&response.json().await?))
1286    }
1287
1288    /// Rows pushed after `since`, oldest first, at most `max`. The next page starts at the last `seq` here.
1289    pub async fn updates(&self, query: FeedQuery) -> Result<Vec<FeedRow>> {
1290        let response = authorized(
1291            client().get(format!(
1292                "{}/marmot/dataset/{}/updates?since={}&max={}",
1293                crate::get_url(),
1294                self.id,
1295                query.since,
1296                query.max
1297            )),
1298            &self.authorization,
1299            None,
1300        )
1301        .send()
1302        .await?;
1303        let json: Value = ok(response).await?.json().await?;
1304        Ok(json
1305            .as_array()
1306            .into_iter()
1307            .flatten()
1308            .map(FeedRow::from_json)
1309            .collect())
1310    }
1311
1312    /// Every row pushed after `since`, then each new one as it lands. It opens the
1313    /// live socket first, catches up through `updates`, and skips rows it already
1314    /// gave, so nothing is missed or repeated. It reconnects by itself until the
1315    /// stream is dropped.
1316    pub fn live(&self, since: Option<u64>) -> BoxStream<'static, FeedRow> {
1317        let (rows, received) = mpsc::unbounded_channel::<FeedRow>();
1318        let dataset = self.clone();
1319        tokio::spawn(async move {
1320            let mut since = since.unwrap_or(0);
1321            while !rows.is_closed() {
1322                dataset.live_once(&mut since, &rows).await;
1323                if rows.is_closed() {
1324                    break;
1325                }
1326                tokio::time::sleep(Duration::from_secs(2)).await;
1327            }
1328        });
1329        stream::unfold(received, |mut received| async move {
1330            received.recv().await.map(|row| (row, received))
1331        })
1332        .boxed()
1333    }
1334
1335    async fn live_once(&self, since: &mut u64, rows: &mpsc::UnboundedSender<FeedRow>) {
1336        let params = json!({
1337            "Authorization": match &self.authorization {
1338                CredentialsOrApiKey::Credentials(credentials) => format!("Token {}", credentials.token),
1339                CredentialsOrApiKey::ApiKey(key) => format!("ApiKey {}", key.id),
1340            },
1341            "Dataset": self.id,
1342        });
1343        let url = format!(
1344            "{}/marmot/dataset/live?authorization={}",
1345            websocket(&crate::get_url()),
1346            encode(&params.to_string())
1347        );
1348        let Ok((socket, _)) = tokio_tungstenite::connect_async(url.as_str()).await else {
1349            return;
1350        };
1351        let (mut write, mut read) = socket.split();
1352        let (held, mut pending) = mpsc::unbounded_channel::<FeedRow>();
1353        let (ready, is_ready) = oneshot::channel::<()>();
1354        let reader = tokio::spawn(async move {
1355            let mut ready = Some(ready);
1356            while let Some(Ok(message)) = read.next().await {
1357                let Message::Text(raw) = message else {
1358                    if matches!(message, Message::Close(_)) {
1359                        break;
1360                    }
1361                    continue;
1362                };
1363                let Ok(packet) = serde_json::from_str::<Value>(raw.as_str()) else {
1364                    continue;
1365                };
1366                if packet["event"] == "ready" {
1367                    if let Some(ready) = ready.take() {
1368                        let _ = ready.send(());
1369                    }
1370                } else if packet["event"] == "rows" {
1371                    for row in packet["rows"].as_array().into_iter().flatten() {
1372                        let _ = held.send(FeedRow::from_json(row));
1373                    }
1374                }
1375            }
1376        });
1377
1378        if is_ready.await.is_ok() {
1379            loop {
1380                let Ok(page) = self
1381                    .updates(FeedQuery {
1382                        since: *since,
1383                        max: FEED_PAGE,
1384                    })
1385                    .await
1386                else {
1387                    break;
1388                };
1389                let count = page.len() as u64;
1390                for row in page {
1391                    if row.seq <= *since {
1392                        continue;
1393                    }
1394                    *since = row.seq;
1395                    if rows.send(row).is_err() {
1396                        reader.abort();
1397                        let _ = write.close().await;
1398                        return;
1399                    }
1400                }
1401                if count < FEED_PAGE {
1402                    break;
1403                }
1404            }
1405            loop {
1406                tokio::select! {
1407                    row = pending.recv() => {
1408                        let Some(row) = row else {
1409                            break;
1410                        };
1411                        if row.seq <= *since {
1412                            continue;
1413                        }
1414                        *since = row.seq;
1415                        if rows.send(row).is_err() {
1416                            break;
1417                        }
1418                    }
1419                    _ = rows.closed() => break,
1420                }
1421            }
1422        }
1423        reader.abort();
1424        let _ = write.close().await;
1425    }
1426
1427    /// Whether this dataset is kept updated, and how it's going. None when it isn't.
1428    pub async fn scraper(&self) -> Result<Option<Scraper>> {
1429        get_scraper(&self.authorization, &self.id).await
1430    }
1431
1432    /// Keep this dataset updated by its workspace's scraper agent. You become the person the agent runs as.
1433    pub async fn keep_updated(&self, settings: &ScraperSettings) -> Result<Scraper> {
1434        create_scraper(&self.authorization, &self.id, settings).await
1435    }
1436
1437    pub async fn update_scraper(&self, settings: &ScraperSettings) -> Result<Scraper> {
1438        update_scraper(&self.authorization, &self.id, settings).await
1439    }
1440
1441    /// Ask the agent to go over this dataset's scraper again, with a note for it.
1442    pub async fn rebuild_scraper(&self, note: Option<&str>) -> Result<()> {
1443        rebuild_scraper(&self.authorization, &self.id, note).await
1444    }
1445
1446    /// Stop keeping this dataset updated.
1447    pub async fn stop_keeping_updated(&self) -> Result<()> {
1448        remove_scraper(&self.authorization, &self.id).await
1449    }
1450}
1451
1452/// The `opts` argument of the calls that take only a group.
1453#[derive(Debug, Clone, Default)]
1454pub struct GroupOptions {
1455    pub group: Option<Group>,
1456}
1457
1458pub async fn list_datasets(
1459    authorization: &CredentialsOrApiKey,
1460    workspace_id: &str,
1461    opts: GroupOptions,
1462) -> Result<Vec<Dataset>> {
1463    let response = authorized(
1464        client().post(format!("{}/marmot/dataset/list", crate::get_url())),
1465        authorization,
1466        opts.group.as_ref(),
1467    )
1468    .json(&json!({ "workspaceId": workspace_id }))
1469    .send()
1470    .await?;
1471    let data: Value = ok(response).await?.json().await?;
1472    data.as_array()
1473        .into_iter()
1474        .flatten()
1475        .map(|d| {
1476            let id = text(&d["id"]);
1477            let snapshots = d["snapshots"]
1478                .as_array()
1479                .into_iter()
1480                .flatten()
1481                .map(|s| {
1482                    Snapshot::new(
1483                        authorization,
1484                        workspace_id,
1485                        &id,
1486                        &text(&s["id"]),
1487                        crate::tickets::parse_date(&s["createdAt"]).unwrap_or_default(),
1488                        s["valid"].as_bool().unwrap_or(true),
1489                        opts.group.as_ref(),
1490                    )
1491                })
1492                .collect();
1493            Ok(Dataset {
1494                workspace_id: workspace_id.to_string(),
1495                name: text(&d["name"]),
1496                schema: Schema::deserialize(&d["schema"])?,
1497                termination: serde_json::from_value(d["termination"].clone()).unwrap_or_default(),
1498                content_types: serde_json::from_value(d["contentTypes"].clone())
1499                    .unwrap_or_default(),
1500                proxy: serde_json::from_value(d["proxy"].clone()).unwrap_or(ProxyMode::Datacenter),
1501                document_modes: serde_json::from_value(d["documentModes"].clone())
1502                    .unwrap_or_else(|_| vec![DocumentMode::Extraction]),
1503                snapshots: Arc::new(Mutex::new(snapshots)),
1504                authorization: authorization.clone(),
1505                group: opts.group.clone(),
1506                ws: Arc::new(Mutex::new(None)),
1507                id,
1508            })
1509        })
1510        .collect()
1511}
1512
1513/// The `options` argument of `create_dataset`.
1514#[derive(Clone)]
1515pub struct CreateDatasetOptions {
1516    pub name: String,
1517    pub schema: Schema,
1518    pub termination: Option<Vec<Termination>>,
1519    pub content_types: Option<Vec<String>>,
1520    pub proxy: Option<ProxyMode>,
1521    pub document_modes: Option<Vec<DocumentMode>>,
1522}
1523
1524pub async fn create_dataset(
1525    authorization: &CredentialsOrApiKey,
1526    workspace_id: &str,
1527    options: &CreateDatasetOptions,
1528    opts: GroupOptions,
1529) -> Result<String> {
1530    let response = authorized(
1531        client().post(format!("{}/marmot/dataset/create", crate::get_url())),
1532        authorization,
1533        opts.group.as_ref(),
1534    )
1535    .json(&json!({
1536        "workspaceId": workspace_id,
1537        "name": options.name,
1538        "schema": options.schema.serialize(),
1539        "termination": options.termination.clone().unwrap_or_default(),
1540        "contentTypes": options.content_types.clone().unwrap_or_default(),
1541        "proxy": options.proxy.unwrap_or(ProxyMode::Datacenter),
1542        "documentModes": options.document_modes.clone().unwrap_or_else(|| vec![DocumentMode::Extraction]),
1543    }))
1544    .send()
1545    .await?;
1546    Ok(ok(response).await?.text().await?)
1547}
1548
1549#[derive(Debug, Clone)]
1550pub struct Workspace {
1551    pub id: String,
1552    pub name: String,
1553    pub description: String,
1554    pub created_at: i64,
1555    pub datasets: u64,
1556    pub files: u64,
1557
1558    authorization: CredentialsOrApiKey,
1559    group: Option<Group>,
1560}
1561
1562impl Workspace {
1563    fn opts(&self) -> GroupOptions {
1564        GroupOptions {
1565            group: self.group.clone(),
1566        }
1567    }
1568
1569    pub async fn list_datasets(&self) -> Result<Vec<Dataset>> {
1570        list_datasets(&self.authorization, &self.id, self.opts()).await
1571    }
1572
1573    pub async fn create_dataset(&self, options: &CreateDatasetOptions) -> Result<String> {
1574        create_dataset(&self.authorization, &self.id, options, self.opts()).await
1575    }
1576
1577    pub async fn rename(&mut self, name: &str) -> Result<()> {
1578        update_workspace(
1579            &self.authorization,
1580            &self.id,
1581            &PartialWorkspace {
1582                name: Some(name.to_string()),
1583                description: None,
1584            },
1585            self.opts(),
1586        )
1587        .await?;
1588        self.name = name.to_string();
1589        Ok(())
1590    }
1591
1592    pub async fn set_description(&mut self, description: &str) -> Result<()> {
1593        update_workspace(
1594            &self.authorization,
1595            &self.id,
1596            &PartialWorkspace {
1597                name: None,
1598                description: Some(description.to_string()),
1599            },
1600            self.opts(),
1601        )
1602        .await?;
1603        self.description = description.to_string();
1604        Ok(())
1605    }
1606
1607    async fn post(&self, route: &str, body: Value) -> Result<reqwest::Response> {
1608        let response = authorized(
1609            client().post(format!("{}{route}", crate::get_url())),
1610            &self.authorization,
1611            self.group.as_ref(),
1612        )
1613        .json(&body)
1614        .send()
1615        .await?;
1616        ok(response).await
1617    }
1618
1619    pub async fn remove(&self) -> Result<()> {
1620        self.post("/marmot/workspace/remove", json!({ "id": self.id }))
1621            .await?;
1622        Ok(())
1623    }
1624
1625    pub async fn get_files(&self) -> Result<Vec<WorkspaceFile>> {
1626        Ok(get_workspace(&self.authorization, &self.id, self.opts())
1627            .await?
1628            .files)
1629    }
1630
1631    pub async fn upload_file(
1632        &self,
1633        name: &str,
1634        content_type: &str,
1635        data: &[u8],
1636    ) -> Result<WorkspaceFile> {
1637        use base64::Engine;
1638        let data = base64::engine::general_purpose::STANDARD.encode(data);
1639        Ok(self
1640            .post(
1641                "/marmot/workspace/file/upload",
1642                json!({ "workspaceId": self.id, "name": name, "contentType": content_type, "data": data }),
1643            )
1644            .await?
1645            .json()
1646            .await?)
1647    }
1648
1649    pub async fn download_file(&self, file: &WorkspaceFile) -> Result<Vec<u8>> {
1650        Ok(self
1651            .post(
1652                "/marmot/workspace/file/get",
1653                json!({ "key": file.key, "contentType": file.content_type, "workspaceId": self.id }),
1654            )
1655            .await?
1656            .bytes()
1657            .await?
1658            .to_vec())
1659    }
1660
1661    pub async fn remove_file(&self, key: &str) -> Result<()> {
1662        self.post(
1663            "/marmot/workspace/file/remove",
1664            json!({ "workspaceId": self.id, "key": key }),
1665        )
1666        .await?;
1667        Ok(())
1668    }
1669}
1670
1671pub async fn list_workspaces(
1672    authorization: &CredentialsOrApiKey,
1673    opts: GroupOptions,
1674) -> Result<Vec<Workspace>> {
1675    let response = authorized(
1676        client().post(format!("{}/marmot/workspace/list", crate::get_url())),
1677        authorization,
1678        opts.group.as_ref(),
1679    )
1680    .header("Content-Type", "application/json")
1681    .send()
1682    .await?;
1683    let data: Value = ok(response).await?.json().await?;
1684    Ok(data
1685        .as_array()
1686        .into_iter()
1687        .flatten()
1688        .map(|w| Workspace {
1689            id: text(&w["id"]),
1690            name: text(&w["name"]),
1691            description: text(&w["description"]),
1692            created_at: w["createdAt"].as_f64().unwrap_or_default() as i64,
1693            datasets: w["datasets"].as_f64().unwrap_or_default() as u64,
1694            files: w["files"].as_f64().unwrap_or_default() as u64,
1695            authorization: authorization.clone(),
1696            group: opts.group.clone(),
1697        })
1698        .collect())
1699}
1700
1701/// `{ id, name, description, createdAt, files }` in the TypeScript SDK.
1702#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
1703#[serde(rename_all = "camelCase")]
1704pub struct WorkspaceDetail {
1705    pub id: String,
1706    pub name: String,
1707    #[serde(default)]
1708    pub description: String,
1709    #[serde(default)]
1710    pub created_at: i64,
1711    #[serde(default)]
1712    pub files: Vec<WorkspaceFile>,
1713}
1714
1715pub async fn get_workspace(
1716    authorization: &CredentialsOrApiKey,
1717    id: &str,
1718    opts: GroupOptions,
1719) -> Result<WorkspaceDetail> {
1720    let response = authorized(
1721        client().post(format!("{}/marmot/workspace/get", crate::get_url())),
1722        authorization,
1723        opts.group.as_ref(),
1724    )
1725    .json(&json!({ "id": id }))
1726    .send()
1727    .await?;
1728    Ok(ok(response).await?.json().await?)
1729}
1730
1731/// `description` defaults to empty.
1732pub async fn create_workspace(
1733    authorization: &CredentialsOrApiKey,
1734    name: &str,
1735    description: Option<&str>,
1736    opts: GroupOptions,
1737) -> Result<String> {
1738    let response = authorized(
1739        client().post(format!("{}/marmot/workspace/create", crate::get_url())),
1740        authorization,
1741        opts.group.as_ref(),
1742    )
1743    .json(&json!({ "name": name, "description": description.unwrap_or_default() }))
1744    .send()
1745    .await?;
1746    Ok(ok(response).await?.text().await?)
1747}
1748
1749pub async fn update_workspace(
1750    authorization: &CredentialsOrApiKey,
1751    id: &str,
1752    update: &PartialWorkspace,
1753    opts: GroupOptions,
1754) -> Result<()> {
1755    let mut body = Map::new();
1756    body.insert("id".to_string(), Value::from(id));
1757    if let Value::Object(fields) = serde_json::to_value(update)? {
1758        body.extend(fields);
1759    }
1760    let response = authorized(
1761        client().post(format!("{}/marmot/workspace/update", crate::get_url())),
1762        authorization,
1763        opts.group.as_ref(),
1764    )
1765    .json(&body)
1766    .send()
1767    .await?;
1768    ok(response).await?;
1769    Ok(())
1770}