1use std::ops::Deref;
2use std::sync::{Arc, Mutex, MutexGuard, PoisonError};
3use std::time::Duration;
4
5use chrono::{DateTime, Utc};
6use futures_util::stream::{self, BoxStream};
7use futures_util::{SinkExt, StreamExt};
8use serde::{Deserialize, Serialize};
9use serde_json::{Map, Value, json};
10use tokio::sync::{mpsc, oneshot};
11use tokio_tungstenite::tungstenite::Message;
12
13use super::scraper::{
14 Scraper, ScraperSettings, create_scraper, get_scraper, rebuild_scraper, remove_scraper,
15 update_scraper,
16};
17use super::workspace::PartialWorkspace;
18use crate::authentification::CredentialsOrApiKey;
19use crate::error::{Error, Result};
20use crate::groups::Group;
21use crate::http::{client, encode, ok, websocket};
22use crate::parrot::format::Format;
23use crate::parrot::generative::{Embedding, Text};
24
25pub use super::media::{DocumentMode, ProxyMode};
26pub use super::workspace::WorkspaceFile;
27
28#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
30pub struct ArrayBounds {
31 pub minimum: u64,
32 pub maximum: u64,
33}
34
35#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
37#[serde(untagged)]
38pub enum StringOrStrings {
39 String(String),
40 Strings(Vec<String>),
41}
42
43#[allow(clippy::large_enum_variant)]
46pub enum FormatOrSchemaLeafOrChildren {
47 Format(Format),
48 SchemaLeaf(Box<SchemaLeaf>),
49 Children(Vec<(String, SchemaLeaf)>),
50}
51
52pub struct SchemaLeaf {
54 pub value: FormatOrSchemaLeafOrChildren,
55 pub prompt: Option<String>,
56 pub sources: Option<u64>,
57 pub optional: Option<bool>,
58 pub array: Option<ArrayBounds>,
59
60 pub max_iterations: Option<u64>,
61 pub filters: Option<StringOrStrings>,
62}
63
64impl Clone for SchemaLeaf {
65 fn clone(&self) -> Self {
66 deserialize_leaf(&serialize_leaf(self)).expect("a leaf reads its own JSON")
67 }
68}
69
70fn insert(fields: &mut Map<String, Value>, key: &str, value: Option<Value>) {
71 if let Some(value) = value {
72 fields.insert(key.to_string(), value);
73 }
74}
75
76fn serialize_leaf(leaf: &SchemaLeaf) -> Value {
77 let mut fields = Map::new();
78 insert(&mut fields, "prompt", leaf.prompt.clone().map(Value::from));
79 insert(&mut fields, "sources", leaf.sources.map(Value::from));
80 insert(&mut fields, "optional", leaf.optional.map(Value::from));
81 insert(&mut fields, "array", leaf.array.map(|array| json!(array)));
82 insert(
83 &mut fields,
84 "maxIterations",
85 leaf.max_iterations.map(Value::from),
86 );
87 insert(
88 &mut fields,
89 "filters",
90 leaf.filters.as_ref().map(|filters| json!(filters)),
91 );
92 let value = match &leaf.value {
93 FormatOrSchemaLeafOrChildren::Format(format) => json!({ "question": format.to_json() }),
94 FormatOrSchemaLeafOrChildren::SchemaLeaf(leaf) => json!({ "schema": serialize_leaf(leaf) }),
95 FormatOrSchemaLeafOrChildren::Children(children) => {
96 let children: Map<String, Value> = children
97 .iter()
98 .map(|(key, leaf)| (key.clone(), serialize_leaf(leaf)))
99 .collect();
100 json!({ "children": children })
101 }
102 };
103 fields.insert("value".to_string(), value);
104 Value::Object(fields)
105}
106
107fn deserialize_leaf(data: &Value) -> Result<SchemaLeaf> {
108 let array = match &data["array"] {
109 Value::Null => None,
110 array => {
111 let bound = |key: &str| {
112 array[key]
113 .as_f64()
114 .filter(|bound| bound.fract() == 0.0 && *bound >= 0.0)
115 };
116 let Some(minimum) = bound("minimum") else {
117 return Err(Error::Message(format!(
118 "SchemaLeaf array.minimum must be a non-negative integer, got {}",
119 array["minimum"]
120 )));
121 };
122 let Some(maximum) = bound("maximum") else {
123 return Err(Error::Message(format!(
124 "SchemaLeaf array.maximum must be a non-negative integer, got {}",
125 array["maximum"]
126 )));
127 };
128 if minimum > maximum {
129 return Err(Error::Message(format!(
130 "SchemaLeaf array.minimum ({minimum}) cannot exceed array.maximum ({maximum})"
131 )));
132 }
133 Some(ArrayBounds {
134 minimum: minimum as u64,
135 maximum: maximum as u64,
136 })
137 }
138 };
139 let value = &data["value"];
140 let value = if value.get("question").is_some() {
141 FormatOrSchemaLeafOrChildren::Format(Format::from_json(&value["question"])?)
142 } else if let Some(children) = value.get("children") {
143 FormatOrSchemaLeafOrChildren::Children(
144 children
145 .as_object()
146 .into_iter()
147 .flatten()
148 .map(|(key, leaf)| Ok((key.clone(), deserialize_leaf(leaf)?)))
149 .collect::<Result<Vec<_>>>()?,
150 )
151 } else {
152 FormatOrSchemaLeafOrChildren::SchemaLeaf(Box::new(deserialize_leaf(&value["schema"])?))
153 };
154 Ok(SchemaLeaf {
155 value,
156 prompt: data["prompt"].as_str().map(str::to_string),
157 sources: data["sources"].as_u64(),
158 optional: data["optional"].as_bool(),
159 array,
160 max_iterations: data["maxIterations"].as_u64(),
161 filters: serde_json::from_value(data["filters"].clone()).ok(),
162 })
163}
164
165#[derive(Clone)]
167#[allow(clippy::large_enum_variant)]
168pub enum Schema {
169 Json(SchemaJSON),
170 Csv(SchemaCSV),
171}
172
173impl Schema {
174 pub fn r#type(&self) -> &'static str {
175 match self {
176 Schema::Json(_) => "json",
177 Schema::Csv(_) => "csv",
178 }
179 }
180
181 pub fn serialize(&self) -> Value {
182 match self {
183 Schema::Json(schema) => schema.serialize(),
184 Schema::Csv(schema) => schema.serialize(),
185 }
186 }
187
188 pub fn deserialize(data: &Value) -> Result<Schema> {
189 match data["type"].as_str() {
190 Some("json") => Ok(Schema::Json(SchemaJSON::new(deserialize_leaf(
191 &data["schema"],
192 )?))),
193 Some("csv") => {
194 let columns = data["schema"]
195 .as_object()
196 .into_iter()
197 .flatten()
198 .map(|(key, leaf)| Ok((key.clone(), deserialize_leaf(leaf)?)))
199 .collect::<Result<Vec<_>>>()?;
200 let bound = |key: &str| data[key].as_f64().unwrap_or(f64::NAN);
201 Ok(Schema::Csv(SchemaCSV::new(
202 columns,
203 bound("minimum"),
204 bound("maximum"),
205 data["prompt"].as_str(),
206 )?))
207 }
208 kind => Err(Error::Message(format!(
209 "Invalid schema type: {}",
210 kind.unwrap_or("undefined")
211 ))),
212 }
213 }
214}
215
216#[derive(Clone)]
217pub struct SchemaJSON {
218 pub schema: SchemaLeaf,
219}
220
221impl SchemaJSON {
222 pub fn new(schema: SchemaLeaf) -> SchemaJSON {
223 SchemaJSON { schema }
224 }
225
226 pub fn r#type(&self) -> &'static str {
227 "json"
228 }
229
230 pub fn serialize(&self) -> Value {
231 json!({ "type": "json", "schema": serialize_leaf(&self.schema) })
232 }
233}
234
235#[derive(Clone)]
236pub struct SchemaCSV {
237 pub schema: Vec<(String, SchemaLeaf)>,
240 pub minimum: u64,
241 pub maximum: u64,
242 pub prompt: String,
243}
244
245impl SchemaCSV {
246 pub fn new(
249 schema: Vec<(String, SchemaLeaf)>,
250 minimum: f64,
251 maximum: f64,
252 prompt: Option<&str>,
253 ) -> Result<SchemaCSV> {
254 if schema
255 .iter()
256 .any(|(_, leaf)| !matches!(leaf.value, FormatOrSchemaLeafOrChildren::Format(_)))
257 {
258 return Err(Error::Message(
259 "Nested schema are not supported for CSV".to_string(),
260 ));
261 }
262 let whole = |count: f64| count.fract() == 0.0 && count >= 0.0;
263 if !whole(minimum) {
264 return Err(Error::Message(format!(
265 "SchemaCSV minimum must be a non-negative integer, got {minimum}"
266 )));
267 }
268 if !whole(maximum) {
269 return Err(Error::Message(format!(
270 "SchemaCSV maximum must be a non-negative integer, got {maximum}"
271 )));
272 }
273 if minimum > maximum {
274 return Err(Error::Message(format!(
275 "SchemaCSV minimum ({minimum}) cannot exceed maximum ({maximum})"
276 )));
277 }
278 Ok(SchemaCSV {
279 schema,
280 minimum: minimum as u64,
281 maximum: maximum as u64,
282 prompt: prompt.unwrap_or_default().to_string(),
283 })
284 }
285
286 pub fn r#type(&self) -> &'static str {
287 "csv"
288 }
289
290 pub fn serialize(&self) -> Value {
291 let schema: Map<String, Value> = self
292 .schema
293 .iter()
294 .map(|(key, leaf)| (key.clone(), serialize_leaf(leaf)))
295 .collect();
296 json!({
297 "type": "csv",
298 "minimum": self.minimum,
299 "maximum": self.maximum,
300 "prompt": self.prompt,
301 "schema": schema,
302 })
303 }
304}
305
306fn format_signature(value: &FormatOrSchemaLeafOrChildren) -> String {
307 match value {
308 FormatOrSchemaLeafOrChildren::Format(format) => format!(
309 "format:{}",
310 format.to_json()["type"].as_str().unwrap_or("unknown")
311 ),
312 _ => "composite".to_string(),
313 }
314}
315
316fn set(out: &mut Vec<(String, String)>, path: String, signature: String) {
317 match out.iter_mut().find(|(known, _)| *known == path) {
318 Some(entry) => entry.1 = signature,
319 None => out.push((path, signature)),
320 }
321}
322
323fn flatten_leaf_paths(leaf: &SchemaLeaf, path: &str, out: &mut Vec<(String, String)>) {
324 if leaf.array.is_some() {
325 if let FormatOrSchemaLeafOrChildren::SchemaLeaf(inner) = &leaf.value {
328 flatten_leaf_paths(inner, &format!("{path}[]"), out);
329 }
330 return;
331 }
332 match &leaf.value {
333 FormatOrSchemaLeafOrChildren::Format(_) => {
334 let path = if path.is_empty() { "value" } else { path };
335 set(out, path.to_string(), format_signature(&leaf.value));
336 }
337 FormatOrSchemaLeafOrChildren::SchemaLeaf(inner) => flatten_leaf_paths(inner, path, out),
339 FormatOrSchemaLeafOrChildren::Children(children) => {
340 for (key, child) in children {
341 let child_path = if path.is_empty() {
342 key.clone()
343 } else {
344 format!("{path}.{key}")
345 };
346 flatten_leaf_paths(child, &child_path, out);
347 }
348 }
349 }
350}
351
352pub fn schema_paths(schema: &Schema) -> Vec<(String, String)> {
357 let mut out = Vec::new();
358 match schema {
359 Schema::Csv(schema) => {
360 for (key, leaf) in &schema.schema {
361 set(&mut out, key.clone(), format_signature(&leaf.value));
362 }
363 }
364 Schema::Json(schema) => flatten_leaf_paths(&schema.schema, "", &mut out),
365 }
366 out
367}
368
369#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
371pub struct SchemaDiff {
372 pub added: Vec<String>,
373 pub dropped: Vec<String>,
374}
375
376pub fn diff_schema(baseline: &Schema, target: &Schema) -> SchemaDiff {
381 let a = schema_paths(baseline);
382 let b = schema_paths(target);
383 let lookup = |paths: &[(String, String)], path: &str| {
384 paths
385 .iter()
386 .find(|(known, _)| known == path)
387 .map(|(_, signature)| signature.clone())
388 };
389 SchemaDiff {
390 added: b
391 .iter()
392 .filter(|(path, signature)| lookup(&a, path).as_ref() != Some(signature))
393 .map(|(path, _)| path.clone())
394 .collect(),
395 dropped: a
396 .iter()
397 .filter(|(path, signature)| lookup(&b, path).as_ref() != Some(signature))
398 .map(|(path, _)| path.clone())
399 .collect(),
400 }
401}
402
403#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
405pub struct Termination {
406 pub r#type: String,
407 #[serde(flatten)]
408 pub fields: Map<String, Value>,
409}
410
411#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
412pub struct TokenTermination {
413 #[serde(flatten)]
414 pub termination: Termination,
415 pub max: f64,
416}
417
418impl Deref for TokenTermination {
419 type Target = Termination;
420 fn deref(&self) -> &Termination {
421 &self.termination
422 }
423}
424
425#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
426pub struct TimeTermination {
427 #[serde(flatten)]
428 pub termination: Termination,
429 pub max: f64,
430}
431
432impl Deref for TimeTermination {
433 type Target = Termination;
434 fn deref(&self) -> &Termination {
435 &self.termination
436 }
437}
438
439#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
440pub struct CostTermination {
441 #[serde(flatten)]
442 pub termination: Termination,
443 pub max: f64,
444}
445
446impl Deref for CostTermination {
447 type Target = Termination;
448 fn deref(&self) -> &Termination {
449 &self.termination
450 }
451}
452
453#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Hash, Serialize, Deserialize)]
455#[serde(rename_all = "lowercase")]
456pub enum DataFormat {
457 Csv,
458 #[default]
459 Json,
460}
461
462#[derive(Debug, Clone, Default, PartialEq, Eq)]
463pub struct DataOptions {
464 pub format: Option<DataFormat>,
465 pub snapshot: Option<String>,
467 pub sources: Option<bool>,
469}
470
471#[derive(Debug, Clone, PartialEq, Serialize)]
473pub struct FeedRow {
474 pub seq: u64,
475 pub value: Value,
476 pub sources: Value,
477 pub created: DateTime<Utc>,
478}
479
480impl FeedRow {
481 pub fn from_json(json: &Value) -> FeedRow {
482 FeedRow {
483 seq: json["seq"].as_f64().unwrap_or_default() as u64,
484 value: json["value"].clone(),
485 sources: json["sources"].clone(),
486 created: crate::tickets::parse_date(&json["created"]).unwrap_or_default(),
487 }
488 }
489}
490
491#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
493pub struct InvalidRow {
494 pub index: u64,
495 pub reason: String,
496}
497
498#[derive(Debug, Clone, Default, PartialEq, Serialize)]
500pub struct PushResult {
501 pub added: u64,
502 pub first: Option<f64>,
503 pub last: Option<f64>,
504 pub invalid: Vec<InvalidRow>,
505}
506
507impl PushResult {
508 pub fn from_json(json: &Value) -> PushResult {
509 PushResult {
510 added: json["added"].as_f64().unwrap_or_default() as u64,
511 first: json["first"].as_f64(),
512 last: json["last"].as_f64(),
513 invalid: json["invalid"]
514 .as_array()
515 .into_iter()
516 .flatten()
517 .map(|row| InvalidRow {
518 index: row["index"].as_f64().unwrap_or_default() as u64,
519 reason: match &row["reason"] {
520 Value::String(reason) => reason.clone(),
521 reason => reason.to_string(),
522 },
523 })
524 .collect(),
525 }
526 }
527}
528
529#[derive(Debug, Clone, PartialEq, Eq)]
531pub struct PushRefused {
532 pub status: u16,
533 pub message: String,
534 pub invalid: Vec<InvalidRow>,
535}
536
537const FEED_PAGE: u64 = 500;
538
539fn dataset_data_url(dataset_id: &str, options: &DataOptions) -> String {
540 let mut query = format!(
541 "format={}",
542 match options.format.unwrap_or_default() {
543 DataFormat::Csv => "csv",
544 DataFormat::Json => "json",
545 }
546 );
547 if let Some(snapshot) = &options.snapshot {
548 query.push_str(&format!("&snapshot={}", encode(snapshot)));
549 }
550 if options.sources == Some(true) {
551 query.push_str("&sources=true");
552 }
553 format!(
554 "{}/marmot/dataset/{dataset_id}/data?{query}",
555 crate::get_url()
556 )
557}
558
559fn authorized(
560 request: reqwest::RequestBuilder,
561 authorization: &CredentialsOrApiKey,
562 group: Option<&Group>,
563) -> reqwest::RequestBuilder {
564 let request = request.header(
565 "Authorization",
566 match authorization {
567 CredentialsOrApiKey::Credentials(credentials) => format!("Token {}", credentials.token),
568 CredentialsOrApiKey::ApiKey(key) => format!("ApiKey {}", key.id),
569 },
570 );
571 match group {
572 Some(group) => request.header("Group", &group.id),
573 None => request,
574 }
575}
576
577async fn data_body(response: reqwest::Response, csv: bool) -> Result<Value> {
578 let response = ok(response).await?;
579 Ok(if csv {
580 Value::String(response.text().await?)
581 } else {
582 response.json().await?
583 })
584}
585
586#[derive(Debug, Clone, PartialEq)]
589pub enum SnapshotData {
590 Json(Value),
591 Text(String),
592 Bytes(Vec<u8>),
593}
594
595#[derive(Debug, Clone, Default)]
597pub struct SnapshotDataOptions {
598 pub format: Option<DataFormat>,
599 pub sources: Option<bool>,
600}
601
602#[derive(Default)]
603struct SnapshotCache {
604 schema: Option<Schema>,
605 data: Option<SnapshotData>,
606}
607
608#[derive(Clone)]
609pub struct Snapshot {
610 pub id: String,
611 pub created_at: DateTime<Utc>,
612
613 pub valid: bool,
614
615 authorization: CredentialsOrApiKey,
616 workspace_id: String,
617 dataset_id: String,
618 group: Option<Group>,
619 cache: Arc<Mutex<SnapshotCache>>,
620}
621
622impl Snapshot {
623 pub fn new(
624 authorization: &CredentialsOrApiKey,
625 workspace_id: &str,
626 dataset_id: &str,
627 id: &str,
628 created_at: DateTime<Utc>,
629 valid: bool,
630 group: Option<&Group>,
631 ) -> Snapshot {
632 Snapshot {
633 id: id.to_string(),
634 created_at,
635 valid,
636 authorization: authorization.clone(),
637 workspace_id: workspace_id.to_string(),
638 dataset_id: dataset_id.to_string(),
639 group: group.cloned(),
640 cache: Arc::new(Mutex::new(SnapshotCache::default())),
641 }
642 }
643
644 fn cache(&self) -> MutexGuard<'_, SnapshotCache> {
645 self.cache.lock().unwrap_or_else(PoisonError::into_inner)
646 }
647
648 fn url(&self, tail: &str) -> String {
649 format!(
650 "{}/marmot/snapshot/{}/{}/{}{tail}",
651 crate::get_url(),
652 self.workspace_id,
653 self.dataset_id,
654 self.id
655 )
656 }
657
658 async fn get(&self, tail: &str) -> Result<reqwest::Response> {
659 let response = authorized(
660 client().get(self.url(tail)),
661 &self.authorization,
662 self.group.as_ref(),
663 )
664 .send()
665 .await?;
666 ok(response).await
667 }
668
669 pub async fn download(&self, assets: bool, revalidate: bool) -> Result<SnapshotData> {
670 if !assets
671 && !revalidate
672 && let Some(data) = self.cache().data.clone()
673 {
674 return Ok(data);
675 }
676 let response = self.get(&format!("/?assets={assets}")).await?;
677 if assets {
678 return Ok(SnapshotData::Bytes(response.bytes().await?.to_vec()));
679 }
680 let json = response
681 .headers()
682 .get("content-type")
683 .and_then(|value| value.to_str().ok())
684 .is_some_and(|value| value.contains("application/json"));
685 let data = if json {
686 SnapshotData::Json(response.json().await?)
687 } else {
688 SnapshotData::Text(response.text().await?)
689 };
690 self.cache().data = Some(data.clone());
691 Ok(data)
692 }
693
694 pub async fn data(&self, options: SnapshotDataOptions) -> Result<Value> {
696 let csv = options.format == Some(DataFormat::Csv);
697 let url = dataset_data_url(
698 &self.dataset_id,
699 &DataOptions {
700 format: options.format,
701 snapshot: Some(self.id.clone()),
702 sources: options.sources,
703 },
704 );
705 let response = authorized(client().get(url), &self.authorization, None)
706 .send()
707 .await?;
708 data_body(response, csv).await
709 }
710
711 pub async fn download_schema(&self, revalidate: bool) -> Result<Schema> {
712 if !revalidate && let Some(schema) = self.cache().schema.clone() {
713 return Ok(schema);
714 }
715 let json: Value = self.get("/schema").await?.json().await?;
716 let schema = Schema::deserialize(&json)?;
717 self.cache().schema = Some(schema.clone());
718 Ok(schema)
719 }
720
721 pub async fn download_log(&self) -> Result<Vec<u8>> {
722 Ok(self.get("/log").await?.bytes().await?.to_vec())
723 }
724
725 pub async fn download_embeddings(&self) -> Result<Vec<u8>> {
726 Ok(self.get("/embeddings").await?.bytes().await?.to_vec())
727 }
728
729 pub async fn delete(&self) -> Result<()> {
730 let response = authorized(
731 client().delete(self.url("")),
732 &self.authorization,
733 self.group.as_ref(),
734 )
735 .send()
736 .await?;
737 ok(response).await?;
738 Ok(())
739 }
740}
741
742#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
746pub struct MachineFrame {
747 pub url: String,
748 pub machine: u64,
749}
750
751#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
753#[serde(rename_all = "lowercase")]
754pub enum CancelReason {
755 User,
756 Tokens,
757 Time,
758 Cost,
759}
760
761#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
763pub struct Candidate {
764 pub value: Value,
765 pub source: String,
766}
767
768pub trait ResearchCallbacks: Send + Sync {
771 fn on_connection(&self, _ongoing: bool) {}
772 fn on_query(&self, _query: &str) {}
773 fn on_source(&self, _url: &str, _title: &str, _content_type: &str) {}
774 fn on_answer(
775 &self,
776 _field: &str,
777 _value: &Value,
778 _converged: bool,
779 _answers_count: u64,
780 _sources: Option<&[String]>,
781 ) {
782 }
783 fn on_nomination(
787 &self,
788 _scope: &str,
789 _row: &Value,
790 _committed_count: u64,
791 _sources: Option<&[String]>,
792 ) {
793 }
794 fn on_supervisor_query(&self, _question: &str) {}
795 fn on_supervisor_search(&self, _query: &str) {}
796 fn on_supervisor_answer(&self, _question: &str, _answer: &str) {}
797 fn on_supervisor_refused(&self, _question: &str) {}
798 fn on_machine(&self, _frame: &MachineFrame) {}
800 fn on_compress(&self) {}
801 fn on_info(&self, _message: &str) {}
802 fn on_done(&self, _snapshot: &Snapshot) {}
803 fn on_cancelled(&self, _reason: CancelReason) {}
804 fn on_convergence_failed(&self, _field: &str, _reason: &str, _candidates: &[Candidate]) {}
805 fn on_error(&self, _message: &str) {}
806}
807
808impl ResearchCallbacks for () {}
809
810#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
811pub struct ResearchOptions {
812 pub text: Option<Text>,
813 pub embedding: Option<Embedding>,
814}
815
816#[derive(Debug, Clone, PartialEq)]
818pub enum RowsOrCsv {
819 Rows(Vec<Value>),
820 Csv(String),
821}
822
823#[derive(Debug, Clone, Default)]
825pub struct AddRowsOptions {
826 pub source: Option<String>,
827}
828
829#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
831pub struct FeedQuery {
832 pub since: u64,
833 pub max: u64,
834}
835
836fn text(value: &Value) -> String {
837 match value {
838 Value::Null => String::new(),
839 Value::String(text) => text.clone(),
840 value => value.to_string(),
841 }
842}
843
844#[derive(Clone)]
845pub struct Dataset {
846 pub id: String,
847 pub workspace_id: String,
848 pub name: String,
849 pub schema: Schema,
850 pub termination: Vec<Termination>,
851 pub content_types: Vec<String>,
852 pub proxy: ProxyMode,
853 pub document_modes: Vec<DocumentMode>,
854 snapshots: Arc<Mutex<Vec<Snapshot>>>,
855
856 authorization: CredentialsOrApiKey,
857 group: Option<Group>,
858 ws: Arc<Mutex<Option<mpsc::UnboundedSender<Message>>>>,
859}
860
861impl Dataset {
862 pub fn snapshots(&self) -> Vec<Snapshot> {
863 self.snapshots
864 .lock()
865 .unwrap_or_else(PoisonError::into_inner)
866 .clone()
867 }
868
869 pub async fn research(
870 &self,
871 callbacks: &dyn ResearchCallbacks,
872 options: ResearchOptions,
873 ) -> Result<()> {
874 self.run_socket(Vec::new(), callbacks, options, false)
875 .await
876 .map(|_| ())
877 }
878
879 pub async fn reconnect(
887 &self,
888 callbacks: &dyn ResearchCallbacks,
889 options: ResearchOptions,
890 ) -> Result<()> {
891 self.run_socket(Vec::new(), callbacks, options, true)
892 .await
893 .map(|_| ())
894 }
895
896 async fn run_socket(
897 &self,
898 extra: Vec<(&str, String)>,
899 callbacks: &dyn ResearchCallbacks,
900 options: ResearchOptions,
901 reconnect: bool,
902 ) -> Result<Option<Snapshot>> {
903 let mut params = Map::new();
904 params.insert(
905 "Authorization".to_string(),
906 Value::from(match &self.authorization {
907 CredentialsOrApiKey::Credentials(credentials) => {
908 format!("Token {}", credentials.token)
909 }
910 CredentialsOrApiKey::ApiKey(key) => format!("ApiKey {}", key.id),
911 }),
912 );
913 if let Some(group) = &self.group {
914 params.insert("Group".to_string(), Value::from(group.id.as_str()));
915 }
916 params.insert(
917 "Workspace".to_string(),
918 Value::from(self.workspace_id.as_str()),
919 );
920 params.insert("Dataset".to_string(), Value::from(self.id.as_str()));
921 for (key, value) in extra {
922 params.insert(key.to_string(), Value::from(value));
923 }
924 if let Some(model) = options.text {
925 params.insert("Text".to_string(), Value::from(model.as_str()));
926 }
927 if let Some(model) = options.embedding {
928 params.insert("Embedding".to_string(), Value::from(model.as_str()));
929 }
930 if reconnect {
933 params.insert("Attach".to_string(), Value::from("true"));
934 }
935 let url = format!(
936 "{}/marmot/research?authorization={}",
937 websocket(&crate::get_url()),
938 encode(&Value::Object(params).to_string())
939 );
940
941 let socket = match tokio_tungstenite::connect_async(url.as_str()).await {
942 Ok((socket, _)) => socket,
943 Err(_) if reconnect => {
944 return Err(Error::Message(
945 "Failed to connect to research run".to_string(),
946 ));
947 }
948 Err(_) => {
949 callbacks.on_error("WebSocket connection failed");
950 return Ok(None);
951 }
952 };
953 let (mut write, mut read) = socket.split();
954 let (outbox, mut queue) = mpsc::unbounded_channel::<Message>();
955 tokio::spawn(async move {
956 while let Some(message) = queue.recv().await {
957 if write.send(message).await.is_err() {
958 break;
959 }
960 }
961 let _ = write.close().await;
962 });
963 *self.ws.lock().unwrap_or_else(PoisonError::into_inner) = Some(outbox);
964 let close = || {
965 if let Some(outbox) = self
966 .ws
967 .lock()
968 .unwrap_or_else(PoisonError::into_inner)
969 .take()
970 {
971 let _ = outbox.send(Message::Close(None));
972 }
973 };
974
975 let mut attached = false;
976 let strings =
977 |value: &Value| -> Option<Vec<String>> { serde_json::from_value(value.clone()).ok() };
978 loop {
979 let packet = match read.next().await {
980 Some(Ok(Message::Text(raw))) => match serde_json::from_str::<Value>(raw.as_str()) {
981 Ok(packet) => packet,
982 Err(_) => continue,
983 },
984 Some(Ok(Message::Close(_))) | None => {
985 self.ws
986 .lock()
987 .unwrap_or_else(PoisonError::into_inner)
988 .take();
989 if reconnect && !attached {
990 return Err(Error::Message("No active research run".to_string()));
991 }
992 return Ok(None);
993 }
994 Some(Err(_)) => {
995 self.ws
996 .lock()
997 .unwrap_or_else(PoisonError::into_inner)
998 .take();
999 if reconnect && !attached {
1002 return Err(Error::Message(
1003 "Failed to connect to research run".to_string(),
1004 ));
1005 }
1006 callbacks.on_error("WebSocket connection failed");
1007 return Ok(None);
1008 }
1009 Some(Ok(_)) => continue,
1010 };
1011 match packet["event"].as_str().unwrap_or_default() {
1012 "connection" => {
1013 attached = packet["ongoing"] == true;
1014 callbacks.on_connection(attached);
1015 }
1016 "inactive" => {
1017 close();
1018 return Err(Error::Message("No active research run".to_string()));
1019 }
1020 "query" => callbacks.on_query(&text(&packet["query"])),
1021 "source" => callbacks.on_source(
1022 &text(&packet["url"]),
1023 &text(&packet["title"]),
1024 &text(&packet["contentType"]),
1025 ),
1026 "answer" => callbacks.on_answer(
1027 &text(&packet["field"]),
1028 &packet["value"],
1029 packet["converged"] == true,
1030 packet["answersCount"].as_u64().unwrap_or_default(),
1031 strings(&packet["sources"]).as_deref(),
1032 ),
1033 "nomination_accepted" => callbacks.on_nomination(
1034 &text(&packet["scope"]),
1035 &packet["row"],
1036 packet["committedCount"].as_u64().unwrap_or_default(),
1037 strings(&packet["sources"]).as_deref(),
1038 ),
1039 "supervisor_query" => callbacks.on_supervisor_query(&text(&packet["question"])),
1040 "supervisor_search" => callbacks.on_supervisor_search(&text(&packet["query"])),
1041 "supervisor_answer" => callbacks
1042 .on_supervisor_answer(&text(&packet["question"]), &text(&packet["answer"])),
1043 "supervisor_refused" => callbacks.on_supervisor_refused(&text(&packet["question"])),
1044 "machine" => {
1045 if let Ok(frame) = serde_json::from_value::<MachineFrame>(packet.clone()) {
1046 callbacks.on_machine(&frame);
1047 }
1048 }
1049 "compress" => callbacks.on_compress(),
1050 "info" => callbacks.on_info(&text(&packet["message"])),
1051 "done" => {
1052 let snapshot = Snapshot::new(
1053 &self.authorization,
1054 &self.workspace_id,
1055 &self.id,
1056 &text(&packet["snapshotId"]),
1057 Utc::now(),
1058 packet["valid"].as_bool().unwrap_or(true),
1059 self.group.as_ref(),
1060 );
1061 self.snapshots
1062 .lock()
1063 .unwrap_or_else(PoisonError::into_inner)
1064 .push(snapshot.clone());
1065 callbacks.on_done(&snapshot);
1066 close();
1067 return Ok(Some(snapshot));
1068 }
1069 "cancelled" => {
1070 callbacks.on_cancelled(
1071 serde_json::from_value(packet["reason"].clone())
1072 .unwrap_or(CancelReason::User),
1073 );
1074 close();
1075 return Ok(None);
1076 }
1077 "convergence_failed" => {
1078 let candidates: Vec<Candidate> =
1079 serde_json::from_value(packet["candidates"].clone()).unwrap_or_default();
1080 callbacks.on_convergence_failed(
1081 &text(&packet["field"]),
1082 &text(&packet["reason"]),
1083 &candidates,
1084 );
1085 close();
1086 return Ok(None);
1087 }
1088 "error" => {
1089 callbacks.on_error(&text(&packet["message"]));
1090 close();
1091 return Ok(None);
1092 }
1093 _ => {}
1094 }
1095 }
1096 }
1097
1098 pub async fn cancel(&self) -> Result<()> {
1102 if let Some(outbox) = self
1103 .ws
1104 .lock()
1105 .unwrap_or_else(PoisonError::into_inner)
1106 .as_ref()
1107 {
1108 let _ = outbox.send(Message::text(json!({ "event": "cancel" }).to_string()));
1109 }
1110 Ok(())
1111 }
1112
1113 async fn update(&self, field: &str, value: Value) -> Result<()> {
1114 let response = authorized(
1115 client().post(format!("{}/marmot/dataset/update", crate::get_url())),
1116 &self.authorization,
1117 self.group.as_ref(),
1118 )
1119 .json(&json!({ "workspaceId": self.workspace_id, "id": self.id, field: value }))
1120 .send()
1121 .await?;
1122 ok(response).await?;
1123 Ok(())
1124 }
1125
1126 pub async fn rename(&mut self, name: &str) -> Result<()> {
1127 self.update("name", Value::from(name)).await?;
1128 self.name = name.to_string();
1129 Ok(())
1130 }
1131
1132 pub async fn save_schema(&mut self, schema: Schema) -> Result<()> {
1136 self.update("schema", schema.serialize()).await?;
1137 self.schema = schema;
1138 Ok(())
1139 }
1140
1141 pub async fn migrate(
1155 &mut self,
1156 schema: Schema,
1157 seed_snapshot_id: Option<&str>,
1158 callbacks: &dyn ResearchCallbacks,
1159 options: ResearchOptions,
1160 ) -> Result<()> {
1161 let migrate =
1162 json!({ "seedSnapshotId": seed_snapshot_id, "schema": schema.serialize() }).to_string();
1163 let done = self
1164 .run_socket(vec![("Migrate", migrate)], callbacks, options, false)
1165 .await?;
1166 if done.is_some() {
1167 self.schema = schema;
1168 }
1169 Ok(())
1170 }
1171
1172 pub async fn set_termination(&mut self, termination: Vec<Termination>) -> Result<()> {
1173 self.update("termination", serde_json::to_value(&termination)?)
1174 .await?;
1175 self.termination = termination;
1176 Ok(())
1177 }
1178
1179 pub async fn set_content_types(&mut self, content_types: Vec<String>) -> Result<()> {
1180 self.update("contentTypes", json!(content_types)).await?;
1181 self.content_types = content_types;
1182 Ok(())
1183 }
1184
1185 pub async fn set_proxy(&mut self, proxy: ProxyMode) -> Result<()> {
1186 self.update("proxy", serde_json::to_value(proxy)?).await?;
1187 self.proxy = proxy;
1188 Ok(())
1189 }
1190
1191 pub async fn set_document_modes(&mut self, document_modes: Vec<DocumentMode>) -> Result<()> {
1192 self.update("documentModes", serde_json::to_value(&document_modes)?)
1193 .await?;
1194 self.document_modes = document_modes;
1195 Ok(())
1196 }
1197
1198 pub async fn remove(&self) -> Result<()> {
1199 let response = authorized(
1200 client().post(format!("{}/marmot/dataset/remove", crate::get_url())),
1201 &self.authorization,
1202 self.group.as_ref(),
1203 )
1204 .json(&json!({ "workspaceId": self.workspace_id, "id": self.id }))
1205 .send()
1206 .await?;
1207 ok(response).await?;
1208 Ok(())
1209 }
1210
1211 pub async fn data(&self, options: DataOptions) -> Result<Value> {
1214 let csv = options.format == Some(DataFormat::Csv);
1215 let response = authorized(
1216 client().get(dataset_data_url(&self.id, &options)),
1217 &self.authorization,
1218 None,
1219 )
1220 .send()
1221 .await?;
1222 data_body(response, csv).await
1223 }
1224
1225 pub fn data_url(&self, options: &DataOptions) -> String {
1227 dataset_data_url(&self.id, options)
1228 }
1229
1230 pub async fn add_rows(&self, rows: RowsOrCsv, options: AddRowsOptions) -> Result<PushResult> {
1234 let mut url = format!("{}/marmot/dataset/{}/rows", crate::get_url(), self.id);
1235 if let Some(source) = options.source.filter(|source| !source.is_empty()) {
1236 url.push_str(&format!("?source={}", encode(&source)));
1237 }
1238 let request = authorized(client().post(url), &self.authorization, None);
1239 let request = match rows {
1240 RowsOrCsv::Csv(csv) => request.header("Content-Type", "text/csv").body(csv),
1241 RowsOrCsv::Rows(rows) => request.json(&rows),
1242 };
1243 let response = request.send().await?;
1244 if !response.status().is_success() {
1245 let status = response.status().as_u16();
1246 let text = response.text().await.unwrap_or_default();
1247 let body: Value = serde_json::from_str(&text).unwrap_or(Value::Null);
1248 return Err(Error::PushRefused(PushRefused {
1249 status,
1250 message: body["message"].as_str().map(str::to_string).unwrap_or(text),
1251 invalid: PushResult::from_json(&body).invalid,
1252 }));
1253 }
1254 Ok(PushResult::from_json(&response.json().await?))
1255 }
1256
1257 pub async fn updates(&self, query: FeedQuery) -> Result<Vec<FeedRow>> {
1259 let response = authorized(
1260 client().get(format!(
1261 "{}/marmot/dataset/{}/updates?since={}&max={}",
1262 crate::get_url(),
1263 self.id,
1264 query.since,
1265 query.max
1266 )),
1267 &self.authorization,
1268 None,
1269 )
1270 .send()
1271 .await?;
1272 let json: Value = ok(response).await?.json().await?;
1273 Ok(json
1274 .as_array()
1275 .into_iter()
1276 .flatten()
1277 .map(FeedRow::from_json)
1278 .collect())
1279 }
1280
1281 pub fn live(&self, since: Option<u64>) -> BoxStream<'static, FeedRow> {
1286 let (rows, received) = mpsc::unbounded_channel::<FeedRow>();
1287 let dataset = self.clone();
1288 tokio::spawn(async move {
1289 let mut since = since.unwrap_or(0);
1290 while !rows.is_closed() {
1291 dataset.live_once(&mut since, &rows).await;
1292 if rows.is_closed() {
1293 break;
1294 }
1295 tokio::time::sleep(Duration::from_secs(2)).await;
1296 }
1297 });
1298 stream::unfold(received, |mut received| async move {
1299 received.recv().await.map(|row| (row, received))
1300 })
1301 .boxed()
1302 }
1303
1304 async fn live_once(&self, since: &mut u64, rows: &mpsc::UnboundedSender<FeedRow>) {
1305 let params = json!({
1306 "Authorization": match &self.authorization {
1307 CredentialsOrApiKey::Credentials(credentials) => format!("Token {}", credentials.token),
1308 CredentialsOrApiKey::ApiKey(key) => format!("ApiKey {}", key.id),
1309 },
1310 "Dataset": self.id,
1311 });
1312 let url = format!(
1313 "{}/marmot/dataset/live?authorization={}",
1314 websocket(&crate::get_url()),
1315 encode(¶ms.to_string())
1316 );
1317 let Ok((socket, _)) = tokio_tungstenite::connect_async(url.as_str()).await else {
1318 return;
1319 };
1320 let (mut write, mut read) = socket.split();
1321 let (held, mut pending) = mpsc::unbounded_channel::<FeedRow>();
1322 let (ready, is_ready) = oneshot::channel::<()>();
1323 let reader = tokio::spawn(async move {
1324 let mut ready = Some(ready);
1325 while let Some(Ok(message)) = read.next().await {
1326 let Message::Text(raw) = message else {
1327 if matches!(message, Message::Close(_)) {
1328 break;
1329 }
1330 continue;
1331 };
1332 let Ok(packet) = serde_json::from_str::<Value>(raw.as_str()) else {
1333 continue;
1334 };
1335 if packet["event"] == "ready" {
1336 if let Some(ready) = ready.take() {
1337 let _ = ready.send(());
1338 }
1339 } else if packet["event"] == "rows" {
1340 for row in packet["rows"].as_array().into_iter().flatten() {
1341 let _ = held.send(FeedRow::from_json(row));
1342 }
1343 }
1344 }
1345 });
1346
1347 if is_ready.await.is_ok() {
1348 loop {
1349 let Ok(page) = self
1350 .updates(FeedQuery {
1351 since: *since,
1352 max: FEED_PAGE,
1353 })
1354 .await
1355 else {
1356 break;
1357 };
1358 let count = page.len() as u64;
1359 for row in page {
1360 if row.seq <= *since {
1361 continue;
1362 }
1363 *since = row.seq;
1364 if rows.send(row).is_err() {
1365 reader.abort();
1366 let _ = write.close().await;
1367 return;
1368 }
1369 }
1370 if count < FEED_PAGE {
1371 break;
1372 }
1373 }
1374 loop {
1375 tokio::select! {
1376 row = pending.recv() => {
1377 let Some(row) = row else {
1378 break;
1379 };
1380 if row.seq <= *since {
1381 continue;
1382 }
1383 *since = row.seq;
1384 if rows.send(row).is_err() {
1385 break;
1386 }
1387 }
1388 _ = rows.closed() => break,
1389 }
1390 }
1391 }
1392 reader.abort();
1393 let _ = write.close().await;
1394 }
1395
1396 pub async fn scraper(&self) -> Result<Option<Scraper>> {
1398 get_scraper(&self.authorization, &self.id).await
1399 }
1400
1401 pub async fn keep_updated(&self, settings: &ScraperSettings) -> Result<Scraper> {
1403 create_scraper(&self.authorization, &self.id, settings).await
1404 }
1405
1406 pub async fn update_scraper(&self, settings: &ScraperSettings) -> Result<Scraper> {
1407 update_scraper(&self.authorization, &self.id, settings).await
1408 }
1409
1410 pub async fn rebuild_scraper(&self, note: Option<&str>) -> Result<()> {
1412 rebuild_scraper(&self.authorization, &self.id, note).await
1413 }
1414
1415 pub async fn stop_keeping_updated(&self) -> Result<()> {
1417 remove_scraper(&self.authorization, &self.id).await
1418 }
1419}
1420
1421#[derive(Debug, Clone, Default)]
1423pub struct GroupOptions {
1424 pub group: Option<Group>,
1425}
1426
1427pub async fn list_datasets(
1428 authorization: &CredentialsOrApiKey,
1429 workspace_id: &str,
1430 opts: GroupOptions,
1431) -> Result<Vec<Dataset>> {
1432 let response = authorized(
1433 client().post(format!("{}/marmot/dataset/list", crate::get_url())),
1434 authorization,
1435 opts.group.as_ref(),
1436 )
1437 .json(&json!({ "workspaceId": workspace_id }))
1438 .send()
1439 .await?;
1440 let data: Value = ok(response).await?.json().await?;
1441 data.as_array()
1442 .into_iter()
1443 .flatten()
1444 .map(|d| {
1445 let id = text(&d["id"]);
1446 let snapshots = d["snapshots"]
1447 .as_array()
1448 .into_iter()
1449 .flatten()
1450 .map(|s| {
1451 Snapshot::new(
1452 authorization,
1453 workspace_id,
1454 &id,
1455 &text(&s["id"]),
1456 crate::tickets::parse_date(&s["createdAt"]).unwrap_or_default(),
1457 s["valid"].as_bool().unwrap_or(true),
1458 opts.group.as_ref(),
1459 )
1460 })
1461 .collect();
1462 Ok(Dataset {
1463 workspace_id: workspace_id.to_string(),
1464 name: text(&d["name"]),
1465 schema: Schema::deserialize(&d["schema"])?,
1466 termination: serde_json::from_value(d["termination"].clone()).unwrap_or_default(),
1467 content_types: serde_json::from_value(d["contentTypes"].clone())
1468 .unwrap_or_default(),
1469 proxy: serde_json::from_value(d["proxy"].clone()).unwrap_or(ProxyMode::Datacenter),
1470 document_modes: serde_json::from_value(d["documentModes"].clone())
1471 .unwrap_or_else(|_| vec![DocumentMode::Extraction]),
1472 snapshots: Arc::new(Mutex::new(snapshots)),
1473 authorization: authorization.clone(),
1474 group: opts.group.clone(),
1475 ws: Arc::new(Mutex::new(None)),
1476 id,
1477 })
1478 })
1479 .collect()
1480}
1481
1482#[derive(Clone)]
1484pub struct CreateDatasetOptions {
1485 pub name: String,
1486 pub schema: Schema,
1487 pub termination: Option<Vec<Termination>>,
1488 pub content_types: Option<Vec<String>>,
1489 pub proxy: Option<ProxyMode>,
1490 pub document_modes: Option<Vec<DocumentMode>>,
1491}
1492
1493pub async fn create_dataset(
1494 authorization: &CredentialsOrApiKey,
1495 workspace_id: &str,
1496 options: &CreateDatasetOptions,
1497 opts: GroupOptions,
1498) -> Result<String> {
1499 let response = authorized(
1500 client().post(format!("{}/marmot/dataset/create", crate::get_url())),
1501 authorization,
1502 opts.group.as_ref(),
1503 )
1504 .json(&json!({
1505 "workspaceId": workspace_id,
1506 "name": options.name,
1507 "schema": options.schema.serialize(),
1508 "termination": options.termination.clone().unwrap_or_default(),
1509 "contentTypes": options.content_types.clone().unwrap_or_default(),
1510 "proxy": options.proxy.unwrap_or(ProxyMode::Datacenter),
1511 "documentModes": options.document_modes.clone().unwrap_or_else(|| vec![DocumentMode::Extraction]),
1512 }))
1513 .send()
1514 .await?;
1515 Ok(ok(response).await?.text().await?)
1516}
1517
1518#[derive(Debug, Clone)]
1519pub struct Workspace {
1520 pub id: String,
1521 pub name: String,
1522 pub description: String,
1523 pub created_at: i64,
1524 pub datasets: u64,
1525 pub files: u64,
1526
1527 authorization: CredentialsOrApiKey,
1528 group: Option<Group>,
1529}
1530
1531impl Workspace {
1532 fn opts(&self) -> GroupOptions {
1533 GroupOptions {
1534 group: self.group.clone(),
1535 }
1536 }
1537
1538 pub async fn list_datasets(&self) -> Result<Vec<Dataset>> {
1539 list_datasets(&self.authorization, &self.id, self.opts()).await
1540 }
1541
1542 pub async fn create_dataset(&self, options: &CreateDatasetOptions) -> Result<String> {
1543 create_dataset(&self.authorization, &self.id, options, self.opts()).await
1544 }
1545
1546 pub async fn rename(&mut self, name: &str) -> Result<()> {
1547 update_workspace(
1548 &self.authorization,
1549 &self.id,
1550 &PartialWorkspace {
1551 name: Some(name.to_string()),
1552 ..Default::default()
1553 },
1554 self.opts(),
1555 )
1556 .await?;
1557 self.name = name.to_string();
1558 Ok(())
1559 }
1560
1561 pub async fn set_description(&mut self, description: &str) -> Result<()> {
1562 update_workspace(
1563 &self.authorization,
1564 &self.id,
1565 &PartialWorkspace {
1566 description: Some(description.to_string()),
1567 ..Default::default()
1568 },
1569 self.opts(),
1570 )
1571 .await?;
1572 self.description = description.to_string();
1573 Ok(())
1574 }
1575
1576 async fn post(&self, route: &str, body: Value) -> Result<reqwest::Response> {
1577 let response = authorized(
1578 client().post(format!("{}{route}", crate::get_url())),
1579 &self.authorization,
1580 self.group.as_ref(),
1581 )
1582 .json(&body)
1583 .send()
1584 .await?;
1585 ok(response).await
1586 }
1587
1588 pub async fn remove(&self) -> Result<()> {
1589 self.post("/marmot/workspace/remove", json!({ "id": self.id }))
1590 .await?;
1591 Ok(())
1592 }
1593
1594 pub async fn get_files(&self) -> Result<Vec<WorkspaceFile>> {
1595 Ok(get_workspace(&self.authorization, &self.id, self.opts())
1596 .await?
1597 .files)
1598 }
1599
1600 pub async fn upload_file(
1601 &self,
1602 name: &str,
1603 content_type: &str,
1604 data: &[u8],
1605 ) -> Result<WorkspaceFile> {
1606 use base64::Engine;
1607 let data = base64::engine::general_purpose::STANDARD.encode(data);
1608 Ok(self
1609 .post(
1610 "/marmot/workspace/file/upload",
1611 json!({ "workspaceId": self.id, "name": name, "contentType": content_type, "data": data }),
1612 )
1613 .await?
1614 .json()
1615 .await?)
1616 }
1617
1618 pub async fn download_file(&self, file: &WorkspaceFile) -> Result<Vec<u8>> {
1619 Ok(self
1620 .post(
1621 "/marmot/workspace/file/get",
1622 json!({ "key": file.key, "contentType": file.content_type, "workspaceId": self.id }),
1623 )
1624 .await?
1625 .bytes()
1626 .await?
1627 .to_vec())
1628 }
1629
1630 pub async fn remove_file(&self, key: &str) -> Result<()> {
1631 self.post(
1632 "/marmot/workspace/file/remove",
1633 json!({ "workspaceId": self.id, "key": key }),
1634 )
1635 .await?;
1636 Ok(())
1637 }
1638}
1639
1640pub async fn list_workspaces(
1641 authorization: &CredentialsOrApiKey,
1642 opts: GroupOptions,
1643) -> Result<Vec<Workspace>> {
1644 let response = authorized(
1645 client().post(format!("{}/marmot/workspace/list", crate::get_url())),
1646 authorization,
1647 opts.group.as_ref(),
1648 )
1649 .header("Content-Type", "application/json")
1650 .send()
1651 .await?;
1652 let data: Value = ok(response).await?.json().await?;
1653 Ok(data
1654 .as_array()
1655 .into_iter()
1656 .flatten()
1657 .map(|w| Workspace {
1658 id: text(&w["id"]),
1659 name: text(&w["name"]),
1660 description: text(&w["description"]),
1661 created_at: w["createdAt"].as_f64().unwrap_or_default() as i64,
1662 datasets: w["datasets"].as_f64().unwrap_or_default() as u64,
1663 files: w["files"].as_f64().unwrap_or_default() as u64,
1664 authorization: authorization.clone(),
1665 group: opts.group.clone(),
1666 })
1667 .collect())
1668}
1669
1670#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
1672#[serde(rename_all = "camelCase")]
1673pub struct WorkspaceDetail {
1674 pub id: String,
1675 pub name: String,
1676 #[serde(default)]
1677 pub description: String,
1678 #[serde(default)]
1679 pub created_at: i64,
1680 #[serde(default)]
1681 pub files: Vec<WorkspaceFile>,
1682}
1683
1684pub async fn get_workspace(
1685 authorization: &CredentialsOrApiKey,
1686 id: &str,
1687 opts: GroupOptions,
1688) -> Result<WorkspaceDetail> {
1689 let response = authorized(
1690 client().post(format!("{}/marmot/workspace/get", crate::get_url())),
1691 authorization,
1692 opts.group.as_ref(),
1693 )
1694 .json(&json!({ "id": id }))
1695 .send()
1696 .await?;
1697 Ok(ok(response).await?.json().await?)
1698}
1699
1700pub async fn create_workspace(
1702 authorization: &CredentialsOrApiKey,
1703 name: &str,
1704 description: Option<&str>,
1705 opts: GroupOptions,
1706) -> Result<String> {
1707 let response = authorized(
1708 client().post(format!("{}/marmot/workspace/create", crate::get_url())),
1709 authorization,
1710 opts.group.as_ref(),
1711 )
1712 .json(&json!({ "name": name, "description": description.unwrap_or_default() }))
1713 .send()
1714 .await?;
1715 Ok(ok(response).await?.text().await?)
1716}
1717
1718pub async fn update_workspace(
1719 authorization: &CredentialsOrApiKey,
1720 id: &str,
1721 update: &PartialWorkspace,
1722 opts: GroupOptions,
1723) -> Result<()> {
1724 let mut body = Map::new();
1725 body.insert("id".to_string(), Value::from(id));
1726 if let Value::Object(fields) = serde_json::to_value(update)? {
1727 body.extend(fields);
1728 }
1729 let response = authorized(
1730 client().post(format!("{}/marmot/workspace/update", crate::get_url())),
1731 authorization,
1732 opts.group.as_ref(),
1733 )
1734 .json(&body)
1735 .send()
1736 .await?;
1737 ok(response).await?;
1738 Ok(())
1739}