1use std::ops::Deref;
2use std::sync::{Arc, Mutex, MutexGuard, PoisonError};
3use std::time::Duration;
4
5use chrono::{DateTime, Utc};
6use futures_util::stream::{self, BoxStream};
7use futures_util::{SinkExt, StreamExt};
8use serde::{Deserialize, Serialize};
9use serde_json::{Map, Value, json};
10use tokio::sync::{mpsc, oneshot};
11use tokio_tungstenite::tungstenite::Message;
12
13use super::scraper::{
14 Scraper, ScraperSettings, create_scraper, get_scraper, rebuild_scraper, remove_scraper,
15 update_scraper,
16};
17use super::workspace::PartialWorkspace;
18use crate::authentification::CredentialsOrApiKey;
19use crate::error::{Error, Result};
20use crate::groups::Group;
21use crate::http::{client, encode, ok, websocket};
22use crate::parrot::format::Format;
23use crate::parrot::generative::{Embedding, Text};
24use crate::shared::server::Presence;
25
26pub use super::browser::BrowserTool;
27pub use super::media::{DocumentMode, ProxyMode};
28pub use super::workspace::WorkspaceFile;
29
30#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
32pub struct ArrayBounds {
33 pub minimum: u64,
34 pub maximum: u64,
35}
36
37#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
39#[serde(untagged)]
40pub enum StringOrStrings {
41 String(String),
42 Strings(Vec<String>),
43}
44
45#[allow(clippy::large_enum_variant)]
48pub enum FormatOrSchemaLeafOrChildren {
49 Format(Format),
50 SchemaLeaf(Box<SchemaLeaf>),
51 Children(Vec<(String, SchemaLeaf)>),
52}
53
54pub struct SchemaLeaf {
56 pub value: FormatOrSchemaLeafOrChildren,
57 pub prompt: Option<String>,
58 pub sources: Option<u64>,
59 pub optional: Option<bool>,
60 pub array: Option<ArrayBounds>,
61
62 pub max_iterations: Option<u64>,
63 pub filters: Option<StringOrStrings>,
64}
65
66impl Clone for SchemaLeaf {
67 fn clone(&self) -> Self {
68 deserialize_leaf(&serialize_leaf(self)).expect("a leaf reads its own JSON")
69 }
70}
71
72fn insert(fields: &mut Map<String, Value>, key: &str, value: Option<Value>) {
73 if let Some(value) = value {
74 fields.insert(key.to_string(), value);
75 }
76}
77
78fn serialize_leaf(leaf: &SchemaLeaf) -> Value {
79 let mut fields = Map::new();
80 insert(&mut fields, "prompt", leaf.prompt.clone().map(Value::from));
81 insert(&mut fields, "sources", leaf.sources.map(Value::from));
82 insert(&mut fields, "optional", leaf.optional.map(Value::from));
83 insert(&mut fields, "array", leaf.array.map(|array| json!(array)));
84 insert(
85 &mut fields,
86 "maxIterations",
87 leaf.max_iterations.map(Value::from),
88 );
89 insert(
90 &mut fields,
91 "filters",
92 leaf.filters.as_ref().map(|filters| json!(filters)),
93 );
94 let value = match &leaf.value {
95 FormatOrSchemaLeafOrChildren::Format(format) => json!({ "question": format.to_json() }),
96 FormatOrSchemaLeafOrChildren::SchemaLeaf(leaf) => json!({ "schema": serialize_leaf(leaf) }),
97 FormatOrSchemaLeafOrChildren::Children(children) => {
98 let children: Map<String, Value> = children
99 .iter()
100 .map(|(key, leaf)| (key.clone(), serialize_leaf(leaf)))
101 .collect();
102 json!({ "children": children })
103 }
104 };
105 fields.insert("value".to_string(), value);
106 Value::Object(fields)
107}
108
109fn deserialize_leaf(data: &Value) -> Result<SchemaLeaf> {
110 let array = match &data["array"] {
111 Value::Null => None,
112 array => {
113 let bound = |key: &str| {
114 array[key]
115 .as_f64()
116 .filter(|bound| bound.fract() == 0.0 && *bound >= 0.0)
117 };
118 let Some(minimum) = bound("minimum") else {
119 return Err(Error::Message(format!(
120 "SchemaLeaf array.minimum must be a non-negative integer, got {}",
121 array["minimum"]
122 )));
123 };
124 let Some(maximum) = bound("maximum") else {
125 return Err(Error::Message(format!(
126 "SchemaLeaf array.maximum must be a non-negative integer, got {}",
127 array["maximum"]
128 )));
129 };
130 if minimum > maximum {
131 return Err(Error::Message(format!(
132 "SchemaLeaf array.minimum ({minimum}) cannot exceed array.maximum ({maximum})"
133 )));
134 }
135 Some(ArrayBounds {
136 minimum: minimum as u64,
137 maximum: maximum as u64,
138 })
139 }
140 };
141 let value = &data["value"];
142 let value = if value.get("question").is_some() {
143 FormatOrSchemaLeafOrChildren::Format(Format::from_json(&value["question"])?)
144 } else if let Some(children) = value.get("children") {
145 FormatOrSchemaLeafOrChildren::Children(
146 children
147 .as_object()
148 .into_iter()
149 .flatten()
150 .map(|(key, leaf)| Ok((key.clone(), deserialize_leaf(leaf)?)))
151 .collect::<Result<Vec<_>>>()?,
152 )
153 } else {
154 FormatOrSchemaLeafOrChildren::SchemaLeaf(Box::new(deserialize_leaf(&value["schema"])?))
155 };
156 Ok(SchemaLeaf {
157 value,
158 prompt: data["prompt"].as_str().map(str::to_string),
159 sources: data["sources"].as_u64(),
160 optional: data["optional"].as_bool(),
161 array,
162 max_iterations: data["maxIterations"].as_u64(),
163 filters: serde_json::from_value(data["filters"].clone()).ok(),
164 })
165}
166
167#[derive(Clone)]
169#[allow(clippy::large_enum_variant)]
170pub enum Schema {
171 Json(SchemaJSON),
172 Csv(SchemaCSV),
173}
174
175impl Schema {
176 pub fn r#type(&self) -> &'static str {
177 match self {
178 Schema::Json(_) => "json",
179 Schema::Csv(_) => "csv",
180 }
181 }
182
183 pub fn serialize(&self) -> Value {
184 match self {
185 Schema::Json(schema) => schema.serialize(),
186 Schema::Csv(schema) => schema.serialize(),
187 }
188 }
189
190 pub fn deserialize(data: &Value) -> Result<Schema> {
191 match data["type"].as_str() {
192 Some("json") => Ok(Schema::Json(SchemaJSON::new(deserialize_leaf(
193 &data["schema"],
194 )?))),
195 Some("csv") => {
196 let columns = data["schema"]
197 .as_object()
198 .into_iter()
199 .flatten()
200 .map(|(key, leaf)| Ok((key.clone(), deserialize_leaf(leaf)?)))
201 .collect::<Result<Vec<_>>>()?;
202 let bound = |key: &str| data[key].as_f64().unwrap_or(f64::NAN);
203 Ok(Schema::Csv(SchemaCSV::new(
204 columns,
205 bound("minimum"),
206 bound("maximum"),
207 data["prompt"].as_str(),
208 )?))
209 }
210 kind => Err(Error::Message(format!(
211 "Invalid schema type: {}",
212 kind.unwrap_or("undefined")
213 ))),
214 }
215 }
216}
217
218#[derive(Clone)]
219pub struct SchemaJSON {
220 pub schema: SchemaLeaf,
221}
222
223impl SchemaJSON {
224 pub fn new(schema: SchemaLeaf) -> SchemaJSON {
225 SchemaJSON { schema }
226 }
227
228 pub fn r#type(&self) -> &'static str {
229 "json"
230 }
231
232 pub fn serialize(&self) -> Value {
233 json!({ "type": "json", "schema": serialize_leaf(&self.schema) })
234 }
235}
236
237#[derive(Clone)]
238pub struct SchemaCSV {
239 pub schema: Vec<(String, SchemaLeaf)>,
242 pub minimum: u64,
243 pub maximum: u64,
244 pub prompt: String,
245}
246
247impl SchemaCSV {
248 pub fn new(
251 schema: Vec<(String, SchemaLeaf)>,
252 minimum: f64,
253 maximum: f64,
254 prompt: Option<&str>,
255 ) -> Result<SchemaCSV> {
256 if schema
257 .iter()
258 .any(|(_, leaf)| !matches!(leaf.value, FormatOrSchemaLeafOrChildren::Format(_)))
259 {
260 return Err(Error::Message(
261 "Nested schema are not supported for CSV".to_string(),
262 ));
263 }
264 let whole = |count: f64| count.fract() == 0.0 && count >= 0.0;
265 if !whole(minimum) {
266 return Err(Error::Message(format!(
267 "SchemaCSV minimum must be a non-negative integer, got {minimum}"
268 )));
269 }
270 if !whole(maximum) {
271 return Err(Error::Message(format!(
272 "SchemaCSV maximum must be a non-negative integer, got {maximum}"
273 )));
274 }
275 if minimum > maximum {
276 return Err(Error::Message(format!(
277 "SchemaCSV minimum ({minimum}) cannot exceed maximum ({maximum})"
278 )));
279 }
280 Ok(SchemaCSV {
281 schema,
282 minimum: minimum as u64,
283 maximum: maximum as u64,
284 prompt: prompt.unwrap_or_default().to_string(),
285 })
286 }
287
288 pub fn r#type(&self) -> &'static str {
289 "csv"
290 }
291
292 pub fn serialize(&self) -> Value {
293 let schema: Map<String, Value> = self
294 .schema
295 .iter()
296 .map(|(key, leaf)| (key.clone(), serialize_leaf(leaf)))
297 .collect();
298 json!({
299 "type": "csv",
300 "minimum": self.minimum,
301 "maximum": self.maximum,
302 "prompt": self.prompt,
303 "schema": schema,
304 })
305 }
306}
307
308fn format_signature(value: &FormatOrSchemaLeafOrChildren) -> String {
309 match value {
310 FormatOrSchemaLeafOrChildren::Format(format) => format!(
311 "format:{}",
312 format.to_json()["type"].as_str().unwrap_or("unknown")
313 ),
314 _ => "composite".to_string(),
315 }
316}
317
318fn set(out: &mut Vec<(String, String)>, path: String, signature: String) {
319 match out.iter_mut().find(|(known, _)| *known == path) {
320 Some(entry) => entry.1 = signature,
321 None => out.push((path, signature)),
322 }
323}
324
325fn flatten_leaf_paths(leaf: &SchemaLeaf, path: &str, out: &mut Vec<(String, String)>) {
326 if leaf.array.is_some() {
327 if let FormatOrSchemaLeafOrChildren::SchemaLeaf(inner) = &leaf.value {
330 flatten_leaf_paths(inner, &format!("{path}[]"), out);
331 }
332 return;
333 }
334 match &leaf.value {
335 FormatOrSchemaLeafOrChildren::Format(_) => {
336 let path = if path.is_empty() { "value" } else { path };
337 set(out, path.to_string(), format_signature(&leaf.value));
338 }
339 FormatOrSchemaLeafOrChildren::SchemaLeaf(inner) => flatten_leaf_paths(inner, path, out),
341 FormatOrSchemaLeafOrChildren::Children(children) => {
342 for (key, child) in children {
343 let child_path = if path.is_empty() {
344 key.clone()
345 } else {
346 format!("{path}.{key}")
347 };
348 flatten_leaf_paths(child, &child_path, out);
349 }
350 }
351 }
352}
353
354pub fn schema_paths(schema: &Schema) -> Vec<(String, String)> {
359 let mut out = Vec::new();
360 match schema {
361 Schema::Csv(schema) => {
362 for (key, leaf) in &schema.schema {
363 set(&mut out, key.clone(), format_signature(&leaf.value));
364 }
365 }
366 Schema::Json(schema) => flatten_leaf_paths(&schema.schema, "", &mut out),
367 }
368 out
369}
370
371#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
373pub struct SchemaDiff {
374 pub added: Vec<String>,
375 pub dropped: Vec<String>,
376}
377
378pub fn diff_schema(baseline: &Schema, target: &Schema) -> SchemaDiff {
383 let a = schema_paths(baseline);
384 let b = schema_paths(target);
385 let lookup = |paths: &[(String, String)], path: &str| {
386 paths
387 .iter()
388 .find(|(known, _)| known == path)
389 .map(|(_, signature)| signature.clone())
390 };
391 SchemaDiff {
392 added: b
393 .iter()
394 .filter(|(path, signature)| lookup(&a, path).as_ref() != Some(signature))
395 .map(|(path, _)| path.clone())
396 .collect(),
397 dropped: a
398 .iter()
399 .filter(|(path, signature)| lookup(&b, path).as_ref() != Some(signature))
400 .map(|(path, _)| path.clone())
401 .collect(),
402 }
403}
404
405#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
407pub struct Termination {
408 pub r#type: String,
409 #[serde(flatten)]
410 pub fields: Map<String, Value>,
411}
412
413#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
414pub struct TokenTermination {
415 #[serde(flatten)]
416 pub termination: Termination,
417 pub max: f64,
418}
419
420impl Deref for TokenTermination {
421 type Target = Termination;
422 fn deref(&self) -> &Termination {
423 &self.termination
424 }
425}
426
427#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
428pub struct TimeTermination {
429 #[serde(flatten)]
430 pub termination: Termination,
431 pub max: f64,
432}
433
434impl Deref for TimeTermination {
435 type Target = Termination;
436 fn deref(&self) -> &Termination {
437 &self.termination
438 }
439}
440
441#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
442pub struct CostTermination {
443 #[serde(flatten)]
444 pub termination: Termination,
445 pub max: f64,
446}
447
448impl Deref for CostTermination {
449 type Target = Termination;
450 fn deref(&self) -> &Termination {
451 &self.termination
452 }
453}
454
455#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Hash, Serialize, Deserialize)]
457#[serde(rename_all = "lowercase")]
458pub enum DataFormat {
459 Csv,
460 #[default]
461 Json,
462}
463
464#[derive(Debug, Clone, Default, PartialEq, Eq)]
465pub struct DataOptions {
466 pub format: Option<DataFormat>,
467 pub snapshot: Option<String>,
469 pub sources: Option<bool>,
471}
472
473#[derive(Debug, Clone, PartialEq, Serialize)]
475pub struct FeedRow {
476 pub seq: u64,
477 pub value: Value,
478 pub sources: Value,
479 pub created: DateTime<Utc>,
480}
481
482impl FeedRow {
483 pub fn from_json(json: &Value) -> FeedRow {
484 FeedRow {
485 seq: json["seq"].as_f64().unwrap_or_default() as u64,
486 value: json["value"].clone(),
487 sources: json["sources"].clone(),
488 created: crate::tickets::parse_date(&json["created"]).unwrap_or_default(),
489 }
490 }
491}
492
493#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
495pub struct InvalidRow {
496 pub index: u64,
497 pub reason: String,
498}
499
500#[derive(Debug, Clone, Default, PartialEq, Serialize)]
502pub struct PushResult {
503 pub added: u64,
504 pub first: Option<f64>,
505 pub last: Option<f64>,
506 pub invalid: Vec<InvalidRow>,
507}
508
509impl PushResult {
510 pub fn from_json(json: &Value) -> PushResult {
511 PushResult {
512 added: json["added"].as_f64().unwrap_or_default() as u64,
513 first: json["first"].as_f64(),
514 last: json["last"].as_f64(),
515 invalid: json["invalid"]
516 .as_array()
517 .into_iter()
518 .flatten()
519 .map(|row| InvalidRow {
520 index: row["index"].as_f64().unwrap_or_default() as u64,
521 reason: match &row["reason"] {
522 Value::String(reason) => reason.clone(),
523 reason => reason.to_string(),
524 },
525 })
526 .collect(),
527 }
528 }
529}
530
531#[derive(Debug, Clone, PartialEq, Eq)]
533pub struct PushRefused {
534 pub status: u16,
535 pub message: String,
536 pub invalid: Vec<InvalidRow>,
537}
538
539const FEED_PAGE: u64 = 500;
540
541fn dataset_data_url(dataset_id: &str, options: &DataOptions) -> String {
542 let mut query = format!(
543 "format={}",
544 match options.format.unwrap_or_default() {
545 DataFormat::Csv => "csv",
546 DataFormat::Json => "json",
547 }
548 );
549 if let Some(snapshot) = &options.snapshot {
550 query.push_str(&format!("&snapshot={}", encode(snapshot)));
551 }
552 if options.sources == Some(true) {
553 query.push_str("&sources=true");
554 }
555 format!(
556 "{}/marmot/dataset/{dataset_id}/data?{query}",
557 crate::get_url()
558 )
559}
560
561fn authorized(
562 request: reqwest::RequestBuilder,
563 authorization: &CredentialsOrApiKey,
564 group: Option<&Group>,
565) -> reqwest::RequestBuilder {
566 let request = request.header(
567 "Authorization",
568 match authorization {
569 CredentialsOrApiKey::Credentials(credentials) => format!("Token {}", credentials.token),
570 CredentialsOrApiKey::ApiKey(key) => format!("ApiKey {}", key.id),
571 },
572 );
573 match group {
574 Some(group) => request.header("Group", &group.id),
575 None => request,
576 }
577}
578
579async fn data_body(response: reqwest::Response, csv: bool) -> Result<Value> {
580 let response = ok(response).await?;
581 Ok(if csv {
582 Value::String(response.text().await?)
583 } else {
584 response.json().await?
585 })
586}
587
588#[derive(Debug, Clone, PartialEq)]
591pub enum SnapshotData {
592 Json(Value),
593 Text(String),
594 Bytes(Vec<u8>),
595}
596
597#[derive(Debug, Clone, Default)]
599pub struct SnapshotDataOptions {
600 pub format: Option<DataFormat>,
601 pub sources: Option<bool>,
602}
603
604#[derive(Default)]
605struct SnapshotCache {
606 schema: Option<Schema>,
607 data: Option<SnapshotData>,
608}
609
610#[derive(Clone)]
611pub struct Snapshot {
612 pub id: String,
613 pub created_at: DateTime<Utc>,
614
615 pub valid: bool,
616
617 authorization: CredentialsOrApiKey,
618 workspace_id: String,
619 dataset_id: String,
620 group: Option<Group>,
621 cache: Arc<Mutex<SnapshotCache>>,
622}
623
624impl Snapshot {
625 pub fn new(
626 authorization: &CredentialsOrApiKey,
627 workspace_id: &str,
628 dataset_id: &str,
629 id: &str,
630 created_at: DateTime<Utc>,
631 valid: bool,
632 group: Option<&Group>,
633 ) -> Snapshot {
634 Snapshot {
635 id: id.to_string(),
636 created_at,
637 valid,
638 authorization: authorization.clone(),
639 workspace_id: workspace_id.to_string(),
640 dataset_id: dataset_id.to_string(),
641 group: group.cloned(),
642 cache: Arc::new(Mutex::new(SnapshotCache::default())),
643 }
644 }
645
646 fn cache(&self) -> MutexGuard<'_, SnapshotCache> {
647 self.cache.lock().unwrap_or_else(PoisonError::into_inner)
648 }
649
650 fn url(&self, tail: &str) -> String {
651 format!(
652 "{}/marmot/snapshot/{}/{}/{}{tail}",
653 crate::get_url(),
654 self.workspace_id,
655 self.dataset_id,
656 self.id
657 )
658 }
659
660 async fn get(&self, tail: &str) -> Result<reqwest::Response> {
661 let response = authorized(
662 client().get(self.url(tail)),
663 &self.authorization,
664 self.group.as_ref(),
665 )
666 .send()
667 .await?;
668 ok(response).await
669 }
670
671 pub async fn download(&self, assets: bool, revalidate: bool) -> Result<SnapshotData> {
672 if !assets
673 && !revalidate
674 && let Some(data) = self.cache().data.clone()
675 {
676 return Ok(data);
677 }
678 let response = self.get(&format!("/?assets={assets}")).await?;
679 if assets {
680 return Ok(SnapshotData::Bytes(response.bytes().await?.to_vec()));
681 }
682 let json = response
683 .headers()
684 .get("content-type")
685 .and_then(|value| value.to_str().ok())
686 .is_some_and(|value| value.contains("application/json"));
687 let data = if json {
688 SnapshotData::Json(response.json().await?)
689 } else {
690 SnapshotData::Text(response.text().await?)
691 };
692 self.cache().data = Some(data.clone());
693 Ok(data)
694 }
695
696 pub async fn data(&self, options: SnapshotDataOptions) -> Result<Value> {
698 let csv = options.format == Some(DataFormat::Csv);
699 let url = dataset_data_url(
700 &self.dataset_id,
701 &DataOptions {
702 format: options.format,
703 snapshot: Some(self.id.clone()),
704 sources: options.sources,
705 },
706 );
707 let response = authorized(client().get(url), &self.authorization, None)
708 .send()
709 .await?;
710 data_body(response, csv).await
711 }
712
713 pub async fn download_schema(&self, revalidate: bool) -> Result<Schema> {
714 if !revalidate && let Some(schema) = self.cache().schema.clone() {
715 return Ok(schema);
716 }
717 let json: Value = self.get("/schema").await?.json().await?;
718 let schema = Schema::deserialize(&json)?;
719 self.cache().schema = Some(schema.clone());
720 Ok(schema)
721 }
722
723 pub async fn download_log(&self) -> Result<Vec<u8>> {
724 Ok(self.get("/log").await?.bytes().await?.to_vec())
725 }
726
727 pub async fn download_embeddings(&self) -> Result<Vec<u8>> {
728 Ok(self.get("/embeddings").await?.bytes().await?.to_vec())
729 }
730
731 pub async fn delete(&self) -> Result<()> {
732 let response = authorized(
733 client().delete(self.url("")),
734 &self.authorization,
735 self.group.as_ref(),
736 )
737 .send()
738 .await?;
739 ok(response).await?;
740 Ok(())
741 }
742}
743
744#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
746#[serde(rename_all = "lowercase")]
747pub enum BrowserMode {
748 Render,
749 Virtualized,
750}
751
752#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
754pub struct BrowserFrame {
755 pub url: String,
756 pub reference: String,
757 pub code: String,
758 pub mode: BrowserMode,
759}
760
761#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
763#[serde(rename_all = "lowercase")]
764pub enum CancelReason {
765 User,
766 Tokens,
767 Time,
768 Cost,
769}
770
771#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
773pub struct Candidate {
774 pub value: Value,
775 pub source: String,
776}
777
778pub trait ResearchCallbacks: Send + Sync {
781 fn on_connection(&self, _ongoing: bool) {}
782 fn on_query(&self, _query: &str) {}
783 fn on_source(&self, _url: &str, _title: &str, _content_type: &str) {}
784 fn on_answer(
785 &self,
786 _field: &str,
787 _value: &Value,
788 _converged: bool,
789 _answers_count: u64,
790 _sources: Option<&[String]>,
791 ) {
792 }
793 fn on_nomination(
797 &self,
798 _scope: &str,
799 _row: &Value,
800 _committed_count: u64,
801 _sources: Option<&[String]>,
802 ) {
803 }
804 fn on_supervisor_query(&self, _question: &str) {}
805 fn on_supervisor_search(&self, _query: &str) {}
806 fn on_supervisor_answer(&self, _question: &str, _answer: &str) {}
807 fn on_supervisor_refused(&self, _question: &str) {}
808 fn on_browser(&self, _frame: &BrowserFrame) {}
810 fn on_browser_step(&self, _url: &str, _step: u64, _action: &BrowserTool) {}
811 fn on_compress(&self) {}
812 fn on_info(&self, _message: &str) {}
813 fn on_done(&self, _snapshot: &Snapshot) {}
814 fn on_cancelled(&self, _reason: CancelReason) {}
815 fn on_convergence_failed(&self, _field: &str, _reason: &str, _candidates: &[Candidate]) {}
816 fn on_error(&self, _message: &str) {}
817}
818
819impl ResearchCallbacks for () {}
820
821#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
822pub struct ResearchOptions {
823 pub text: Option<Text>,
824 pub embedding: Option<Embedding>,
825}
826
827#[derive(Debug, Clone, PartialEq)]
829pub enum RowsOrCsv {
830 Rows(Vec<Value>),
831 Csv(String),
832}
833
834#[derive(Debug, Clone, Default)]
836pub struct AddRowsOptions {
837 pub source: Option<String>,
838}
839
840#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
842pub struct FeedQuery {
843 pub since: u64,
844 pub max: u64,
845}
846
847fn text(value: &Value) -> String {
848 match value {
849 Value::Null => String::new(),
850 Value::String(text) => text.clone(),
851 value => value.to_string(),
852 }
853}
854
855#[derive(Clone)]
856pub struct Dataset {
857 pub id: String,
858 pub workspace_id: String,
859 pub name: String,
860 pub schema: Schema,
861 pub termination: Vec<Termination>,
862 pub content_types: Vec<String>,
863 pub proxy: ProxyMode,
864 pub document_modes: Vec<DocumentMode>,
865 snapshots: Arc<Mutex<Vec<Snapshot>>>,
866
867 authorization: CredentialsOrApiKey,
868 group: Option<Group>,
869 ws: Arc<Mutex<Option<mpsc::UnboundedSender<Message>>>>,
870}
871
872impl Dataset {
873 pub fn snapshots(&self) -> Vec<Snapshot> {
874 self.snapshots
875 .lock()
876 .unwrap_or_else(PoisonError::into_inner)
877 .clone()
878 }
879
880 pub async fn research(
881 &self,
882 callbacks: &dyn ResearchCallbacks,
883 options: ResearchOptions,
884 ) -> Result<()> {
885 self.run_socket(Vec::new(), callbacks, options, false)
886 .await
887 .map(|_| ())
888 }
889
890 pub async fn join_browser(
892 &self,
893 access_code: &str,
894 ) -> Result<(super::browser::BrowserRoom, Presence)> {
895 super::browser::BrowserRoom::join(&self.authorization, access_code, Default::default())
896 .await
897 }
898
899 pub async fn reconnect(
907 &self,
908 callbacks: &dyn ResearchCallbacks,
909 options: ResearchOptions,
910 ) -> Result<()> {
911 self.run_socket(Vec::new(), callbacks, options, true)
912 .await
913 .map(|_| ())
914 }
915
916 async fn run_socket(
917 &self,
918 extra: Vec<(&str, String)>,
919 callbacks: &dyn ResearchCallbacks,
920 options: ResearchOptions,
921 reconnect: bool,
922 ) -> Result<Option<Snapshot>> {
923 let mut params = Map::new();
924 params.insert(
925 "Authorization".to_string(),
926 Value::from(match &self.authorization {
927 CredentialsOrApiKey::Credentials(credentials) => {
928 format!("Token {}", credentials.token)
929 }
930 CredentialsOrApiKey::ApiKey(key) => format!("ApiKey {}", key.id),
931 }),
932 );
933 if let Some(group) = &self.group {
934 params.insert("Group".to_string(), Value::from(group.id.as_str()));
935 }
936 params.insert(
937 "Workspace".to_string(),
938 Value::from(self.workspace_id.as_str()),
939 );
940 params.insert("Dataset".to_string(), Value::from(self.id.as_str()));
941 for (key, value) in extra {
942 params.insert(key.to_string(), Value::from(value));
943 }
944 if let Some(model) = options.text {
945 params.insert("Text".to_string(), Value::from(model.as_str()));
946 }
947 if let Some(model) = options.embedding {
948 params.insert("Embedding".to_string(), Value::from(model.as_str()));
949 }
950 if reconnect {
953 params.insert("Attach".to_string(), Value::from("true"));
954 }
955 let url = format!(
956 "{}/marmot/research?authorization={}",
957 websocket(&crate::get_url()),
958 encode(&Value::Object(params).to_string())
959 );
960
961 let socket = match tokio_tungstenite::connect_async(url.as_str()).await {
962 Ok((socket, _)) => socket,
963 Err(_) if reconnect => {
964 return Err(Error::Message(
965 "Failed to connect to research run".to_string(),
966 ));
967 }
968 Err(_) => {
969 callbacks.on_error("WebSocket connection failed");
970 return Ok(None);
971 }
972 };
973 let (mut write, mut read) = socket.split();
974 let (outbox, mut queue) = mpsc::unbounded_channel::<Message>();
975 tokio::spawn(async move {
976 while let Some(message) = queue.recv().await {
977 if write.send(message).await.is_err() {
978 break;
979 }
980 }
981 let _ = write.close().await;
982 });
983 *self.ws.lock().unwrap_or_else(PoisonError::into_inner) = Some(outbox);
984 let close = || {
985 if let Some(outbox) = self
986 .ws
987 .lock()
988 .unwrap_or_else(PoisonError::into_inner)
989 .take()
990 {
991 let _ = outbox.send(Message::Close(None));
992 }
993 };
994
995 let mut attached = false;
996 let strings =
997 |value: &Value| -> Option<Vec<String>> { serde_json::from_value(value.clone()).ok() };
998 loop {
999 let packet = match read.next().await {
1000 Some(Ok(Message::Text(raw))) => match serde_json::from_str::<Value>(raw.as_str()) {
1001 Ok(packet) => packet,
1002 Err(_) => continue,
1003 },
1004 Some(Ok(Message::Close(_))) | None => {
1005 self.ws
1006 .lock()
1007 .unwrap_or_else(PoisonError::into_inner)
1008 .take();
1009 if reconnect && !attached {
1010 return Err(Error::Message("No active research run".to_string()));
1011 }
1012 return Ok(None);
1013 }
1014 Some(Err(_)) => {
1015 self.ws
1016 .lock()
1017 .unwrap_or_else(PoisonError::into_inner)
1018 .take();
1019 if reconnect && !attached {
1022 return Err(Error::Message(
1023 "Failed to connect to research run".to_string(),
1024 ));
1025 }
1026 callbacks.on_error("WebSocket connection failed");
1027 return Ok(None);
1028 }
1029 Some(Ok(_)) => continue,
1030 };
1031 match packet["event"].as_str().unwrap_or_default() {
1032 "connection" => {
1033 attached = packet["ongoing"] == true;
1034 callbacks.on_connection(attached);
1035 }
1036 "inactive" => {
1037 close();
1038 return Err(Error::Message("No active research run".to_string()));
1039 }
1040 "query" => callbacks.on_query(&text(&packet["query"])),
1041 "source" => callbacks.on_source(
1042 &text(&packet["url"]),
1043 &text(&packet["title"]),
1044 &text(&packet["contentType"]),
1045 ),
1046 "answer" => callbacks.on_answer(
1047 &text(&packet["field"]),
1048 &packet["value"],
1049 packet["converged"] == true,
1050 packet["answersCount"].as_u64().unwrap_or_default(),
1051 strings(&packet["sources"]).as_deref(),
1052 ),
1053 "nomination_accepted" => callbacks.on_nomination(
1054 &text(&packet["scope"]),
1055 &packet["row"],
1056 packet["committedCount"].as_u64().unwrap_or_default(),
1057 strings(&packet["sources"]).as_deref(),
1058 ),
1059 "supervisor_query" => callbacks.on_supervisor_query(&text(&packet["question"])),
1060 "supervisor_search" => callbacks.on_supervisor_search(&text(&packet["query"])),
1061 "supervisor_answer" => callbacks
1062 .on_supervisor_answer(&text(&packet["question"]), &text(&packet["answer"])),
1063 "supervisor_refused" => callbacks.on_supervisor_refused(&text(&packet["question"])),
1064 "browser" => {
1065 if let Ok(frame) = serde_json::from_value::<BrowserFrame>(packet.clone()) {
1066 callbacks.on_browser(&frame);
1067 }
1068 }
1069 "browser_step" => {
1070 if let Ok(action) =
1071 serde_json::from_value::<BrowserTool>(packet["action"].clone())
1072 {
1073 callbacks.on_browser_step(
1074 &text(&packet["url"]),
1075 packet["step"].as_u64().unwrap_or_default(),
1076 &action,
1077 );
1078 }
1079 }
1080 "compress" => callbacks.on_compress(),
1081 "info" => callbacks.on_info(&text(&packet["message"])),
1082 "done" => {
1083 let snapshot = Snapshot::new(
1084 &self.authorization,
1085 &self.workspace_id,
1086 &self.id,
1087 &text(&packet["snapshotId"]),
1088 Utc::now(),
1089 packet["valid"].as_bool().unwrap_or(true),
1090 self.group.as_ref(),
1091 );
1092 self.snapshots
1093 .lock()
1094 .unwrap_or_else(PoisonError::into_inner)
1095 .push(snapshot.clone());
1096 callbacks.on_done(&snapshot);
1097 close();
1098 return Ok(Some(snapshot));
1099 }
1100 "cancelled" => {
1101 callbacks.on_cancelled(
1102 serde_json::from_value(packet["reason"].clone())
1103 .unwrap_or(CancelReason::User),
1104 );
1105 close();
1106 return Ok(None);
1107 }
1108 "convergence_failed" => {
1109 let candidates: Vec<Candidate> =
1110 serde_json::from_value(packet["candidates"].clone()).unwrap_or_default();
1111 callbacks.on_convergence_failed(
1112 &text(&packet["field"]),
1113 &text(&packet["reason"]),
1114 &candidates,
1115 );
1116 close();
1117 return Ok(None);
1118 }
1119 "error" => {
1120 callbacks.on_error(&text(&packet["message"]));
1121 close();
1122 return Ok(None);
1123 }
1124 _ => {}
1125 }
1126 }
1127 }
1128
1129 pub async fn cancel(&self) -> Result<()> {
1133 if let Some(outbox) = self
1134 .ws
1135 .lock()
1136 .unwrap_or_else(PoisonError::into_inner)
1137 .as_ref()
1138 {
1139 let _ = outbox.send(Message::text(json!({ "event": "cancel" }).to_string()));
1140 }
1141 Ok(())
1142 }
1143
1144 async fn update(&self, field: &str, value: Value) -> Result<()> {
1145 let response = authorized(
1146 client().post(format!("{}/marmot/dataset/update", crate::get_url())),
1147 &self.authorization,
1148 self.group.as_ref(),
1149 )
1150 .json(&json!({ "workspaceId": self.workspace_id, "id": self.id, field: value }))
1151 .send()
1152 .await?;
1153 ok(response).await?;
1154 Ok(())
1155 }
1156
1157 pub async fn rename(&mut self, name: &str) -> Result<()> {
1158 self.update("name", Value::from(name)).await?;
1159 self.name = name.to_string();
1160 Ok(())
1161 }
1162
1163 pub async fn save_schema(&mut self, schema: Schema) -> Result<()> {
1167 self.update("schema", schema.serialize()).await?;
1168 self.schema = schema;
1169 Ok(())
1170 }
1171
1172 pub async fn migrate(
1186 &mut self,
1187 schema: Schema,
1188 seed_snapshot_id: Option<&str>,
1189 callbacks: &dyn ResearchCallbacks,
1190 options: ResearchOptions,
1191 ) -> Result<()> {
1192 let migrate =
1193 json!({ "seedSnapshotId": seed_snapshot_id, "schema": schema.serialize() }).to_string();
1194 let done = self
1195 .run_socket(vec![("Migrate", migrate)], callbacks, options, false)
1196 .await?;
1197 if done.is_some() {
1198 self.schema = schema;
1199 }
1200 Ok(())
1201 }
1202
1203 pub async fn set_termination(&mut self, termination: Vec<Termination>) -> Result<()> {
1204 self.update("termination", serde_json::to_value(&termination)?)
1205 .await?;
1206 self.termination = termination;
1207 Ok(())
1208 }
1209
1210 pub async fn set_content_types(&mut self, content_types: Vec<String>) -> Result<()> {
1211 self.update("contentTypes", json!(content_types)).await?;
1212 self.content_types = content_types;
1213 Ok(())
1214 }
1215
1216 pub async fn set_proxy(&mut self, proxy: ProxyMode) -> Result<()> {
1217 self.update("proxy", serde_json::to_value(proxy)?).await?;
1218 self.proxy = proxy;
1219 Ok(())
1220 }
1221
1222 pub async fn set_document_modes(&mut self, document_modes: Vec<DocumentMode>) -> Result<()> {
1223 self.update("documentModes", serde_json::to_value(&document_modes)?)
1224 .await?;
1225 self.document_modes = document_modes;
1226 Ok(())
1227 }
1228
1229 pub async fn remove(&self) -> Result<()> {
1230 let response = authorized(
1231 client().post(format!("{}/marmot/dataset/remove", crate::get_url())),
1232 &self.authorization,
1233 self.group.as_ref(),
1234 )
1235 .json(&json!({ "workspaceId": self.workspace_id, "id": self.id }))
1236 .send()
1237 .await?;
1238 ok(response).await?;
1239 Ok(())
1240 }
1241
1242 pub async fn data(&self, options: DataOptions) -> Result<Value> {
1245 let csv = options.format == Some(DataFormat::Csv);
1246 let response = authorized(
1247 client().get(dataset_data_url(&self.id, &options)),
1248 &self.authorization,
1249 None,
1250 )
1251 .send()
1252 .await?;
1253 data_body(response, csv).await
1254 }
1255
1256 pub fn data_url(&self, options: &DataOptions) -> String {
1258 dataset_data_url(&self.id, options)
1259 }
1260
1261 pub async fn add_rows(&self, rows: RowsOrCsv, options: AddRowsOptions) -> Result<PushResult> {
1265 let mut url = format!("{}/marmot/dataset/{}/rows", crate::get_url(), self.id);
1266 if let Some(source) = options.source.filter(|source| !source.is_empty()) {
1267 url.push_str(&format!("?source={}", encode(&source)));
1268 }
1269 let request = authorized(client().post(url), &self.authorization, None);
1270 let request = match rows {
1271 RowsOrCsv::Csv(csv) => request.header("Content-Type", "text/csv").body(csv),
1272 RowsOrCsv::Rows(rows) => request.json(&rows),
1273 };
1274 let response = request.send().await?;
1275 if !response.status().is_success() {
1276 let status = response.status().as_u16();
1277 let text = response.text().await.unwrap_or_default();
1278 let body: Value = serde_json::from_str(&text).unwrap_or(Value::Null);
1279 return Err(Error::PushRefused(PushRefused {
1280 status,
1281 message: body["message"].as_str().map(str::to_string).unwrap_or(text),
1282 invalid: PushResult::from_json(&body).invalid,
1283 }));
1284 }
1285 Ok(PushResult::from_json(&response.json().await?))
1286 }
1287
1288 pub async fn updates(&self, query: FeedQuery) -> Result<Vec<FeedRow>> {
1290 let response = authorized(
1291 client().get(format!(
1292 "{}/marmot/dataset/{}/updates?since={}&max={}",
1293 crate::get_url(),
1294 self.id,
1295 query.since,
1296 query.max
1297 )),
1298 &self.authorization,
1299 None,
1300 )
1301 .send()
1302 .await?;
1303 let json: Value = ok(response).await?.json().await?;
1304 Ok(json
1305 .as_array()
1306 .into_iter()
1307 .flatten()
1308 .map(FeedRow::from_json)
1309 .collect())
1310 }
1311
1312 pub fn live(&self, since: Option<u64>) -> BoxStream<'static, FeedRow> {
1317 let (rows, received) = mpsc::unbounded_channel::<FeedRow>();
1318 let dataset = self.clone();
1319 tokio::spawn(async move {
1320 let mut since = since.unwrap_or(0);
1321 while !rows.is_closed() {
1322 dataset.live_once(&mut since, &rows).await;
1323 if rows.is_closed() {
1324 break;
1325 }
1326 tokio::time::sleep(Duration::from_secs(2)).await;
1327 }
1328 });
1329 stream::unfold(received, |mut received| async move {
1330 received.recv().await.map(|row| (row, received))
1331 })
1332 .boxed()
1333 }
1334
1335 async fn live_once(&self, since: &mut u64, rows: &mpsc::UnboundedSender<FeedRow>) {
1336 let params = json!({
1337 "Authorization": match &self.authorization {
1338 CredentialsOrApiKey::Credentials(credentials) => format!("Token {}", credentials.token),
1339 CredentialsOrApiKey::ApiKey(key) => format!("ApiKey {}", key.id),
1340 },
1341 "Dataset": self.id,
1342 });
1343 let url = format!(
1344 "{}/marmot/dataset/live?authorization={}",
1345 websocket(&crate::get_url()),
1346 encode(¶ms.to_string())
1347 );
1348 let Ok((socket, _)) = tokio_tungstenite::connect_async(url.as_str()).await else {
1349 return;
1350 };
1351 let (mut write, mut read) = socket.split();
1352 let (held, mut pending) = mpsc::unbounded_channel::<FeedRow>();
1353 let (ready, is_ready) = oneshot::channel::<()>();
1354 let reader = tokio::spawn(async move {
1355 let mut ready = Some(ready);
1356 while let Some(Ok(message)) = read.next().await {
1357 let Message::Text(raw) = message else {
1358 if matches!(message, Message::Close(_)) {
1359 break;
1360 }
1361 continue;
1362 };
1363 let Ok(packet) = serde_json::from_str::<Value>(raw.as_str()) else {
1364 continue;
1365 };
1366 if packet["event"] == "ready" {
1367 if let Some(ready) = ready.take() {
1368 let _ = ready.send(());
1369 }
1370 } else if packet["event"] == "rows" {
1371 for row in packet["rows"].as_array().into_iter().flatten() {
1372 let _ = held.send(FeedRow::from_json(row));
1373 }
1374 }
1375 }
1376 });
1377
1378 if is_ready.await.is_ok() {
1379 loop {
1380 let Ok(page) = self
1381 .updates(FeedQuery {
1382 since: *since,
1383 max: FEED_PAGE,
1384 })
1385 .await
1386 else {
1387 break;
1388 };
1389 let count = page.len() as u64;
1390 for row in page {
1391 if row.seq <= *since {
1392 continue;
1393 }
1394 *since = row.seq;
1395 if rows.send(row).is_err() {
1396 reader.abort();
1397 let _ = write.close().await;
1398 return;
1399 }
1400 }
1401 if count < FEED_PAGE {
1402 break;
1403 }
1404 }
1405 loop {
1406 tokio::select! {
1407 row = pending.recv() => {
1408 let Some(row) = row else {
1409 break;
1410 };
1411 if row.seq <= *since {
1412 continue;
1413 }
1414 *since = row.seq;
1415 if rows.send(row).is_err() {
1416 break;
1417 }
1418 }
1419 _ = rows.closed() => break,
1420 }
1421 }
1422 }
1423 reader.abort();
1424 let _ = write.close().await;
1425 }
1426
1427 pub async fn scraper(&self) -> Result<Option<Scraper>> {
1429 get_scraper(&self.authorization, &self.id).await
1430 }
1431
1432 pub async fn keep_updated(&self, settings: &ScraperSettings) -> Result<Scraper> {
1434 create_scraper(&self.authorization, &self.id, settings).await
1435 }
1436
1437 pub async fn update_scraper(&self, settings: &ScraperSettings) -> Result<Scraper> {
1438 update_scraper(&self.authorization, &self.id, settings).await
1439 }
1440
1441 pub async fn rebuild_scraper(&self, note: Option<&str>) -> Result<()> {
1443 rebuild_scraper(&self.authorization, &self.id, note).await
1444 }
1445
1446 pub async fn stop_keeping_updated(&self) -> Result<()> {
1448 remove_scraper(&self.authorization, &self.id).await
1449 }
1450}
1451
1452#[derive(Debug, Clone, Default)]
1454pub struct GroupOptions {
1455 pub group: Option<Group>,
1456}
1457
1458pub async fn list_datasets(
1459 authorization: &CredentialsOrApiKey,
1460 workspace_id: &str,
1461 opts: GroupOptions,
1462) -> Result<Vec<Dataset>> {
1463 let response = authorized(
1464 client().post(format!("{}/marmot/dataset/list", crate::get_url())),
1465 authorization,
1466 opts.group.as_ref(),
1467 )
1468 .json(&json!({ "workspaceId": workspace_id }))
1469 .send()
1470 .await?;
1471 let data: Value = ok(response).await?.json().await?;
1472 data.as_array()
1473 .into_iter()
1474 .flatten()
1475 .map(|d| {
1476 let id = text(&d["id"]);
1477 let snapshots = d["snapshots"]
1478 .as_array()
1479 .into_iter()
1480 .flatten()
1481 .map(|s| {
1482 Snapshot::new(
1483 authorization,
1484 workspace_id,
1485 &id,
1486 &text(&s["id"]),
1487 crate::tickets::parse_date(&s["createdAt"]).unwrap_or_default(),
1488 s["valid"].as_bool().unwrap_or(true),
1489 opts.group.as_ref(),
1490 )
1491 })
1492 .collect();
1493 Ok(Dataset {
1494 workspace_id: workspace_id.to_string(),
1495 name: text(&d["name"]),
1496 schema: Schema::deserialize(&d["schema"])?,
1497 termination: serde_json::from_value(d["termination"].clone()).unwrap_or_default(),
1498 content_types: serde_json::from_value(d["contentTypes"].clone())
1499 .unwrap_or_default(),
1500 proxy: serde_json::from_value(d["proxy"].clone()).unwrap_or(ProxyMode::Datacenter),
1501 document_modes: serde_json::from_value(d["documentModes"].clone())
1502 .unwrap_or_else(|_| vec![DocumentMode::Extraction]),
1503 snapshots: Arc::new(Mutex::new(snapshots)),
1504 authorization: authorization.clone(),
1505 group: opts.group.clone(),
1506 ws: Arc::new(Mutex::new(None)),
1507 id,
1508 })
1509 })
1510 .collect()
1511}
1512
1513#[derive(Clone)]
1515pub struct CreateDatasetOptions {
1516 pub name: String,
1517 pub schema: Schema,
1518 pub termination: Option<Vec<Termination>>,
1519 pub content_types: Option<Vec<String>>,
1520 pub proxy: Option<ProxyMode>,
1521 pub document_modes: Option<Vec<DocumentMode>>,
1522}
1523
1524pub async fn create_dataset(
1525 authorization: &CredentialsOrApiKey,
1526 workspace_id: &str,
1527 options: &CreateDatasetOptions,
1528 opts: GroupOptions,
1529) -> Result<String> {
1530 let response = authorized(
1531 client().post(format!("{}/marmot/dataset/create", crate::get_url())),
1532 authorization,
1533 opts.group.as_ref(),
1534 )
1535 .json(&json!({
1536 "workspaceId": workspace_id,
1537 "name": options.name,
1538 "schema": options.schema.serialize(),
1539 "termination": options.termination.clone().unwrap_or_default(),
1540 "contentTypes": options.content_types.clone().unwrap_or_default(),
1541 "proxy": options.proxy.unwrap_or(ProxyMode::Datacenter),
1542 "documentModes": options.document_modes.clone().unwrap_or_else(|| vec![DocumentMode::Extraction]),
1543 }))
1544 .send()
1545 .await?;
1546 Ok(ok(response).await?.text().await?)
1547}
1548
1549#[derive(Debug, Clone)]
1550pub struct Workspace {
1551 pub id: String,
1552 pub name: String,
1553 pub description: String,
1554 pub created_at: i64,
1555 pub datasets: u64,
1556 pub files: u64,
1557
1558 authorization: CredentialsOrApiKey,
1559 group: Option<Group>,
1560}
1561
1562impl Workspace {
1563 fn opts(&self) -> GroupOptions {
1564 GroupOptions {
1565 group: self.group.clone(),
1566 }
1567 }
1568
1569 pub async fn list_datasets(&self) -> Result<Vec<Dataset>> {
1570 list_datasets(&self.authorization, &self.id, self.opts()).await
1571 }
1572
1573 pub async fn create_dataset(&self, options: &CreateDatasetOptions) -> Result<String> {
1574 create_dataset(&self.authorization, &self.id, options, self.opts()).await
1575 }
1576
1577 pub async fn rename(&mut self, name: &str) -> Result<()> {
1578 update_workspace(
1579 &self.authorization,
1580 &self.id,
1581 &PartialWorkspace {
1582 name: Some(name.to_string()),
1583 description: None,
1584 },
1585 self.opts(),
1586 )
1587 .await?;
1588 self.name = name.to_string();
1589 Ok(())
1590 }
1591
1592 pub async fn set_description(&mut self, description: &str) -> Result<()> {
1593 update_workspace(
1594 &self.authorization,
1595 &self.id,
1596 &PartialWorkspace {
1597 name: None,
1598 description: Some(description.to_string()),
1599 },
1600 self.opts(),
1601 )
1602 .await?;
1603 self.description = description.to_string();
1604 Ok(())
1605 }
1606
1607 async fn post(&self, route: &str, body: Value) -> Result<reqwest::Response> {
1608 let response = authorized(
1609 client().post(format!("{}{route}", crate::get_url())),
1610 &self.authorization,
1611 self.group.as_ref(),
1612 )
1613 .json(&body)
1614 .send()
1615 .await?;
1616 ok(response).await
1617 }
1618
1619 pub async fn remove(&self) -> Result<()> {
1620 self.post("/marmot/workspace/remove", json!({ "id": self.id }))
1621 .await?;
1622 Ok(())
1623 }
1624
1625 pub async fn get_files(&self) -> Result<Vec<WorkspaceFile>> {
1626 Ok(get_workspace(&self.authorization, &self.id, self.opts())
1627 .await?
1628 .files)
1629 }
1630
1631 pub async fn upload_file(
1632 &self,
1633 name: &str,
1634 content_type: &str,
1635 data: &[u8],
1636 ) -> Result<WorkspaceFile> {
1637 use base64::Engine;
1638 let data = base64::engine::general_purpose::STANDARD.encode(data);
1639 Ok(self
1640 .post(
1641 "/marmot/workspace/file/upload",
1642 json!({ "workspaceId": self.id, "name": name, "contentType": content_type, "data": data }),
1643 )
1644 .await?
1645 .json()
1646 .await?)
1647 }
1648
1649 pub async fn download_file(&self, file: &WorkspaceFile) -> Result<Vec<u8>> {
1650 Ok(self
1651 .post(
1652 "/marmot/workspace/file/get",
1653 json!({ "key": file.key, "contentType": file.content_type, "workspaceId": self.id }),
1654 )
1655 .await?
1656 .bytes()
1657 .await?
1658 .to_vec())
1659 }
1660
1661 pub async fn remove_file(&self, key: &str) -> Result<()> {
1662 self.post(
1663 "/marmot/workspace/file/remove",
1664 json!({ "workspaceId": self.id, "key": key }),
1665 )
1666 .await?;
1667 Ok(())
1668 }
1669}
1670
1671pub async fn list_workspaces(
1672 authorization: &CredentialsOrApiKey,
1673 opts: GroupOptions,
1674) -> Result<Vec<Workspace>> {
1675 let response = authorized(
1676 client().post(format!("{}/marmot/workspace/list", crate::get_url())),
1677 authorization,
1678 opts.group.as_ref(),
1679 )
1680 .header("Content-Type", "application/json")
1681 .send()
1682 .await?;
1683 let data: Value = ok(response).await?.json().await?;
1684 Ok(data
1685 .as_array()
1686 .into_iter()
1687 .flatten()
1688 .map(|w| Workspace {
1689 id: text(&w["id"]),
1690 name: text(&w["name"]),
1691 description: text(&w["description"]),
1692 created_at: w["createdAt"].as_f64().unwrap_or_default() as i64,
1693 datasets: w["datasets"].as_f64().unwrap_or_default() as u64,
1694 files: w["files"].as_f64().unwrap_or_default() as u64,
1695 authorization: authorization.clone(),
1696 group: opts.group.clone(),
1697 })
1698 .collect())
1699}
1700
1701#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
1703#[serde(rename_all = "camelCase")]
1704pub struct WorkspaceDetail {
1705 pub id: String,
1706 pub name: String,
1707 #[serde(default)]
1708 pub description: String,
1709 #[serde(default)]
1710 pub created_at: i64,
1711 #[serde(default)]
1712 pub files: Vec<WorkspaceFile>,
1713}
1714
1715pub async fn get_workspace(
1716 authorization: &CredentialsOrApiKey,
1717 id: &str,
1718 opts: GroupOptions,
1719) -> Result<WorkspaceDetail> {
1720 let response = authorized(
1721 client().post(format!("{}/marmot/workspace/get", crate::get_url())),
1722 authorization,
1723 opts.group.as_ref(),
1724 )
1725 .json(&json!({ "id": id }))
1726 .send()
1727 .await?;
1728 Ok(ok(response).await?.json().await?)
1729}
1730
1731pub async fn create_workspace(
1733 authorization: &CredentialsOrApiKey,
1734 name: &str,
1735 description: Option<&str>,
1736 opts: GroupOptions,
1737) -> Result<String> {
1738 let response = authorized(
1739 client().post(format!("{}/marmot/workspace/create", crate::get_url())),
1740 authorization,
1741 opts.group.as_ref(),
1742 )
1743 .json(&json!({ "name": name, "description": description.unwrap_or_default() }))
1744 .send()
1745 .await?;
1746 Ok(ok(response).await?.text().await?)
1747}
1748
1749pub async fn update_workspace(
1750 authorization: &CredentialsOrApiKey,
1751 id: &str,
1752 update: &PartialWorkspace,
1753 opts: GroupOptions,
1754) -> Result<()> {
1755 let mut body = Map::new();
1756 body.insert("id".to_string(), Value::from(id));
1757 if let Value::Object(fields) = serde_json::to_value(update)? {
1758 body.extend(fields);
1759 }
1760 let response = authorized(
1761 client().post(format!("{}/marmot/workspace/update", crate::get_url())),
1762 authorization,
1763 opts.group.as_ref(),
1764 )
1765 .json(&body)
1766 .send()
1767 .await?;
1768 ok(response).await?;
1769 Ok(())
1770}