Skip to main content

fauna_api/marmot/
workspace.rs

1use std::collections::HashMap;
2use std::ops::Deref;
3use std::sync::{Arc, Mutex, MutexGuard, PoisonError};
4
5use serde::{Deserialize, Serialize};
6use serde_json::{Map, Value, json};
7
8use super::scraper::Scraper;
9use crate::authentification::CredentialsOrApiKey;
10use crate::error::{Error, Result};
11use crate::groups::Group;
12use crate::http::{client, ok};
13use crate::parrot::room::{ChatRoom, RoomHooks, connect_room};
14use crate::parrot::share::ChatTarget;
15use crate::shared::server::{Authorization, Hooks, Presence, Room, connect as connect_server};
16
17#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
18#[serde(rename_all = "camelCase")]
19pub struct WorkspaceFile {
20    pub name: String,
21    pub content_type: String,
22    pub size: u64,
23    pub key: String,
24}
25
26#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
27#[serde(rename_all = "camelCase")]
28pub struct WorkspaceDatasetMeta {
29    pub id: String,
30    pub name: String,
31    #[serde(default)]
32    pub schema: Value,
33    #[serde(default)]
34    pub termination: Vec<Value>,
35    #[serde(default)]
36    pub content_types: Vec<String>,
37    #[serde(default)]
38    pub proxy: String,
39    #[serde(default)]
40    pub document_modes: Vec<String>,
41    #[serde(default)]
42    pub created_at: i64,
43}
44
45#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
46#[serde(rename_all = "camelCase")]
47pub struct WorkspaceSummary {
48    pub id: String,
49    pub name: String,
50    pub description: String,
51    pub created_at: i64,
52    pub datasets: u64,
53    pub files: u64,
54}
55
56/// The `opts` argument of the workspace calls: `group` acts on that group's workspaces.
57#[derive(Debug, Clone, Default)]
58pub struct WorkspaceOptions {
59    pub group: Option<Group>,
60}
61
62async fn post(
63    authorization: &CredentialsOrApiKey,
64    route: &str,
65    body: Option<Value>,
66    group: Option<&Group>,
67) -> Result<reqwest::Response> {
68    let mut request = client()
69        .post(format!("{}{route}", super::get_url()))
70        .header("Content-Type", "application/json")
71        .header(
72            "Authorization",
73            match authorization {
74                CredentialsOrApiKey::Credentials(credentials) => {
75                    format!("Token {}", credentials.token)
76                }
77                CredentialsOrApiKey::ApiKey(key) => format!("ApiKey {}", key.id),
78            },
79        );
80    if let Some(group) = group {
81        request = request.header("Group", &group.id);
82    }
83    if let Some(body) = body {
84        request = request.json(&body);
85    }
86    ok(request.send().await?).await
87}
88
89pub async fn list_workspaces(
90    authorization: &CredentialsOrApiKey,
91    opts: WorkspaceOptions,
92) -> Result<Vec<WorkspaceSummary>> {
93    let response = post(
94        authorization,
95        "/marmot/workspace/list",
96        None,
97        opts.group.as_ref(),
98    )
99    .await?;
100    Ok(response.json().await?)
101}
102
103/// `description` defaults to empty.
104pub async fn create_workspace(
105    authorization: &CredentialsOrApiKey,
106    name: &str,
107    description: Option<&str>,
108    opts: WorkspaceOptions,
109) -> Result<String> {
110    let response = post(
111        authorization,
112        "/marmot/workspace/create",
113        Some(json!({ "name": name, "description": description.unwrap_or_default() })),
114        opts.group.as_ref(),
115    )
116    .await?;
117    Ok(response.text().await?)
118}
119
120/// `Partial<{ name, description, repositoryId }>` in the TypeScript SDK.
121#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
122#[serde(rename_all = "camelCase")]
123pub struct PartialWorkspace {
124    #[serde(default, skip_serializing_if = "Option::is_none")]
125    pub name: Option<String>,
126    #[serde(default, skip_serializing_if = "Option::is_none")]
127    pub description: Option<String>,
128    #[serde(default, skip_serializing_if = "Option::is_none")]
129    pub repository_id: Option<u64>,
130}
131
132/// Renames a workspace or changes its description, or sets its computer once: `repository_id`,
133/// a repository of the workspace's owner, whose machine runs its scrapers and renders and
134/// investigates pages for research.
135pub async fn update_workspace(
136    authorization: &CredentialsOrApiKey,
137    id: &str,
138    update: &PartialWorkspace,
139    opts: WorkspaceOptions,
140) -> Result<()> {
141    let mut body = Map::new();
142    body.insert("id".to_string(), Value::from(id));
143    if let Value::Object(fields) = serde_json::to_value(update)? {
144        body.extend(fields);
145    }
146    post(
147        authorization,
148        "/marmot/workspace/update",
149        Some(Value::Object(body)),
150        opts.group.as_ref(),
151    )
152    .await?;
153    Ok(())
154}
155
156pub async fn remove_workspace(
157    authorization: &CredentialsOrApiKey,
158    id: &str,
159    opts: WorkspaceOptions,
160) -> Result<()> {
161    post(
162        authorization,
163        "/marmot/workspace/remove",
164        Some(json!({ "id": id })),
165        opts.group.as_ref(),
166    )
167    .await?;
168    Ok(())
169}
170
171struct Workspace {
172    workspace_id: String,
173    name: String,
174    description: String,
175    files: HashMap<String, WorkspaceFile>,
176    datasets: HashMap<String, WorkspaceDatasetMeta>,
177    scrapers: HashMap<String, Scraper>,
178    repository_id: Option<u64>,
179    agent_text: String,
180}
181
182/// `{ room, self, chatId }` in the TypeScript SDK, with `self` as `me`.
183#[derive(Clone)]
184pub struct WorkspaceChat {
185    pub room: ChatRoom,
186    pub me: Presence,
187    pub chat_id: String,
188}
189
190/// The `opts` argument of `WorkspaceRoom::connect`. `server` names the room when
191/// it isn't the workspace's id.
192#[derive(Default)]
193pub struct ConnectOptions {
194    pub group: Option<Group>,
195    pub hooks: Option<Arc<dyn Hooks>>,
196    pub server: Option<String>,
197}
198
199#[derive(Clone)]
200pub struct WorkspaceRoom {
201    room: Room,
202    state: Arc<Mutex<Workspace>>,
203}
204
205impl Deref for WorkspaceRoom {
206    type Target = Room;
207    fn deref(&self) -> &Room {
208        &self.room
209    }
210}
211
212fn field<'a>(packet: &'a Value, key: &str) -> &'a Value {
213    if packet[key].is_null() {
214        &packet["data"][key]
215    } else {
216        &packet[key]
217    }
218}
219
220fn number_or_none(value: &Value) -> Option<u64> {
221    match value {
222        Value::Null | Value::Bool(false) => None,
223        Value::Number(number) if number.as_f64() == Some(0.0) => None,
224        Value::String(text) if text.is_empty() => None,
225        value => Some(crate::js::js_number(Some(value)) as u64),
226    }
227}
228
229impl WorkspaceRoom {
230    fn workspace(&self) -> MutexGuard<'_, Workspace> {
231        self.state.lock().unwrap_or_else(PoisonError::into_inner)
232    }
233
234    pub fn workspace_id(&self) -> String {
235        self.workspace().workspace_id.clone()
236    }
237
238    pub fn name(&self) -> String {
239        self.workspace().name.clone()
240    }
241
242    pub fn description(&self) -> String {
243        self.workspace().description.clone()
244    }
245
246    pub fn files(&self) -> HashMap<String, WorkspaceFile> {
247        self.workspace().files.clone()
248    }
249
250    pub fn datasets(&self) -> HashMap<String, WorkspaceDatasetMeta> {
251        self.workspace().datasets.clone()
252    }
253
254    /// Kept datasets, by dataset id.
255    pub fn scrapers(&self) -> HashMap<String, Scraper> {
256        self.workspace().scrapers.clone()
257    }
258
259    /// The Termite repository whose machine runs the workspace's scrapers, once there is one.
260    pub fn repository_id(&self) -> Option<u64> {
261        self.workspace().repository_id
262    }
263
264    /// The model the workspace's scraper agent thinks with.
265    pub fn agent_text(&self) -> String {
266        self.workspace().agent_text.clone()
267    }
268
269    pub fn hydrate(&self, state: &Value) {
270        let mut workspace = self.workspace();
271        if let Some(id) = state["id"].as_str() {
272            workspace.workspace_id = id.to_string();
273        }
274        if let Some(name) = state["name"].as_str() {
275            workspace.name = name.to_string();
276        }
277        if let Some(description) = state["description"].as_str() {
278            workspace.description = description.to_string();
279        }
280        workspace.files = state["files"]
281            .as_array()
282            .into_iter()
283            .flatten()
284            .filter(|file| file["key"].as_str().is_some_and(|key| !key.is_empty()))
285            .filter_map(|file| serde_json::from_value::<WorkspaceFile>(file.clone()).ok())
286            .map(|file| (file.key.clone(), file))
287            .collect();
288        workspace.datasets = state["datasets"]
289            .as_array()
290            .into_iter()
291            .flatten()
292            .filter_map(|dataset| {
293                serde_json::from_value::<WorkspaceDatasetMeta>(dataset.clone()).ok()
294            })
295            .filter(|dataset| !dataset.id.is_empty())
296            .map(|dataset| (dataset.id.clone(), dataset))
297            .collect();
298        workspace.scrapers = state["scrapers"]
299            .as_array()
300            .into_iter()
301            .flatten()
302            .filter(|scraper| !scraper["datasetId"].is_null() && scraper["datasetId"] != "")
303            .map(Scraper::from_json)
304            .map(|scraper| (scraper.dataset_id.clone(), scraper))
305            .collect();
306        workspace.repository_id = number_or_none(&state["repositoryId"]);
307        workspace.agent_text = state["agentText"].as_str().unwrap_or_default().to_string();
308    }
309
310    pub fn handle_event(&self, packet: &Value) {
311        let mut workspace = self.workspace();
312        match packet["event"].as_str().unwrap_or_default() {
313            "upsert-file" => {
314                if let Ok(file) =
315                    serde_json::from_value::<WorkspaceFile>(field(packet, "file").clone())
316                    && !file.key.is_empty()
317                {
318                    workspace.files.insert(file.key.clone(), file);
319                }
320            }
321            "remove-file" => {
322                if let Some(key) = field(packet, "key").as_str().filter(|key| !key.is_empty()) {
323                    workspace.files.remove(key);
324                }
325            }
326            "upsert-dataset" => {
327                if let Ok(dataset) =
328                    serde_json::from_value::<WorkspaceDatasetMeta>(field(packet, "dataset").clone())
329                    && !dataset.id.is_empty()
330                {
331                    workspace.datasets.insert(dataset.id.clone(), dataset);
332                }
333            }
334            "remove-dataset" => {
335                if let Some(id) = field(packet, "id").as_str().filter(|id| !id.is_empty()) {
336                    workspace.datasets.remove(id);
337                }
338            }
339            "upsert-scraper" => {
340                let scraper = field(packet, "scraper");
341                if !scraper["datasetId"].is_null() && scraper["datasetId"] != "" {
342                    let scraper = Scraper::from_json(scraper);
343                    workspace
344                        .scrapers
345                        .insert(scraper.dataset_id.clone(), scraper);
346                }
347            }
348            "remove-scraper" => {
349                let dataset_id = field(packet, "datasetId");
350                if !dataset_id.is_null() && dataset_id != "" {
351                    let dataset_id = crate::js::js_string(Some(dataset_id), "");
352                    workspace.scrapers.remove(&dataset_id);
353                }
354            }
355            "update-workspace" => {
356                let update = field(packet, "workspace");
357                if let Some(name) = update["name"].as_str().filter(|name| !name.is_empty()) {
358                    workspace.name = name.to_string();
359                }
360                if let Some(description) = update.get("description") {
361                    workspace.description = description.as_str().unwrap_or_default().to_string();
362                }
363                if let Some(repository_id) = update.get("repositoryId") {
364                    workspace.repository_id = number_or_none(repository_id);
365                }
366                if let Some(agent_text) = update.get("agentText") {
367                    workspace.agent_text = agent_text.as_str().unwrap_or_default().to_string();
368                }
369            }
370            _ => {}
371        }
372    }
373
374    pub async fn get_chat(&self, hooks: Option<Arc<dyn RoomHooks>>) -> Result<WorkspaceChat> {
375        let authorization = self.get_authorization();
376        let request = client()
377            .post(format!("{}/marmot/workspace/chat/ensure", super::get_url()))
378            .header("Content-Type", "application/json")
379            .header(
380                "Authorization",
381                match authorization {
382                    CredentialsOrApiKey::Credentials(credentials) => {
383                        format!("Token {}", credentials.token)
384                    }
385                    CredentialsOrApiKey::ApiKey(key) => format!("ApiKey {}", key.id),
386                },
387            );
388        let request = match self.authorization() {
389            Authorization::AccessCode { access_code, .. } => {
390                request.header("Access-Code", access_code)
391            }
392            Authorization::Server { group, .. } => {
393                let request = request.header("Server", self.workspace_id());
394                match group {
395                    Some(group) => request.header("Group", group),
396                    None => request,
397                }
398            }
399        };
400        let json: Value = ok(request.send().await?).await?.json().await?;
401        let chat_id = json["chatId"].as_str().unwrap_or_default().to_string();
402        let code = json["code"].as_str().unwrap_or_default().to_string();
403        if chat_id.is_empty() || code.is_empty() {
404            return Err(Error::Message("Invalid ensure response.".to_string()));
405        }
406
407        let target = match self.authorization() {
408            Authorization::AccessCode { .. } => ChatTarget::Code { code },
409            Authorization::Server { .. } => ChatTarget::Reference {
410                reference: json["reference"]
411                    .as_str()
412                    .filter(|reference| !reference.is_empty())
413                    .map(str::to_string)
414                    .unwrap_or_else(|| format!("parrot:chat:{chat_id}")),
415            },
416        };
417        let (room, me) = connect_room(authorization, &target, hooks).await?;
418        Ok(WorkspaceChat { room, me, chat_id })
419    }
420
421    pub async fn connect(
422        authorization: &CredentialsOrApiKey,
423        target_id: &str,
424        opts: ConnectOptions,
425    ) -> Result<(WorkspaceRoom, Presence)> {
426        let server = opts.server.unwrap_or_else(|| target_id.to_string());
427        WorkspaceRoom::open(
428            Authorization::Server {
429                authorization: authorization.clone(),
430                server,
431                group: opts.group.as_ref().map(|group| group.id.clone()),
432            },
433            opts.hooks,
434        )
435        .await
436    }
437
438    pub async fn join(
439        authorization: &CredentialsOrApiKey,
440        access_code: &str,
441        hooks: Option<Arc<dyn Hooks>>,
442    ) -> Result<(WorkspaceRoom, Presence)> {
443        WorkspaceRoom::open(
444            Authorization::AccessCode {
445                authorization: authorization.clone(),
446                access_code: access_code.to_string(),
447            },
448            hooks,
449        )
450        .await
451    }
452
453    async fn open(
454        auth: Authorization,
455        hooks: Option<Arc<dyn Hooks>>,
456    ) -> Result<(WorkspaceRoom, Presence)> {
457        let state = Arc::new(Mutex::new(Workspace {
458            workspace_id: String::new(),
459            name: String::new(),
460            description: String::new(),
461            files: HashMap::new(),
462            datasets: HashMap::new(),
463            scrapers: HashMap::new(),
464            repository_id: None,
465            agent_text: String::new(),
466        }));
467        let me = connect_server(
468            auth,
469            &format!("{}/marmot/workspace", super::get_url()),
470            Arc::new(WorkspaceHooks {
471                state: state.clone(),
472                user: hooks.unwrap_or_else(|| Arc::new(())),
473            }),
474        )
475        .await?;
476        let room = WorkspaceRoom {
477            room: me.room().clone(),
478            state,
479        };
480        Ok((room, me))
481    }
482}
483
484struct WorkspaceHooks {
485    state: Arc<Mutex<Workspace>>,
486    user: Arc<dyn Hooks>,
487}
488
489impl WorkspaceHooks {
490    fn room(&self, room: &Room) -> WorkspaceRoom {
491        WorkspaceRoom {
492            room: room.clone(),
493            state: self.state.clone(),
494        }
495    }
496}
497
498impl Hooks for WorkspaceHooks {
499    fn create_room(&self, room: &Room, state: &Value) {
500        self.state
501            .lock()
502            .unwrap_or_else(PoisonError::into_inner)
503            .workspace_id = room.id().to_string();
504        self.user.create_room(room, state);
505    }
506
507    fn create_presence(&self, room: &Room, presence: &Presence) {
508        self.user.create_presence(room, presence);
509    }
510
511    fn remove_presence(&self, room: &Room, presence: &Presence) {
512        self.user.remove_presence(room, presence);
513    }
514
515    fn on_connection(&self, room: &Room, state: &Value) {
516        self.room(room).hydrate(state);
517        self.user.on_connection(room, state);
518    }
519
520    fn on_event(&self, room: &Room, packet: &Value) -> bool {
521        self.room(room).handle_event(packet);
522        self.user.on_event(room, packet);
523        true
524    }
525
526    fn on_presence(&self, room: &Room) {
527        self.user.on_presence(room);
528    }
529
530    fn on_closed(&self, room: &Room, reason: Option<&str>) {
531        self.user.on_closed(room, reason);
532    }
533}
534
535#[cfg(test)]
536mod tests {
537    use serde_json::json;
538    use tokio::sync::mpsc;
539
540    use super::*;
541    use crate::authentification::ApiKey;
542
543    #[test]
544    fn workspace_rooms_follow_their_snapshot_and_events() {
545        let (ws, _outbox) = mpsc::unbounded_channel();
546        let room = WorkspaceRoom {
547            room: Room::new(
548                "http://localhost/marmot/workspace".to_string(),
549                ws,
550                Authorization::Server {
551                    authorization: CredentialsOrApiKey::ApiKey(ApiKey::default()),
552                    server: "w1".to_string(),
553                    group: None,
554                },
555                "w1".to_string(),
556            ),
557            state: Arc::new(Mutex::new(Workspace {
558                workspace_id: "w1".to_string(),
559                name: String::new(),
560                description: String::new(),
561                files: HashMap::new(),
562                datasets: HashMap::new(),
563                scrapers: HashMap::new(),
564                repository_id: None,
565                agent_text: String::new(),
566            })),
567        };
568        room.hydrate(&json!({
569            "name": "Prices",
570            "files": [{ "name": "a.csv", "contentType": "text/csv", "size": 3, "key": "k1" }, { "name": "x" }],
571            "datasets": [{ "id": "d1", "name": "Shoes", "schema": {}, "termination": [], "contentTypes": [], "proxy": "datacenter", "documentModes": [], "createdAt": 1 }],
572            "scrapers": [{ "datasetId": "d1", "workspaceId": "w1", "status": "active" }],
573            "repositoryId": "7",
574            "agentText": "gpt-5.4"
575        }));
576        assert_eq!(room.name(), "Prices");
577        assert_eq!(room.files().len(), 1);
578        assert_eq!(room.datasets()["d1"].name, "Shoes");
579        assert_eq!(room.repository_id(), Some(7));
580
581        room.handle_event(&json!({ "event": "remove-file", "data": { "key": "k1" } }));
582        room.handle_event(&json!({ "event": "remove-scraper", "datasetId": "d1" }));
583        room.handle_event(&json!({ "event": "update-workspace", "data": { "workspace": { "name": "Deals", "repositoryId": null } } }));
584        assert!(room.files().is_empty());
585        assert!(room.scrapers().is_empty());
586        assert_eq!(room.name(), "Deals");
587        assert_eq!(room.repository_id(), None);
588    }
589}