use std::ops::Deref;
use std::sync::{Arc, Mutex, MutexGuard, PoisonError};
use std::time::Duration;
use chrono::{DateTime, Utc};
use futures_util::stream::{self, BoxStream};
use futures_util::{SinkExt, StreamExt};
use serde::{Deserialize, Serialize};
use serde_json::{Map, Value, json};
use tokio::sync::{mpsc, oneshot};
use tokio_tungstenite::tungstenite::Message;
use super::scraper::{
Scraper, ScraperSettings, create_scraper, get_scraper, rebuild_scraper, remove_scraper,
update_scraper,
};
use super::workspace::PartialWorkspace;
use crate::authentification::CredentialsOrApiKey;
use crate::error::{Error, Result};
use crate::groups::Group;
use crate::http::{client, encode, ok, websocket};
use crate::parrot::format::Format;
use crate::parrot::generative::{Embedding, Text};
use crate::shared::server::Presence;
pub use super::browser::BrowserTool;
pub use super::media::{DocumentMode, ProxyMode};
pub use super::workspace::WorkspaceFile;
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
pub struct ArrayBounds {
pub minimum: u64,
pub maximum: u64,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(untagged)]
pub enum StringOrStrings {
String(String),
Strings(Vec<String>),
}
#[allow(clippy::large_enum_variant)]
pub enum FormatOrSchemaLeafOrChildren {
Format(Format),
SchemaLeaf(Box<SchemaLeaf>),
Children(Vec<(String, SchemaLeaf)>),
}
pub struct SchemaLeaf {
pub value: FormatOrSchemaLeafOrChildren,
pub prompt: Option<String>,
pub sources: Option<u64>,
pub optional: Option<bool>,
pub array: Option<ArrayBounds>,
pub max_iterations: Option<u64>,
pub filters: Option<StringOrStrings>,
}
impl Clone for SchemaLeaf {
fn clone(&self) -> Self {
deserialize_leaf(&serialize_leaf(self)).expect("a leaf reads its own JSON")
}
}
fn insert(fields: &mut Map<String, Value>, key: &str, value: Option<Value>) {
if let Some(value) = value {
fields.insert(key.to_string(), value);
}
}
fn serialize_leaf(leaf: &SchemaLeaf) -> Value {
let mut fields = Map::new();
insert(&mut fields, "prompt", leaf.prompt.clone().map(Value::from));
insert(&mut fields, "sources", leaf.sources.map(Value::from));
insert(&mut fields, "optional", leaf.optional.map(Value::from));
insert(&mut fields, "array", leaf.array.map(|array| json!(array)));
insert(
&mut fields,
"maxIterations",
leaf.max_iterations.map(Value::from),
);
insert(
&mut fields,
"filters",
leaf.filters.as_ref().map(|filters| json!(filters)),
);
let value = match &leaf.value {
FormatOrSchemaLeafOrChildren::Format(format) => json!({ "question": format.to_json() }),
FormatOrSchemaLeafOrChildren::SchemaLeaf(leaf) => json!({ "schema": serialize_leaf(leaf) }),
FormatOrSchemaLeafOrChildren::Children(children) => {
let children: Map<String, Value> = children
.iter()
.map(|(key, leaf)| (key.clone(), serialize_leaf(leaf)))
.collect();
json!({ "children": children })
}
};
fields.insert("value".to_string(), value);
Value::Object(fields)
}
fn deserialize_leaf(data: &Value) -> Result<SchemaLeaf> {
let array = match &data["array"] {
Value::Null => None,
array => {
let bound = |key: &str| {
array[key]
.as_f64()
.filter(|bound| bound.fract() == 0.0 && *bound >= 0.0)
};
let Some(minimum) = bound("minimum") else {
return Err(Error::Message(format!(
"SchemaLeaf array.minimum must be a non-negative integer, got {}",
array["minimum"]
)));
};
let Some(maximum) = bound("maximum") else {
return Err(Error::Message(format!(
"SchemaLeaf array.maximum must be a non-negative integer, got {}",
array["maximum"]
)));
};
if minimum > maximum {
return Err(Error::Message(format!(
"SchemaLeaf array.minimum ({minimum}) cannot exceed array.maximum ({maximum})"
)));
}
Some(ArrayBounds {
minimum: minimum as u64,
maximum: maximum as u64,
})
}
};
let value = &data["value"];
let value = if value.get("question").is_some() {
FormatOrSchemaLeafOrChildren::Format(Format::from_json(&value["question"])?)
} else if let Some(children) = value.get("children") {
FormatOrSchemaLeafOrChildren::Children(
children
.as_object()
.into_iter()
.flatten()
.map(|(key, leaf)| Ok((key.clone(), deserialize_leaf(leaf)?)))
.collect::<Result<Vec<_>>>()?,
)
} else {
FormatOrSchemaLeafOrChildren::SchemaLeaf(Box::new(deserialize_leaf(&value["schema"])?))
};
Ok(SchemaLeaf {
value,
prompt: data["prompt"].as_str().map(str::to_string),
sources: data["sources"].as_u64(),
optional: data["optional"].as_bool(),
array,
max_iterations: data["maxIterations"].as_u64(),
filters: serde_json::from_value(data["filters"].clone()).ok(),
})
}
#[derive(Clone)]
#[allow(clippy::large_enum_variant)]
pub enum Schema {
Json(SchemaJSON),
Csv(SchemaCSV),
}
impl Schema {
pub fn r#type(&self) -> &'static str {
match self {
Schema::Json(_) => "json",
Schema::Csv(_) => "csv",
}
}
pub fn serialize(&self) -> Value {
match self {
Schema::Json(schema) => schema.serialize(),
Schema::Csv(schema) => schema.serialize(),
}
}
pub fn deserialize(data: &Value) -> Result<Schema> {
match data["type"].as_str() {
Some("json") => Ok(Schema::Json(SchemaJSON::new(deserialize_leaf(
&data["schema"],
)?))),
Some("csv") => {
let columns = data["schema"]
.as_object()
.into_iter()
.flatten()
.map(|(key, leaf)| Ok((key.clone(), deserialize_leaf(leaf)?)))
.collect::<Result<Vec<_>>>()?;
let bound = |key: &str| data[key].as_f64().unwrap_or(f64::NAN);
Ok(Schema::Csv(SchemaCSV::new(
columns,
bound("minimum"),
bound("maximum"),
data["prompt"].as_str(),
)?))
}
kind => Err(Error::Message(format!(
"Invalid schema type: {}",
kind.unwrap_or("undefined")
))),
}
}
}
#[derive(Clone)]
pub struct SchemaJSON {
pub schema: SchemaLeaf,
}
impl SchemaJSON {
pub fn new(schema: SchemaLeaf) -> SchemaJSON {
SchemaJSON { schema }
}
pub fn r#type(&self) -> &'static str {
"json"
}
pub fn serialize(&self) -> Value {
json!({ "type": "json", "schema": serialize_leaf(&self.schema) })
}
}
#[derive(Clone)]
pub struct SchemaCSV {
pub schema: Vec<(String, SchemaLeaf)>,
pub minimum: u64,
pub maximum: u64,
pub prompt: String,
}
impl SchemaCSV {
pub fn new(
schema: Vec<(String, SchemaLeaf)>,
minimum: f64,
maximum: f64,
prompt: Option<&str>,
) -> Result<SchemaCSV> {
if schema
.iter()
.any(|(_, leaf)| !matches!(leaf.value, FormatOrSchemaLeafOrChildren::Format(_)))
{
return Err(Error::Message(
"Nested schema are not supported for CSV".to_string(),
));
}
let whole = |count: f64| count.fract() == 0.0 && count >= 0.0;
if !whole(minimum) {
return Err(Error::Message(format!(
"SchemaCSV minimum must be a non-negative integer, got {minimum}"
)));
}
if !whole(maximum) {
return Err(Error::Message(format!(
"SchemaCSV maximum must be a non-negative integer, got {maximum}"
)));
}
if minimum > maximum {
return Err(Error::Message(format!(
"SchemaCSV minimum ({minimum}) cannot exceed maximum ({maximum})"
)));
}
Ok(SchemaCSV {
schema,
minimum: minimum as u64,
maximum: maximum as u64,
prompt: prompt.unwrap_or_default().to_string(),
})
}
pub fn r#type(&self) -> &'static str {
"csv"
}
pub fn serialize(&self) -> Value {
let schema: Map<String, Value> = self
.schema
.iter()
.map(|(key, leaf)| (key.clone(), serialize_leaf(leaf)))
.collect();
json!({
"type": "csv",
"minimum": self.minimum,
"maximum": self.maximum,
"prompt": self.prompt,
"schema": schema,
})
}
}
fn format_signature(value: &FormatOrSchemaLeafOrChildren) -> String {
match value {
FormatOrSchemaLeafOrChildren::Format(format) => format!(
"format:{}",
format.to_json()["type"].as_str().unwrap_or("unknown")
),
_ => "composite".to_string(),
}
}
fn set(out: &mut Vec<(String, String)>, path: String, signature: String) {
match out.iter_mut().find(|(known, _)| *known == path) {
Some(entry) => entry.1 = signature,
None => out.push((path, signature)),
}
}
fn flatten_leaf_paths(leaf: &SchemaLeaf, path: &str, out: &mut Vec<(String, String)>) {
if leaf.array.is_some() {
if let FormatOrSchemaLeafOrChildren::SchemaLeaf(inner) = &leaf.value {
flatten_leaf_paths(inner, &format!("{path}[]"), out);
}
return;
}
match &leaf.value {
FormatOrSchemaLeafOrChildren::Format(_) => {
let path = if path.is_empty() { "value" } else { path };
set(out, path.to_string(), format_signature(&leaf.value));
}
FormatOrSchemaLeafOrChildren::SchemaLeaf(inner) => flatten_leaf_paths(inner, path, out),
FormatOrSchemaLeafOrChildren::Children(children) => {
for (key, child) in children {
let child_path = if path.is_empty() {
key.clone()
} else {
format!("{path}.{key}")
};
flatten_leaf_paths(child, &child_path, out);
}
}
}
}
pub fn schema_paths(schema: &Schema) -> Vec<(String, String)> {
let mut out = Vec::new();
match schema {
Schema::Csv(schema) => {
for (key, leaf) in &schema.schema {
set(&mut out, key.clone(), format_signature(&leaf.value));
}
}
Schema::Json(schema) => flatten_leaf_paths(&schema.schema, "", &mut out),
}
out
}
#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
pub struct SchemaDiff {
pub added: Vec<String>,
pub dropped: Vec<String>,
}
pub fn diff_schema(baseline: &Schema, target: &Schema) -> SchemaDiff {
let a = schema_paths(baseline);
let b = schema_paths(target);
let lookup = |paths: &[(String, String)], path: &str| {
paths
.iter()
.find(|(known, _)| known == path)
.map(|(_, signature)| signature.clone())
};
SchemaDiff {
added: b
.iter()
.filter(|(path, signature)| lookup(&a, path).as_ref() != Some(signature))
.map(|(path, _)| path.clone())
.collect(),
dropped: a
.iter()
.filter(|(path, signature)| lookup(&b, path).as_ref() != Some(signature))
.map(|(path, _)| path.clone())
.collect(),
}
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct Termination {
pub r#type: String,
#[serde(flatten)]
pub fields: Map<String, Value>,
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct TokenTermination {
#[serde(flatten)]
pub termination: Termination,
pub max: f64,
}
impl Deref for TokenTermination {
type Target = Termination;
fn deref(&self) -> &Termination {
&self.termination
}
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct TimeTermination {
#[serde(flatten)]
pub termination: Termination,
pub max: f64,
}
impl Deref for TimeTermination {
type Target = Termination;
fn deref(&self) -> &Termination {
&self.termination
}
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct CostTermination {
#[serde(flatten)]
pub termination: Termination,
pub max: f64,
}
impl Deref for CostTermination {
type Target = Termination;
fn deref(&self) -> &Termination {
&self.termination
}
}
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Hash, Serialize, Deserialize)]
#[serde(rename_all = "lowercase")]
pub enum DataFormat {
Csv,
#[default]
Json,
}
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub struct DataOptions {
pub format: Option<DataFormat>,
pub snapshot: Option<String>,
pub sources: Option<bool>,
}
#[derive(Debug, Clone, PartialEq, Serialize)]
pub struct FeedRow {
pub seq: u64,
pub value: Value,
pub sources: Value,
pub created: DateTime<Utc>,
}
impl FeedRow {
pub fn from_json(json: &Value) -> FeedRow {
FeedRow {
seq: json["seq"].as_f64().unwrap_or_default() as u64,
value: json["value"].clone(),
sources: json["sources"].clone(),
created: crate::tickets::parse_date(&json["created"]).unwrap_or_default(),
}
}
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct InvalidRow {
pub index: u64,
pub reason: String,
}
#[derive(Debug, Clone, Default, PartialEq, Serialize)]
pub struct PushResult {
pub added: u64,
pub first: Option<f64>,
pub last: Option<f64>,
pub invalid: Vec<InvalidRow>,
}
impl PushResult {
pub fn from_json(json: &Value) -> PushResult {
PushResult {
added: json["added"].as_f64().unwrap_or_default() as u64,
first: json["first"].as_f64(),
last: json["last"].as_f64(),
invalid: json["invalid"]
.as_array()
.into_iter()
.flatten()
.map(|row| InvalidRow {
index: row["index"].as_f64().unwrap_or_default() as u64,
reason: match &row["reason"] {
Value::String(reason) => reason.clone(),
reason => reason.to_string(),
},
})
.collect(),
}
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct PushRefused {
pub status: u16,
pub message: String,
pub invalid: Vec<InvalidRow>,
}
const FEED_PAGE: u64 = 500;
fn dataset_data_url(dataset_id: &str, options: &DataOptions) -> String {
let mut query = format!(
"format={}",
match options.format.unwrap_or_default() {
DataFormat::Csv => "csv",
DataFormat::Json => "json",
}
);
if let Some(snapshot) = &options.snapshot {
query.push_str(&format!("&snapshot={}", encode(snapshot)));
}
if options.sources == Some(true) {
query.push_str("&sources=true");
}
format!(
"{}/marmot/dataset/{dataset_id}/data?{query}",
crate::get_url()
)
}
fn authorized(
request: reqwest::RequestBuilder,
authorization: &CredentialsOrApiKey,
group: Option<&Group>,
) -> reqwest::RequestBuilder {
let request = request.header(
"Authorization",
match authorization {
CredentialsOrApiKey::Credentials(credentials) => format!("Token {}", credentials.token),
CredentialsOrApiKey::ApiKey(key) => format!("ApiKey {}", key.id),
},
);
match group {
Some(group) => request.header("Group", &group.id),
None => request,
}
}
async fn data_body(response: reqwest::Response, csv: bool) -> Result<Value> {
let response = ok(response).await?;
Ok(if csv {
Value::String(response.text().await?)
} else {
response.json().await?
})
}
#[derive(Debug, Clone, PartialEq)]
pub enum SnapshotData {
Json(Value),
Text(String),
Bytes(Vec<u8>),
}
#[derive(Debug, Clone, Default)]
pub struct SnapshotDataOptions {
pub format: Option<DataFormat>,
pub sources: Option<bool>,
}
#[derive(Default)]
struct SnapshotCache {
schema: Option<Schema>,
data: Option<SnapshotData>,
}
#[derive(Clone)]
pub struct Snapshot {
pub id: String,
pub created_at: DateTime<Utc>,
pub valid: bool,
authorization: CredentialsOrApiKey,
workspace_id: String,
dataset_id: String,
group: Option<Group>,
cache: Arc<Mutex<SnapshotCache>>,
}
impl Snapshot {
pub fn new(
authorization: &CredentialsOrApiKey,
workspace_id: &str,
dataset_id: &str,
id: &str,
created_at: DateTime<Utc>,
valid: bool,
group: Option<&Group>,
) -> Snapshot {
Snapshot {
id: id.to_string(),
created_at,
valid,
authorization: authorization.clone(),
workspace_id: workspace_id.to_string(),
dataset_id: dataset_id.to_string(),
group: group.cloned(),
cache: Arc::new(Mutex::new(SnapshotCache::default())),
}
}
fn cache(&self) -> MutexGuard<'_, SnapshotCache> {
self.cache.lock().unwrap_or_else(PoisonError::into_inner)
}
fn url(&self, tail: &str) -> String {
format!(
"{}/marmot/snapshot/{}/{}/{}{tail}",
crate::get_url(),
self.workspace_id,
self.dataset_id,
self.id
)
}
async fn get(&self, tail: &str) -> Result<reqwest::Response> {
let response = authorized(
client().get(self.url(tail)),
&self.authorization,
self.group.as_ref(),
)
.send()
.await?;
ok(response).await
}
pub async fn download(&self, assets: bool, revalidate: bool) -> Result<SnapshotData> {
if !assets
&& !revalidate
&& let Some(data) = self.cache().data.clone()
{
return Ok(data);
}
let response = self.get(&format!("/?assets={assets}")).await?;
if assets {
return Ok(SnapshotData::Bytes(response.bytes().await?.to_vec()));
}
let json = response
.headers()
.get("content-type")
.and_then(|value| value.to_str().ok())
.is_some_and(|value| value.contains("application/json"));
let data = if json {
SnapshotData::Json(response.json().await?)
} else {
SnapshotData::Text(response.text().await?)
};
self.cache().data = Some(data.clone());
Ok(data)
}
pub async fn data(&self, options: SnapshotDataOptions) -> Result<Value> {
let csv = options.format == Some(DataFormat::Csv);
let url = dataset_data_url(
&self.dataset_id,
&DataOptions {
format: options.format,
snapshot: Some(self.id.clone()),
sources: options.sources,
},
);
let response = authorized(client().get(url), &self.authorization, None)
.send()
.await?;
data_body(response, csv).await
}
pub async fn download_schema(&self, revalidate: bool) -> Result<Schema> {
if !revalidate && let Some(schema) = self.cache().schema.clone() {
return Ok(schema);
}
let json: Value = self.get("/schema").await?.json().await?;
let schema = Schema::deserialize(&json)?;
self.cache().schema = Some(schema.clone());
Ok(schema)
}
pub async fn download_log(&self) -> Result<Vec<u8>> {
Ok(self.get("/log").await?.bytes().await?.to_vec())
}
pub async fn download_embeddings(&self) -> Result<Vec<u8>> {
Ok(self.get("/embeddings").await?.bytes().await?.to_vec())
}
pub async fn delete(&self) -> Result<()> {
let response = authorized(
client().delete(self.url("")),
&self.authorization,
self.group.as_ref(),
)
.send()
.await?;
ok(response).await?;
Ok(())
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
#[serde(rename_all = "lowercase")]
pub enum BrowserMode {
Render,
Virtualized,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct BrowserFrame {
pub url: String,
pub reference: String,
pub code: String,
pub mode: BrowserMode,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
#[serde(rename_all = "lowercase")]
pub enum CancelReason {
User,
Tokens,
Time,
Cost,
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct Candidate {
pub value: Value,
pub source: String,
}
pub trait ResearchCallbacks: Send + Sync {
fn on_connection(&self, _ongoing: bool) {}
fn on_query(&self, _query: &str) {}
fn on_source(&self, _url: &str, _title: &str, _content_type: &str) {}
fn on_answer(
&self,
_field: &str,
_value: &Value,
_converged: bool,
_answers_count: u64,
_sources: Option<&[String]>,
) {
}
fn on_nomination(
&self,
_scope: &str,
_row: &Value,
_committed_count: u64,
_sources: Option<&[String]>,
) {
}
fn on_supervisor_query(&self, _question: &str) {}
fn on_supervisor_search(&self, _query: &str) {}
fn on_supervisor_answer(&self, _question: &str, _answer: &str) {}
fn on_supervisor_refused(&self, _question: &str) {}
fn on_browser(&self, _frame: &BrowserFrame) {}
fn on_browser_step(&self, _url: &str, _step: u64, _action: &BrowserTool) {}
fn on_compress(&self) {}
fn on_info(&self, _message: &str) {}
fn on_done(&self, _snapshot: &Snapshot) {}
fn on_cancelled(&self, _reason: CancelReason) {}
fn on_convergence_failed(&self, _field: &str, _reason: &str, _candidates: &[Candidate]) {}
fn on_error(&self, _message: &str) {}
}
impl ResearchCallbacks for () {}
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
pub struct ResearchOptions {
pub text: Option<Text>,
pub embedding: Option<Embedding>,
}
#[derive(Debug, Clone, PartialEq)]
pub enum RowsOrCsv {
Rows(Vec<Value>),
Csv(String),
}
#[derive(Debug, Clone, Default)]
pub struct AddRowsOptions {
pub source: Option<String>,
}
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
pub struct FeedQuery {
pub since: u64,
pub max: u64,
}
fn text(value: &Value) -> String {
match value {
Value::Null => String::new(),
Value::String(text) => text.clone(),
value => value.to_string(),
}
}
#[derive(Clone)]
pub struct Dataset {
pub id: String,
pub workspace_id: String,
pub name: String,
pub schema: Schema,
pub termination: Vec<Termination>,
pub content_types: Vec<String>,
pub proxy: ProxyMode,
pub document_modes: Vec<DocumentMode>,
snapshots: Arc<Mutex<Vec<Snapshot>>>,
authorization: CredentialsOrApiKey,
group: Option<Group>,
ws: Arc<Mutex<Option<mpsc::UnboundedSender<Message>>>>,
}
impl Dataset {
pub fn snapshots(&self) -> Vec<Snapshot> {
self.snapshots
.lock()
.unwrap_or_else(PoisonError::into_inner)
.clone()
}
pub async fn research(
&self,
callbacks: &dyn ResearchCallbacks,
options: ResearchOptions,
) -> Result<()> {
self.run_socket(Vec::new(), callbacks, options, false)
.await
.map(|_| ())
}
pub async fn join_browser(
&self,
access_code: &str,
) -> Result<(super::browser::BrowserRoom, Presence)> {
super::browser::BrowserRoom::join(&self.authorization, access_code, Default::default())
.await
}
pub async fn reconnect(
&self,
callbacks: &dyn ResearchCallbacks,
options: ResearchOptions,
) -> Result<()> {
self.run_socket(Vec::new(), callbacks, options, true)
.await
.map(|_| ())
}
async fn run_socket(
&self,
extra: Vec<(&str, String)>,
callbacks: &dyn ResearchCallbacks,
options: ResearchOptions,
reconnect: bool,
) -> Result<Option<Snapshot>> {
let mut params = Map::new();
params.insert(
"Authorization".to_string(),
Value::from(match &self.authorization {
CredentialsOrApiKey::Credentials(credentials) => {
format!("Token {}", credentials.token)
}
CredentialsOrApiKey::ApiKey(key) => format!("ApiKey {}", key.id),
}),
);
if let Some(group) = &self.group {
params.insert("Group".to_string(), Value::from(group.id.as_str()));
}
params.insert(
"Workspace".to_string(),
Value::from(self.workspace_id.as_str()),
);
params.insert("Dataset".to_string(), Value::from(self.id.as_str()));
for (key, value) in extra {
params.insert(key.to_string(), Value::from(value));
}
if let Some(model) = options.text {
params.insert("Text".to_string(), Value::from(model.as_str()));
}
if let Some(model) = options.embedding {
params.insert("Embedding".to_string(), Value::from(model.as_str()));
}
if reconnect {
params.insert("Attach".to_string(), Value::from("true"));
}
let url = format!(
"{}/marmot/research?authorization={}",
websocket(&crate::get_url()),
encode(&Value::Object(params).to_string())
);
let socket = match tokio_tungstenite::connect_async(url.as_str()).await {
Ok((socket, _)) => socket,
Err(_) if reconnect => {
return Err(Error::Message(
"Failed to connect to research run".to_string(),
));
}
Err(_) => {
callbacks.on_error("WebSocket connection failed");
return Ok(None);
}
};
let (mut write, mut read) = socket.split();
let (outbox, mut queue) = mpsc::unbounded_channel::<Message>();
tokio::spawn(async move {
while let Some(message) = queue.recv().await {
if write.send(message).await.is_err() {
break;
}
}
let _ = write.close().await;
});
*self.ws.lock().unwrap_or_else(PoisonError::into_inner) = Some(outbox);
let close = || {
if let Some(outbox) = self
.ws
.lock()
.unwrap_or_else(PoisonError::into_inner)
.take()
{
let _ = outbox.send(Message::Close(None));
}
};
let mut attached = false;
let strings =
|value: &Value| -> Option<Vec<String>> { serde_json::from_value(value.clone()).ok() };
loop {
let packet = match read.next().await {
Some(Ok(Message::Text(raw))) => match serde_json::from_str::<Value>(raw.as_str()) {
Ok(packet) => packet,
Err(_) => continue,
},
Some(Ok(Message::Close(_))) | None => {
self.ws
.lock()
.unwrap_or_else(PoisonError::into_inner)
.take();
if reconnect && !attached {
return Err(Error::Message("No active research run".to_string()));
}
return Ok(None);
}
Some(Err(_)) => {
self.ws
.lock()
.unwrap_or_else(PoisonError::into_inner)
.take();
if reconnect && !attached {
return Err(Error::Message(
"Failed to connect to research run".to_string(),
));
}
callbacks.on_error("WebSocket connection failed");
return Ok(None);
}
Some(Ok(_)) => continue,
};
match packet["event"].as_str().unwrap_or_default() {
"connection" => {
attached = packet["ongoing"] == true;
callbacks.on_connection(attached);
}
"inactive" => {
close();
return Err(Error::Message("No active research run".to_string()));
}
"query" => callbacks.on_query(&text(&packet["query"])),
"source" => callbacks.on_source(
&text(&packet["url"]),
&text(&packet["title"]),
&text(&packet["contentType"]),
),
"answer" => callbacks.on_answer(
&text(&packet["field"]),
&packet["value"],
packet["converged"] == true,
packet["answersCount"].as_u64().unwrap_or_default(),
strings(&packet["sources"]).as_deref(),
),
"nomination_accepted" => callbacks.on_nomination(
&text(&packet["scope"]),
&packet["row"],
packet["committedCount"].as_u64().unwrap_or_default(),
strings(&packet["sources"]).as_deref(),
),
"supervisor_query" => callbacks.on_supervisor_query(&text(&packet["question"])),
"supervisor_search" => callbacks.on_supervisor_search(&text(&packet["query"])),
"supervisor_answer" => callbacks
.on_supervisor_answer(&text(&packet["question"]), &text(&packet["answer"])),
"supervisor_refused" => callbacks.on_supervisor_refused(&text(&packet["question"])),
"browser" => {
if let Ok(frame) = serde_json::from_value::<BrowserFrame>(packet.clone()) {
callbacks.on_browser(&frame);
}
}
"browser_step" => {
if let Ok(action) =
serde_json::from_value::<BrowserTool>(packet["action"].clone())
{
callbacks.on_browser_step(
&text(&packet["url"]),
packet["step"].as_u64().unwrap_or_default(),
&action,
);
}
}
"compress" => callbacks.on_compress(),
"info" => callbacks.on_info(&text(&packet["message"])),
"done" => {
let snapshot = Snapshot::new(
&self.authorization,
&self.workspace_id,
&self.id,
&text(&packet["snapshotId"]),
Utc::now(),
packet["valid"].as_bool().unwrap_or(true),
self.group.as_ref(),
);
self.snapshots
.lock()
.unwrap_or_else(PoisonError::into_inner)
.push(snapshot.clone());
callbacks.on_done(&snapshot);
close();
return Ok(Some(snapshot));
}
"cancelled" => {
callbacks.on_cancelled(
serde_json::from_value(packet["reason"].clone())
.unwrap_or(CancelReason::User),
);
close();
return Ok(None);
}
"convergence_failed" => {
let candidates: Vec<Candidate> =
serde_json::from_value(packet["candidates"].clone()).unwrap_or_default();
callbacks.on_convergence_failed(
&text(&packet["field"]),
&text(&packet["reason"]),
&candidates,
);
close();
return Ok(None);
}
"error" => {
callbacks.on_error(&text(&packet["message"]));
close();
return Ok(None);
}
_ => {}
}
}
}
pub async fn cancel(&self) -> Result<()> {
if let Some(outbox) = self
.ws
.lock()
.unwrap_or_else(PoisonError::into_inner)
.as_ref()
{
let _ = outbox.send(Message::text(json!({ "event": "cancel" }).to_string()));
}
Ok(())
}
async fn update(&self, field: &str, value: Value) -> Result<()> {
let response = authorized(
client().post(format!("{}/marmot/dataset/update", crate::get_url())),
&self.authorization,
self.group.as_ref(),
)
.json(&json!({ "workspaceId": self.workspace_id, "id": self.id, field: value }))
.send()
.await?;
ok(response).await?;
Ok(())
}
pub async fn rename(&mut self, name: &str) -> Result<()> {
self.update("name", Value::from(name)).await?;
self.name = name.to_string();
Ok(())
}
pub async fn save_schema(&mut self, schema: Schema) -> Result<()> {
self.update("schema", schema.serialize()).await?;
self.schema = schema;
Ok(())
}
pub async fn migrate(
&mut self,
schema: Schema,
seed_snapshot_id: Option<&str>,
callbacks: &dyn ResearchCallbacks,
options: ResearchOptions,
) -> Result<()> {
let migrate =
json!({ "seedSnapshotId": seed_snapshot_id, "schema": schema.serialize() }).to_string();
let done = self
.run_socket(vec![("Migrate", migrate)], callbacks, options, false)
.await?;
if done.is_some() {
self.schema = schema;
}
Ok(())
}
pub async fn set_termination(&mut self, termination: Vec<Termination>) -> Result<()> {
self.update("termination", serde_json::to_value(&termination)?)
.await?;
self.termination = termination;
Ok(())
}
pub async fn set_content_types(&mut self, content_types: Vec<String>) -> Result<()> {
self.update("contentTypes", json!(content_types)).await?;
self.content_types = content_types;
Ok(())
}
pub async fn set_proxy(&mut self, proxy: ProxyMode) -> Result<()> {
self.update("proxy", serde_json::to_value(proxy)?).await?;
self.proxy = proxy;
Ok(())
}
pub async fn set_document_modes(&mut self, document_modes: Vec<DocumentMode>) -> Result<()> {
self.update("documentModes", serde_json::to_value(&document_modes)?)
.await?;
self.document_modes = document_modes;
Ok(())
}
pub async fn remove(&self) -> Result<()> {
let response = authorized(
client().post(format!("{}/marmot/dataset/remove", crate::get_url())),
&self.authorization,
self.group.as_ref(),
)
.json(&json!({ "workspaceId": self.workspace_id, "id": self.id }))
.send()
.await?;
ok(response).await?;
Ok(())
}
pub async fn data(&self, options: DataOptions) -> Result<Value> {
let csv = options.format == Some(DataFormat::Csv);
let response = authorized(
client().get(dataset_data_url(&self.id, &options)),
&self.authorization,
None,
)
.send()
.await?;
data_body(response, csv).await
}
pub fn data_url(&self, options: &DataOptions) -> String {
dataset_data_url(&self.id, options)
}
pub async fn add_rows(&self, rows: RowsOrCsv, options: AddRowsOptions) -> Result<PushResult> {
let mut url = format!("{}/marmot/dataset/{}/rows", crate::get_url(), self.id);
if let Some(source) = options.source.filter(|source| !source.is_empty()) {
url.push_str(&format!("?source={}", encode(&source)));
}
let request = authorized(client().post(url), &self.authorization, None);
let request = match rows {
RowsOrCsv::Csv(csv) => request.header("Content-Type", "text/csv").body(csv),
RowsOrCsv::Rows(rows) => request.json(&rows),
};
let response = request.send().await?;
if !response.status().is_success() {
let status = response.status().as_u16();
let text = response.text().await.unwrap_or_default();
let body: Value = serde_json::from_str(&text).unwrap_or(Value::Null);
return Err(Error::PushRefused(PushRefused {
status,
message: body["message"].as_str().map(str::to_string).unwrap_or(text),
invalid: PushResult::from_json(&body).invalid,
}));
}
Ok(PushResult::from_json(&response.json().await?))
}
pub async fn updates(&self, query: FeedQuery) -> Result<Vec<FeedRow>> {
let response = authorized(
client().get(format!(
"{}/marmot/dataset/{}/updates?since={}&max={}",
crate::get_url(),
self.id,
query.since,
query.max
)),
&self.authorization,
None,
)
.send()
.await?;
let json: Value = ok(response).await?.json().await?;
Ok(json
.as_array()
.into_iter()
.flatten()
.map(FeedRow::from_json)
.collect())
}
pub fn live(&self, since: Option<u64>) -> BoxStream<'static, FeedRow> {
let (rows, received) = mpsc::unbounded_channel::<FeedRow>();
let dataset = self.clone();
tokio::spawn(async move {
let mut since = since.unwrap_or(0);
while !rows.is_closed() {
dataset.live_once(&mut since, &rows).await;
if rows.is_closed() {
break;
}
tokio::time::sleep(Duration::from_secs(2)).await;
}
});
stream::unfold(received, |mut received| async move {
received.recv().await.map(|row| (row, received))
})
.boxed()
}
async fn live_once(&self, since: &mut u64, rows: &mpsc::UnboundedSender<FeedRow>) {
let params = json!({
"Authorization": match &self.authorization {
CredentialsOrApiKey::Credentials(credentials) => format!("Token {}", credentials.token),
CredentialsOrApiKey::ApiKey(key) => format!("ApiKey {}", key.id),
},
"Dataset": self.id,
});
let url = format!(
"{}/marmot/dataset/live?authorization={}",
websocket(&crate::get_url()),
encode(¶ms.to_string())
);
let Ok((socket, _)) = tokio_tungstenite::connect_async(url.as_str()).await else {
return;
};
let (mut write, mut read) = socket.split();
let (held, mut pending) = mpsc::unbounded_channel::<FeedRow>();
let (ready, is_ready) = oneshot::channel::<()>();
let reader = tokio::spawn(async move {
let mut ready = Some(ready);
while let Some(Ok(message)) = read.next().await {
let Message::Text(raw) = message else {
if matches!(message, Message::Close(_)) {
break;
}
continue;
};
let Ok(packet) = serde_json::from_str::<Value>(raw.as_str()) else {
continue;
};
if packet["event"] == "ready" {
if let Some(ready) = ready.take() {
let _ = ready.send(());
}
} else if packet["event"] == "rows" {
for row in packet["rows"].as_array().into_iter().flatten() {
let _ = held.send(FeedRow::from_json(row));
}
}
}
});
if is_ready.await.is_ok() {
loop {
let Ok(page) = self
.updates(FeedQuery {
since: *since,
max: FEED_PAGE,
})
.await
else {
break;
};
let count = page.len() as u64;
for row in page {
if row.seq <= *since {
continue;
}
*since = row.seq;
if rows.send(row).is_err() {
reader.abort();
let _ = write.close().await;
return;
}
}
if count < FEED_PAGE {
break;
}
}
loop {
tokio::select! {
row = pending.recv() => {
let Some(row) = row else {
break;
};
if row.seq <= *since {
continue;
}
*since = row.seq;
if rows.send(row).is_err() {
break;
}
}
_ = rows.closed() => break,
}
}
}
reader.abort();
let _ = write.close().await;
}
pub async fn scraper(&self) -> Result<Option<Scraper>> {
get_scraper(&self.authorization, &self.id).await
}
pub async fn keep_updated(&self, settings: &ScraperSettings) -> Result<Scraper> {
create_scraper(&self.authorization, &self.id, settings).await
}
pub async fn update_scraper(&self, settings: &ScraperSettings) -> Result<Scraper> {
update_scraper(&self.authorization, &self.id, settings).await
}
pub async fn rebuild_scraper(&self, note: Option<&str>) -> Result<()> {
rebuild_scraper(&self.authorization, &self.id, note).await
}
pub async fn stop_keeping_updated(&self) -> Result<()> {
remove_scraper(&self.authorization, &self.id).await
}
}
#[derive(Debug, Clone, Default)]
pub struct GroupOptions {
pub group: Option<Group>,
}
pub async fn list_datasets(
authorization: &CredentialsOrApiKey,
workspace_id: &str,
opts: GroupOptions,
) -> Result<Vec<Dataset>> {
let response = authorized(
client().post(format!("{}/marmot/dataset/list", crate::get_url())),
authorization,
opts.group.as_ref(),
)
.json(&json!({ "workspaceId": workspace_id }))
.send()
.await?;
let data: Value = ok(response).await?.json().await?;
data.as_array()
.into_iter()
.flatten()
.map(|d| {
let id = text(&d["id"]);
let snapshots = d["snapshots"]
.as_array()
.into_iter()
.flatten()
.map(|s| {
Snapshot::new(
authorization,
workspace_id,
&id,
&text(&s["id"]),
crate::tickets::parse_date(&s["createdAt"]).unwrap_or_default(),
s["valid"].as_bool().unwrap_or(true),
opts.group.as_ref(),
)
})
.collect();
Ok(Dataset {
workspace_id: workspace_id.to_string(),
name: text(&d["name"]),
schema: Schema::deserialize(&d["schema"])?,
termination: serde_json::from_value(d["termination"].clone()).unwrap_or_default(),
content_types: serde_json::from_value(d["contentTypes"].clone())
.unwrap_or_default(),
proxy: serde_json::from_value(d["proxy"].clone()).unwrap_or(ProxyMode::Datacenter),
document_modes: serde_json::from_value(d["documentModes"].clone())
.unwrap_or_else(|_| vec![DocumentMode::Extraction]),
snapshots: Arc::new(Mutex::new(snapshots)),
authorization: authorization.clone(),
group: opts.group.clone(),
ws: Arc::new(Mutex::new(None)),
id,
})
})
.collect()
}
#[derive(Clone)]
pub struct CreateDatasetOptions {
pub name: String,
pub schema: Schema,
pub termination: Option<Vec<Termination>>,
pub content_types: Option<Vec<String>>,
pub proxy: Option<ProxyMode>,
pub document_modes: Option<Vec<DocumentMode>>,
}
pub async fn create_dataset(
authorization: &CredentialsOrApiKey,
workspace_id: &str,
options: &CreateDatasetOptions,
opts: GroupOptions,
) -> Result<String> {
let response = authorized(
client().post(format!("{}/marmot/dataset/create", crate::get_url())),
authorization,
opts.group.as_ref(),
)
.json(&json!({
"workspaceId": workspace_id,
"name": options.name,
"schema": options.schema.serialize(),
"termination": options.termination.clone().unwrap_or_default(),
"contentTypes": options.content_types.clone().unwrap_or_default(),
"proxy": options.proxy.unwrap_or(ProxyMode::Datacenter),
"documentModes": options.document_modes.clone().unwrap_or_else(|| vec![DocumentMode::Extraction]),
}))
.send()
.await?;
Ok(ok(response).await?.text().await?)
}
#[derive(Debug, Clone)]
pub struct Workspace {
pub id: String,
pub name: String,
pub description: String,
pub created_at: i64,
pub datasets: u64,
pub files: u64,
authorization: CredentialsOrApiKey,
group: Option<Group>,
}
impl Workspace {
fn opts(&self) -> GroupOptions {
GroupOptions {
group: self.group.clone(),
}
}
pub async fn list_datasets(&self) -> Result<Vec<Dataset>> {
list_datasets(&self.authorization, &self.id, self.opts()).await
}
pub async fn create_dataset(&self, options: &CreateDatasetOptions) -> Result<String> {
create_dataset(&self.authorization, &self.id, options, self.opts()).await
}
pub async fn rename(&mut self, name: &str) -> Result<()> {
update_workspace(
&self.authorization,
&self.id,
&PartialWorkspace {
name: Some(name.to_string()),
description: None,
},
self.opts(),
)
.await?;
self.name = name.to_string();
Ok(())
}
pub async fn set_description(&mut self, description: &str) -> Result<()> {
update_workspace(
&self.authorization,
&self.id,
&PartialWorkspace {
name: None,
description: Some(description.to_string()),
},
self.opts(),
)
.await?;
self.description = description.to_string();
Ok(())
}
async fn post(&self, route: &str, body: Value) -> Result<reqwest::Response> {
let response = authorized(
client().post(format!("{}{route}", crate::get_url())),
&self.authorization,
self.group.as_ref(),
)
.json(&body)
.send()
.await?;
ok(response).await
}
pub async fn remove(&self) -> Result<()> {
self.post("/marmot/workspace/remove", json!({ "id": self.id }))
.await?;
Ok(())
}
pub async fn get_files(&self) -> Result<Vec<WorkspaceFile>> {
Ok(get_workspace(&self.authorization, &self.id, self.opts())
.await?
.files)
}
pub async fn upload_file(
&self,
name: &str,
content_type: &str,
data: &[u8],
) -> Result<WorkspaceFile> {
use base64::Engine;
let data = base64::engine::general_purpose::STANDARD.encode(data);
Ok(self
.post(
"/marmot/workspace/file/upload",
json!({ "workspaceId": self.id, "name": name, "contentType": content_type, "data": data }),
)
.await?
.json()
.await?)
}
pub async fn download_file(&self, file: &WorkspaceFile) -> Result<Vec<u8>> {
Ok(self
.post(
"/marmot/workspace/file/get",
json!({ "key": file.key, "contentType": file.content_type, "workspaceId": self.id }),
)
.await?
.bytes()
.await?
.to_vec())
}
pub async fn remove_file(&self, key: &str) -> Result<()> {
self.post(
"/marmot/workspace/file/remove",
json!({ "workspaceId": self.id, "key": key }),
)
.await?;
Ok(())
}
}
pub async fn list_workspaces(
authorization: &CredentialsOrApiKey,
opts: GroupOptions,
) -> Result<Vec<Workspace>> {
let response = authorized(
client().post(format!("{}/marmot/workspace/list", crate::get_url())),
authorization,
opts.group.as_ref(),
)
.header("Content-Type", "application/json")
.send()
.await?;
let data: Value = ok(response).await?.json().await?;
Ok(data
.as_array()
.into_iter()
.flatten()
.map(|w| Workspace {
id: text(&w["id"]),
name: text(&w["name"]),
description: text(&w["description"]),
created_at: w["createdAt"].as_f64().unwrap_or_default() as i64,
datasets: w["datasets"].as_f64().unwrap_or_default() as u64,
files: w["files"].as_f64().unwrap_or_default() as u64,
authorization: authorization.clone(),
group: opts.group.clone(),
})
.collect())
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct WorkspaceDetail {
pub id: String,
pub name: String,
#[serde(default)]
pub description: String,
#[serde(default)]
pub created_at: i64,
#[serde(default)]
pub files: Vec<WorkspaceFile>,
}
pub async fn get_workspace(
authorization: &CredentialsOrApiKey,
id: &str,
opts: GroupOptions,
) -> Result<WorkspaceDetail> {
let response = authorized(
client().post(format!("{}/marmot/workspace/get", crate::get_url())),
authorization,
opts.group.as_ref(),
)
.json(&json!({ "id": id }))
.send()
.await?;
Ok(ok(response).await?.json().await?)
}
pub async fn create_workspace(
authorization: &CredentialsOrApiKey,
name: &str,
description: Option<&str>,
opts: GroupOptions,
) -> Result<String> {
let response = authorized(
client().post(format!("{}/marmot/workspace/create", crate::get_url())),
authorization,
opts.group.as_ref(),
)
.json(&json!({ "name": name, "description": description.unwrap_or_default() }))
.send()
.await?;
Ok(ok(response).await?.text().await?)
}
pub async fn update_workspace(
authorization: &CredentialsOrApiKey,
id: &str,
update: &PartialWorkspace,
opts: GroupOptions,
) -> Result<()> {
let mut body = Map::new();
body.insert("id".to_string(), Value::from(id));
if let Value::Object(fields) = serde_json::to_value(update)? {
body.extend(fields);
}
let response = authorized(
client().post(format!("{}/marmot/workspace/update", crate::get_url())),
authorization,
opts.group.as_ref(),
)
.json(&body)
.send()
.await?;
ok(response).await?;
Ok(())
}