Skip to main content

fauna_api/marmot/
media.rs

1use std::collections::HashMap;
2use std::sync::{Mutex, PoisonError};
3
4use base64::Engine;
5use base64::engine::general_purpose::STANDARD;
6use chrono::{DateTime, SecondsFormat, Utc};
7use serde::{Deserialize, Serialize};
8use serde_json::{Map, Value};
9
10use crate::authentification::CredentialsOrApiKey;
11use crate::drive::{FileDates, Progress, from_bytes, parse_stream};
12use crate::error::{Error, Result};
13use crate::http::{client, encode, ok, progress_body};
14use crate::parrot::generative::{File, Part, TextPart};
15
16pub const ATTACHMENT_TYPES: [(&str, &str); 15] = [
17    ("image/jpeg", "image"),
18    ("image/png", "image"),
19    ("image/webp", "image"),
20    ("image/heic", "image"),
21    ("image/heif", "image"),
22    ("audio/mpeg", "audio_file"),
23    ("audio/wav", "audio_file"),
24    ("video/mp4", "movie"),
25    ("video/webm", "movie"),
26    ("video/quicktime", "movie"),
27    ("application/pdf", "picture_as_pdf"),
28    ("text/plain", "description"),
29    ("text/markdown", "description"),
30    ("text/csv", "table_chart"),
31    ("text/html", "language"),
32];
33
34/// Resolve a usable MIME when a file's type is unknown (common for HEIC).
35pub fn mime_from_file_name(name: &str) -> Option<&'static str> {
36    let extension = if name.contains('.') {
37        name.rsplit('.').next().unwrap_or_default().to_lowercase()
38    } else {
39        String::new()
40    };
41    match extension.as_str() {
42        "jpg" | "jpeg" => Some("image/jpeg"),
43        "png" => Some("image/png"),
44        "webp" => Some("image/webp"),
45        "heic" => Some("image/heic"),
46        "heif" => Some("image/heif"),
47        "mp3" => Some("audio/mpeg"),
48        "wav" => Some("audio/wav"),
49        "mp4" => Some("video/mp4"),
50        "webm" => Some("video/webm"),
51        "mov" | "qt" => Some("video/quicktime"),
52        "pdf" => Some("application/pdf"),
53        "txt" => Some("text/plain"),
54        "md" | "markdown" => Some("text/markdown"),
55        "csv" => Some("text/csv"),
56        "html" | "htm" => Some("text/html"),
57        "json" => Some("application/json"),
58        "xml" => Some("application/xml"),
59        _ => None,
60    }
61}
62
63#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
64#[serde(rename_all = "camelCase")]
65pub struct Website {
66    pub url: String,
67    pub content_type: String,
68    pub title: String,
69    pub description: String,
70}
71
72/// How content is pulled from URLs: `datacenter` fetches directly from Cloud Run
73/// egress, `residential` routes through residential proxies for sites that block
74/// datacenter IPs.
75#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
76#[serde(rename_all = "lowercase")]
77pub enum ProxyMode {
78    Residential,
79    Datacenter,
80}
81
82/// How a document source is pulled: `extraction` parses HTML/PDF text directly,
83/// `render` screenshots the headless page for the model to read visually,
84/// `virtualized` drives the live page with a bounded Puppeteer agent toward a
85/// goal. A dataset enables one or more; with several, the research agent picks
86/// per source.
87#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
88#[serde(rename_all = "lowercase")]
89pub enum DocumentMode {
90    Extraction,
91    Render,
92    Virtualized,
93}
94
95#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
96#[serde(rename_all = "camelCase")]
97pub struct ContentTypeInfo {
98    pub content_type: String,
99    pub name: String,
100    pub extensions: Vec<String>,
101}
102
103static TYPES_CACHE: Mutex<Option<HashMap<String, Vec<ContentTypeInfo>>>> = Mutex::new(None);
104
105pub async fn get_types() -> Result<HashMap<String, Vec<ContentTypeInfo>>> {
106    if let Some(types) = TYPES_CACHE
107        .lock()
108        .unwrap_or_else(PoisonError::into_inner)
109        .clone()
110    {
111        return Ok(types);
112    }
113
114    let response = client()
115        .get(format!("{}/media/types", super::get_url()))
116        .send()
117        .await?;
118    let types: HashMap<String, Vec<ContentTypeInfo>> = ok(response).await?.json().await?;
119
120    *TYPES_CACHE.lock().unwrap_or_else(PoisonError::into_inner) = Some(types.clone());
121    Ok(types)
122}
123
124/// `Uint8Array | Uint8Array[]` in the TypeScript SDK: one body, or one per segment.
125#[derive(Debug, Clone, PartialEq, Eq)]
126pub enum BytesOrBytesList {
127    Bytes(Vec<u8>),
128    BytesList(Vec<Vec<u8>>),
129}
130
131#[derive(Debug, Clone, PartialEq, Eq)]
132pub struct PullResponse {
133    pub data: BytesOrBytesList,
134    pub content_type: String,
135}
136
137/// The `options` argument of `query`.
138#[derive(Debug, Clone, Default)]
139pub struct QueryOptions {
140    pub cursor: u32,
141    pub max: u32,
142    pub start: Option<DateTime<Utc>>,
143    pub end: Option<DateTime<Utc>>,
144}
145
146pub async fn query(
147    authorization: &CredentialsOrApiKey,
148    query: &str,
149    options: QueryOptions,
150) -> Result<Vec<Website>> {
151    let mut body = Map::new();
152    body.insert("query".to_string(), Value::from(query));
153    body.insert("cursor".to_string(), Value::from(options.cursor));
154    body.insert("max".to_string(), Value::from(options.max));
155    if let Some(start) = options.start {
156        body.insert(
157            "start".to_string(),
158            Value::from(start.to_rfc3339_opts(SecondsFormat::Millis, true)),
159        );
160    }
161    if let Some(end) = options.end {
162        body.insert(
163            "end".to_string(),
164            Value::from(end.to_rfc3339_opts(SecondsFormat::Millis, true)),
165        );
166    }
167    let response = client()
168        .post(format!("{}/media/query", super::get_url()))
169        .header(
170            "Authorization",
171            match authorization {
172                CredentialsOrApiKey::Credentials(credentials) => {
173                    format!("Token {}", credentials.token)
174                }
175                CredentialsOrApiKey::ApiKey(key) => format!("ApiKey {}", key.id),
176            },
177        )
178        .json(&body)
179        .send()
180        .await?;
181    Ok(response.json().await?)
182}
183
184/// The `options` argument of `pull`.
185#[derive(Debug, Clone, Default, Serialize)]
186pub struct PullOptions {
187    #[serde(skip_serializing_if = "Option::is_none")]
188    pub modals: Option<Vec<String>>,
189    #[serde(skip_serializing_if = "Option::is_none")]
190    pub segment: Option<[f64; 2]>,
191    #[serde(skip_serializing_if = "Option::is_none")]
192    pub split: Option<f64>,
193    #[serde(skip_serializing_if = "Option::is_none")]
194    pub proxy: Option<ProxyMode>,
195}
196
197/// Fetches `url` through Marmot. `request` is the fetch options (method, headers,
198/// body) as JSON.
199pub async fn pull(
200    authorization: &CredentialsOrApiKey,
201    url: &str,
202    request: Option<&Value>,
203    options: PullOptions,
204) -> Result<Option<PullResponse>> {
205    let effective_options = PullOptions {
206        proxy: Some(options.proxy.unwrap_or(ProxyMode::Datacenter)),
207        ..options
208    };
209    let mut builder = client()
210        .post(format!("{}/media/pull", super::get_url()))
211        .header("Content-Type", "application/json")
212        .header(
213            "Authorization",
214            match authorization {
215                CredentialsOrApiKey::Credentials(credentials) => {
216                    format!("Token {}", credentials.token)
217                }
218                CredentialsOrApiKey::ApiKey(key) => format!("ApiKey {}", key.id),
219            },
220        )
221        .header("Url", url);
222    if let Some(request) = request {
223        builder = builder.header("Request", request.to_string());
224    }
225    let response = builder
226        .header("Options", serde_json::to_string(&effective_options)?)
227        .send()
228        .await?;
229
230    if !response.status().is_success() {
231        return Err(Error::Message(format!("Failed to fetch website at {url}.")));
232    }
233
234    let content_type = response
235        .headers()
236        .get("Content-Type")
237        .and_then(|value| value.to_str().ok())
238        .filter(|value| !value.is_empty())
239        .unwrap_or("application/octet-stream")
240        .to_string();
241
242    let buffer = parse_stream(response, None::<fn(u64)>).await?;
243
244    Ok(Some(PullResponse {
245        content_type,
246        data: BytesOrBytesList::Bytes(buffer.into_iter().next().unwrap_or_default()),
247    }))
248}
249
250/// A kind of file a text model reads itself.
251#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
252#[serde(rename_all = "lowercase")]
253pub enum Input {
254    Image,
255    Video,
256    Audio,
257}
258
259impl Input {
260    fn as_str(self) -> &'static str {
261        match self {
262            Input::Image => "image",
263            Input::Video => "video",
264            Input::Audio => "audio",
265        }
266    }
267}
268
269fn chat_parts(authorization: &CredentialsOrApiKey, raw: &[Value]) -> Vec<Part> {
270    raw.iter()
271        .map(|part| {
272            if part["type"] == "FILE" {
273                let text = |key: &str, fallback: &str| match &part[key] {
274                    Value::Null => fallback.to_string(),
275                    Value::String(value) => value.clone(),
276                    value => value.to_string(),
277                };
278                Part::File(File {
279                    file: from_bytes(
280                        authorization.clone(),
281                        &text("name", "file"),
282                        &text("contentType", "application/octet-stream"),
283                        STANDARD
284                            .decode(part["data"].as_str().unwrap_or_default())
285                            .unwrap_or_default(),
286                        FileDates::default(),
287                    ),
288                })
289            } else {
290                Part::Text(TextPart {
291                    text: match &part["text"] {
292                        Value::Null => String::new(),
293                        Value::String(text) => text.clone(),
294                        text => text.to_string(),
295                    },
296                })
297            }
298        })
299        .collect()
300}
301
302/// `Part[] | Part[][]` in the TypeScript SDK: one list, or one per segment.
303#[derive(Debug, Clone)]
304pub enum PartsOrPartsList {
305    Parts(Vec<Part>),
306    PartsList(Vec<Vec<Part>>),
307}
308
309/// The `options` argument of `chat`.
310#[derive(Debug, Clone, Default)]
311pub struct ChatOptions {
312    pub name: Option<String>,
313    pub takes: Option<Vec<Input>>,
314    pub segment: Option<[f64; 2]>,
315    pub split: Option<f64>,
316}
317
318/// A media file as message parts for a model that takes `takes` natively (images only
319/// by default). What it takes passes through; the rest becomes what it can take: a
320/// video becomes a line, its frames and its sound (audio, or a description of it),
321/// audio becomes a description, and documents become their text. With `split`, one
322/// list of parts per segment of that many milliseconds. None when it can't be read.
323pub async fn chat(
324    authorization: &CredentialsOrApiKey,
325    content_type: &str,
326    data: &[u8],
327    options: ChatOptions,
328) -> Result<Option<PartsOrPartsList>> {
329    let takes = options
330        .takes
331        .unwrap_or_else(|| vec![Input::Image])
332        .iter()
333        .map(|input| input.as_str())
334        .collect::<Vec<_>>()
335        .join(",");
336    let mut request = client()
337        .post(format!("{}/media/chat", super::get_url()))
338        .header("Content-Type", content_type)
339        .header(
340            "Authorization",
341            match authorization {
342                CredentialsOrApiKey::Credentials(credentials) => {
343                    format!("Token {}", credentials.token)
344                }
345                CredentialsOrApiKey::ApiKey(key) => format!("ApiKey {}", key.id),
346            },
347        )
348        .header("Takes", takes);
349    if let Some(name) = options.name.as_deref().filter(|name| !name.is_empty()) {
350        request = request.header("Name", encode(name));
351    }
352    if let Some([start, end]) = options.segment {
353        request = request.header("Segment", format!("{start},{end}"));
354    }
355    if let Some(split) = options.split.filter(|split| *split != 0.0) {
356        request = request.header("Split", split.to_string());
357    }
358    let response = request.body(data.to_vec()).send().await?;
359
360    if !response.status().is_success() {
361        return Ok(None);
362    }
363
364    let json: Value = response.json().await?;
365    let Some(rows) = json.as_array() else {
366        return Ok(None);
367    };
368    if options.split.is_some_and(|split| split != 0.0) {
369        return Ok(Some(PartsOrPartsList::PartsList(
370            rows.iter()
371                .map(|segment| {
372                    chat_parts(authorization, segment.as_array().map_or(&[], Vec::as_slice))
373                })
374                .collect(),
375        )));
376    }
377    Ok(Some(PartsOrPartsList::Parts(chat_parts(
378        authorization,
379        rows,
380    ))))
381}
382
383pub async fn parse(
384    authorization: &CredentialsOrApiKey,
385    content_type: &str,
386    data: &[u8],
387    on_progress: Option<Progress>,
388) -> Result<String> {
389    let response = client()
390        .post(format!("{}/media/parse", super::get_url()))
391        .header(
392            "Authorization",
393            match authorization {
394                CredentialsOrApiKey::Credentials(credentials) => {
395                    format!("Token {}", credentials.token)
396                }
397                CredentialsOrApiKey::ApiKey(key) => format!("ApiKey {}", key.id),
398            },
399        )
400        .header(
401            "Content-Type",
402            if content_type.is_empty() {
403                "application/octet-stream"
404            } else {
405                content_type
406            },
407        )
408        .header("Content-Length", data.len())
409        .body(progress_body(data.to_vec(), on_progress))
410        .send()
411        .await
412        .map_err(|_| Error::Message("Failed to parse media.".to_string()))?;
413    if !response.status().is_success() {
414        return Err(Error::Message(format!(
415            "Failed to parse media: {}",
416            response.status().as_u16()
417        )));
418    }
419    Ok(response.text().await?)
420}
421
422/// The `options` argument of `slice`.
423#[derive(Debug, Clone, Default)]
424pub struct SliceOptions {
425    pub segment: Option<[f64; 2]>,
426    pub split: Option<f64>,
427}
428
429pub async fn slice(
430    authorization: &CredentialsOrApiKey,
431    content_type: &str,
432    data: &[u8],
433    options: SliceOptions,
434) -> Result<Option<BytesOrBytesList>> {
435    let mut request = client()
436        .post(format!("{}/media/slice", super::get_url()))
437        .header(
438            "Content-Type",
439            if content_type.is_empty() {
440                "application/octet-stream"
441            } else {
442                content_type
443            },
444        )
445        .header(
446            "Authorization",
447            match authorization {
448                CredentialsOrApiKey::Credentials(credentials) => {
449                    format!("Token {}", credentials.token)
450                }
451                CredentialsOrApiKey::ApiKey(key) => format!("ApiKey {}", key.id),
452            },
453        );
454    if let Some([start, end]) = options.segment {
455        request = request.header("Segment", format!("{start},{end}"));
456    }
457    if let Some(split) = options.split.filter(|split| *split != 0.0) {
458        request = request.header("Split", split.to_string());
459    }
460    let response = request.body(data.to_vec()).send().await?;
461
462    if !response.status().is_success() {
463        return Ok(None);
464    }
465
466    let multipart = response
467        .headers()
468        .get("Encoding-Size")
469        .and_then(|value| value.to_str().ok())
470        .and_then(|value| value.trim().parse::<i64>().ok())
471        .is_some_and(|size| size > 1);
472    let mut parts = parse_stream(response, None::<fn(u64)>).await?;
473    Ok(Some(if multipart {
474        BytesOrBytesList::BytesList(parts)
475    } else {
476        BytesOrBytesList::Bytes(parts.pop().unwrap_or_default())
477    }))
478}
479
480pub async fn duration(
481    authorization: &CredentialsOrApiKey,
482    content_type: &str,
483    data: &[u8],
484    on_progress: Option<Progress>,
485) -> Result<f64> {
486    let response = client()
487        .post(format!("{}/media/duration", super::get_url()))
488        .header(
489            "Authorization",
490            match authorization {
491                CredentialsOrApiKey::Credentials(credentials) => {
492                    format!("Token {}", credentials.token)
493                }
494                CredentialsOrApiKey::ApiKey(key) => format!("ApiKey {}", key.id),
495            },
496        )
497        .header(
498            "Content-Type",
499            if content_type.is_empty() {
500                "application/octet-stream"
501            } else {
502                content_type
503            },
504        )
505        .header("Content-Length", data.len())
506        .body(progress_body(data.to_vec(), on_progress))
507        .send()
508        .await
509        .map_err(|_| Error::Message("Failed to duration media.".to_string()))?;
510    if !response.status().is_success() {
511        return Err(Error::Message(format!(
512            "Failed to duration media: {}",
513            response.status().as_u16()
514        )));
515    }
516    let text = response.text().await?;
517    Ok(text.trim().parse::<f64>().unwrap_or(f64::NAN))
518}