fauna-api 1.0.861

The Fauna SDK for Rust: accounts, Drive, groups, calendar, chats, payments, Parrot and live rooms.
Documentation
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
//! What people ask a Parrot chat with a repository to do on its computer. Each benchmark makes a
//! repository, seeds it through the repository's file routes, asks in a new chat on it the way
//! a person does, waits for the whole reply (its `use_computer` job included), saves and stops
//! the machine, and checks what came back and what the machine left behind.
//!
//! A new account can't open the Fauna app's Drive before it verifies its email and phone, so
//! `sign-in` signs in to a site the repository serves: a sign in page that refuses to be framed,
//! a SameSite=Lax session cookie, and a Drive only that session opens.

use std::sync::LazyLock;
use std::time::{Duration, Instant};

use futures_util::future::BoxFuture;
use regex::Regex;

use super::gallery::mesh;
use super::media::keep;
use super::{Benchmark, Made, Run, Work, ensure, fail, nonce, number, read, short, stopped};
use crate::billing::PageQuery;
use crate::error::Result;
use crate::parrot::chat::{self, CreateChatOptions, OpenChatOptions};
use crate::parrot::generative::{Invocation, Part, Role, TextPart};
use crate::parrot::room::{ChatRoom, RoomMessage};
use crate::parrot::share::{AppendOptions, AppendResult};
use crate::private::benchmarks::BenchmarkGroup;
use crate::termite::os::computer_tool;
use crate::termite::spec::{STANDARD_SPEC, Spec};
use crate::termite::versioning::Repository;

/// What the chats reply with, and what machine agents think with when nobody picks. The server
/// prices a benchmark's steps and turns at this model's price.
pub const MODEL: &str = "gemini-3-flash-preview";

/// The machine `blender-chat` asks for: the small pool's largest, so it renders on one small node.
const BLENDER_SPEC: Spec = Spec {
    vcpus: 7,
    memory_bytes: 16 << 30,
    storage_bytes: 32 << 30,
    ..STANDARD_SPEC
};

const SITE_PORT: u16 = 8125;

/// The site `sign-in` signs in to. It refuses frames everywhere; signing in as the account in
/// `account` sets this server's session cookie and goes home, whose link opens the Drive: the
/// files in `drive`, for that session only, each visit logged in `visits.log`.
const SITE: &str = r#"import html, http.server, pathlib, secrets, urllib.parse

ROOT = pathlib.Path(__file__).parent
USER, PASSWORD = (ROOT / "account").read_text().split()
SESSION = secrets.token_hex(16)
STYLE = "<style>body{font:18px sans-serif;max-width:480px;margin:64px auto}form{display:grid;gap:12px}</style>"


def sign_in_page(note):
    return f"""<!doctype html><title>Sign in</title>{STYLE}<h1>Sign in</h1>{note}
<form method="post" action="/signin">
<label>User name <input name="user" autocomplete="username"></label>
<label>Password <input name="password" type="password" autocomplete="current-password"></label>
<button>Sign in</button></form>"""


class Site(http.server.BaseHTTPRequestHandler):
    def send(self, status, body="", headers=()):
        data = body.encode()
        self.send_response(status)
        self.send_header("Content-Type", "text/html; charset=utf-8")
        self.send_header("X-Frame-Options", "DENY")
        self.send_header("Content-Security-Policy", "frame-ancestors 'none'")
        for name, value in headers:
            self.send_header(name, value)
        self.send_header("Content-Length", str(len(data)))
        self.end_headers()
        self.wfile.write(data)

    def signed_in(self):
        cookies = dict(part.strip().split("=", 1) for part in self.headers.get("Cookie", "").split(";") if "=" in part)
        return cookies.get("session") == SESSION

    def do_GET(self):
        if self.path.startswith("/home") and self.signed_in():
            self.send(200, f'<!doctype html><title>Home</title>{STYLE}<h1>Welcome, {html.escape(USER)}</h1><p><a href="/drive">Open your Drive</a></p>')
        elif self.path.startswith("/drive") and self.signed_in():
            with open(ROOT / "visits.log", "a") as visits:
                visits.write(f"drive {USER}\n")
            files = "".join(f"<li>{html.escape(p.name)}</li>" for p in sorted((ROOT / "drive").iterdir()))
            self.send(200, f"<!doctype html><title>Drive</title>{STYLE}<h1>{html.escape(USER)}'s Drive</h1><ul>{files}</ul>")
        elif self.path.startswith(("/home", "/drive")):
            self.send(303, headers=[("Location", "/")])
        else:
            self.send(200, sign_in_page(""))

    def do_POST(self):
        length = int(self.headers.get("Content-Length", "0"))
        fields = urllib.parse.parse_qs(self.rfile.read(length).decode())
        given = (fields.get("user", [""])[0], fields.get("password", [""])[0])
        if given == (USER, PASSWORD):
            self.send(303, headers=[("Set-Cookie", f"session={SESSION}; SameSite=Lax; Path=/; HttpOnly"), ("Location", "/home")])
        else:
            self.send(200, sign_in_page("<p>Wrong user name or password.</p>"))

    def log_message(self, *args):
        pass


http.server.ThreadingHTTPServer(("127.0.0.1", 8125), Site).serve_forever()
"#;
/// entry.sh runs on every boot, so the site is up before anyone asks for it.
const SERVE_SITE: &str = "#!/usr/bin/env bash\nexec python3 /repository/site/site.py\n";

const CALC: &str = "def add(a, b):\n    return a + b\n\n\ndef mean(values):\n    return sum(values) / len(values)\n";
/// Running it leaves `.tests-ran` with the run's marker, which only running it can.
const TEST_CALC: &str = r#"import pathlib
import unittest

import calc


def setUpModule():
    pathlib.Path(__file__).with_name(".tests-ran").write_text("MARKER\n")


class Calc(unittest.TestCase):
    def test_add(self):
        self.assertEqual(calc.add(2, 3), 5)

    def test_mean(self):
        self.assertEqual(calc.mean([1, 2, 3, 4]), 2.5)


if __name__ == "__main__":
    unittest.main()
"#;

static PASSING: LazyLock<Regex> =
    LazyLock::new(|| Regex::new(r"(?i)\bpass(ed|es|ing)?\b|\bOK\b|succe").expect("pattern"));

fn chat_work(steps: f64, turns: f64, machine_hours: f64) -> Work {
    Work {
        steps,
        turns,
        machine_hours,
        dollars: 0.0,
    }
}

// ─── Sign in ─────────────────────────────────────────────────────────────────

pub(super) struct SignIn;

impl Benchmark for SignIn {
    fn id(&self) -> &'static str {
        "sign-in"
    }
    fn name(&self) -> &'static str {
        "Sign in"
    }
    fn group(&self) -> BenchmarkGroup {
        BenchmarkGroup::Product
    }
    fn about(&self) -> &'static str {
        "A Parrot chat asked to sign in to a site its repository serves, open the Drive behind the sign in, name its files and send a photo, done with use_computer."
    }
    fn work(&self) -> Work {
        chat_work(40.0, 3.0, 0.5)
    }
    fn timeout(&self) -> Duration {
        Duration::from_secs(20 * 60)
    }
    fn run<'a>(&'a self, run: &'a Run) -> BoxFuture<'a, Result<String>> {
        Box::pin(async move {
            run.progress(0.0, "Seeding the repository's site");
            let repository = run.repository("sign in").await?;
            let user = format!("ada{}", nonce(4));
            let password = nonce(12);
            let file = format!("spring-plans-{}.txt", nonce(4));
            repository
                .write("/repository/site/site.py", SITE, None)
                .await?;
            repository
                .write(
                    "/repository/site/account",
                    format!("{user} {password}\n"),
                    None,
                )
                .await?;
            repository
                .write(
                    &format!("/repository/site/drive/{file}"),
                    "Plant the tomatoes in May.\n",
                    None,
                )
                .await?;
            repository
                .write("/repository/entry.sh", SERVE_SITE, None)
                .await?;
            run.progress(0.1, "Asking the chat");
            let reply = ask(
                run,
                &repository,
                &format!(
                    "My computer serves a site at http://localhost:{SITE_PORT}/. Sign in there as {user} with the password {password}, open my Drive on it, and tell me which files are in it. Send me a photo of the Drive too."
                ),
            )
            .await?;
            run.progress(0.85, "Saving the machine and keeping its pictures");
            stopped(&run.key, repository.id).await?;
            let media = keep(run, repository.id, "").await?;
            let visits = read(&repository, "/repository/site/visits.log")
                .await
                .unwrap_or_else(|err| format!("(no visits: {err})"));
            ensure(
                visits.lines().any(|line| line == format!("drive {user}")),
                || {
                    format!(
                        "The site never opened its Drive for {user} signed in. The reply said: {}",
                        short(&reply.text)
                    )
                },
            )?;
            ensure(reply.text.contains(&file), || {
                format!(
                    "The reply didn't name {file}, the file in the Drive: {}",
                    short(&reply.text)
                )
            })?;
            ensure(reply.attached("image/") || reply.attached("video/"), || {
                format!(
                    "The reply attached no photo or recording of the Drive: {}",
                    short(&reply.text)
                )
            })?;
            ensure(!media.is_empty(), || {
                "The machine agent saved no photo or recording.".into()
            })?;
            Ok(format!(
                "Signed in as {user} in a Web window and opened the Drive, and the reply named {file}."
            ))
        })
    }
}

// ─── Tests on video ──────────────────────────────────────────────────────────

pub(super) struct TestsVideo;

impl Benchmark for TestsVideo {
    fn id(&self) -> &'static str {
        "tests-video"
    }
    fn name(&self) -> &'static str {
        "Tests on video"
    }
    fn group(&self) -> BenchmarkGroup {
        BenchmarkGroup::Product
    }
    fn about(&self) -> &'static str {
        "A Parrot chat asked to run its repository's tests in a terminal and show a video of them passing, with captions."
    }
    fn work(&self) -> Work {
        chat_work(40.0, 3.0, 0.5)
    }
    fn timeout(&self) -> Duration {
        Duration::from_secs(20 * 60)
    }
    fn run<'a>(&'a self, run: &'a Run) -> BoxFuture<'a, Result<String>> {
        Box::pin(async move {
            run.progress(0.0, "Seeding the repository's tests");
            let repository = run.repository("tests").await?;
            let marker = nonce(12);
            repository.write("/repository/calc.py", CALC, None).await?;
            repository
                .write(
                    "/repository/test_calc.py",
                    TEST_CALC.replace("MARKER", &marker),
                    None,
                )
                .await?;
            run.progress(0.1, "Asking the chat");
            let reply = ask(
                run,
                &repository,
                "Run this repository's tests in a terminal and show me a video of them passing.",
            )
            .await?;
            run.progress(0.85, "Saving the machine and keeping its recording");
            stopped(&run.key, repository.id).await?;
            let media = keep(run, repository.id, "").await?;
            let ran = read(&repository, "/repository/.tests-ran")
                .await
                .unwrap_or_default();
            ensure(ran.trim() == marker, || {
                format!(
                    "The tests never ran. The reply said: {}",
                    short(&reply.text)
                )
            })?;
            ensure(reply.attached("video/"), || {
                format!("The reply attached no video: {}", short(&reply.text))
            })?;
            ensure(media.iter().any(|kept| kept.captioned_video()), || {
                "No recording came with captions.".into()
            })?;
            ensure(PASSING.is_match(&reply.text), || {
                format!(
                    "The reply didn't say the tests passed: {}",
                    short(&reply.text)
                )
            })?;
            Ok("The tests ran in a terminal, and the reply says they passed with a captioned video of it.".into())
        })
    }
}

// ─── Blender in a chat ───────────────────────────────────────────────────────

pub(super) struct BlenderChat;

impl Benchmark for BlenderChat {
    fn id(&self) -> &'static str {
        "blender-chat"
    }
    fn name(&self) -> &'static str {
        "Blender in a chat"
    }
    fn group(&self) -> BenchmarkGroup {
        BenchmarkGroup::Product
    }
    fn about(&self) -> &'static str {
        "A Parrot chat asked to install Blender, script an OBJ of a tree and show a turntable render of it, on a machine of the size it names."
    }
    fn work(&self) -> Work {
        chat_work(200.0, 4.0, 1.0)
    }
    fn timeout(&self) -> Duration {
        Duration::from_secs(60 * 60)
    }
    fn run<'a>(&'a self, run: &'a Run) -> BoxFuture<'a, Result<String>> {
        Box::pin(async move {
            run.progress(0.0, "Asking the chat");
            let repository = run.repository("tree").await?;
            let from = Instant::now();
            let reply = ask(
                run,
                &repository,
                &format!(
                    "Install Blender with apt (sudo fauna-install blender). Then write a Python script that makes an OBJ model of a tree, with a trunk, branches that split recursively and leaves, and save the model as tree.obj in this repository. Render a short turntable video of the tree in Blender with Cycles on the CPU, at a modest resolution and sample count, and show me the video. Use a machine with {} vCPUs, {} GiB of memory and {} GiB of disk for all of it.",
                    BLENDER_SPEC.vcpus,
                    BLENDER_SPEC.memory_bytes >> 30,
                    BLENDER_SPEC.storage_bytes >> 30
                ),
            )
            .await?;
            run.measure(number("Reply", from.elapsed().as_secs_f64().round(), "s"));
            run.progress(0.85, "Saving the machine and keeping its recording");
            stopped(&run.key, repository.id).await?;
            let media = keep(run, repository.id, "").await?;
            let latest = repository
                .revisions(PageQuery { cursor: 0, max: 1 })
                .await?
                .into_iter()
                .next();
            let Some(latest) = latest else {
                return fail("The machine never saved a revision.");
            };
            let obj = read(&repository, "/repository/tree.obj")
                .await
                .unwrap_or_default();
            let (vertices, faces) = mesh(&obj);
            if !obj.is_empty() {
                run.file("tree.obj", "model/obj", obj.into_bytes()).await?;
            }
            ensure(vertices >= 8 && faces >= 6, || {
                format!(
                    "tree.obj isn't a mesh: it has {vertices} vertices and {faces} faces. The reply said: {}",
                    short(&reply.text)
                )
            })?;
            ensure(
                reply.attached("video/") && media.iter().any(|kept| kept.captioned_video()),
                || {
                    format!(
                        "No captioned video of the render came back: {}",
                        short(&reply.text)
                    )
                },
            )?;
            Ok(format!(
                "tree.obj has {vertices} vertices and {faces} faces, rendered in Blender with Cycles on {} vCPUs.",
                latest.spec.vcpus
            ))
        })
    }
}

// ─── Asking ──────────────────────────────────────────────────────────────────

/// A whole reply: what it said, the files it attached by content type, and the tools it called.
struct Reply {
    text: String,
    files: Vec<String>,
    calls: Vec<Invocation>,
}

impl Reply {
    fn of(messages: &[RoomMessage]) -> Reply {
        let mut reply = Reply {
            text: String::new(),
            files: Vec::new(),
            calls: Vec::new(),
        };
        for part in messages.iter().flat_map(|message| &message.parts) {
            match part {
                Part::Text(part) => {
                    reply.text.push_str(&part.text);
                    reply.text.push('\n');
                }
                Part::File(part) => reply.files.push(part.file.content_type.clone()),
                Part::Invocation(call) => reply.calls.push(call.clone()),
                Part::Html(_) | Part::Reaction(_) => {}
            }
        }
        reply
    }

    /// Whether it attached a file whose content type starts with `kind`.
    fn attached(&self, kind: &str) -> bool {
        self.files.iter().any(|file| file.starts_with(kind))
    }
}

/// Asks `message` in a new chat on `repository`, the way a person does in Parrot, and waits for
/// the whole reply.
async fn ask(run: &Run, repository: &Repository, message: &str) -> Result<Reply> {
    let options = CreateChatOptions {
        repository: Some(repository.clone()),
        ..CreateChatOptions::default()
    };
    let id = chat::create_chat(&run.key, Some(&run.named("chat")), None, options).await?;
    run.made(Made::Chat(id.clone()));
    let opened = chat::open_chat(&run.key, &id, OpenChatOptions::default()).await?;
    let room = opened.room;
    let sent = room
        .send_message(
            Role::User,
            vec![Part::Text(TextPart {
                text: message.to_string(),
            })],
            AppendOptions {
                model: Some(MODEL.to_string()),
                ..AppendOptions::default()
            },
        )
        .await;
    let reply = match sent {
        Ok(AppendResult::Sent { id: Some(id) }) => replied(&room, &id).await,
        Ok(sent) => fail(format!("The message didn't land in the chat: {sent:?}")),
        Err(err) => Err(err),
    };
    room.disconnect();
    let reply = reply?;
    let computer = computer_tool(Some(repository)).name;
    if reply.calls.iter().all(|call| call.name != computer) {
        return fail(format!(
            "The reply never used the computer: {}",
            short(&reply.text)
        ));
    }
    Ok(reply)
}

/// The reply to message `asked`, once its turn has landed and the room stopped thinking.
async fn replied(room: &ChatRoom, asked: &str) -> Result<Reply> {
    loop {
        let messages = room.get_messages();
        if let Some(position) = messages
            .iter()
            .find(|message| message.id == asked)
            .map(|message| message.position)
        {
            let replies: Vec<RoomMessage> = messages
                .into_iter()
                .filter(|message| message.role == Role::Model && message.position > position)
                .collect();
            if !replies.is_empty() && !room.is_thinking() && room.get_queue().is_empty() {
                return Ok(Reply::of(&replies));
            }
        }
        if !room.ws().is_open() {
            return fail("The chat's room closed before the reply finished.");
        }
        tokio::time::sleep(Duration::from_secs(2)).await;
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn hears_the_tests_passing() {
        for said in [
            "All 2 tests passed.",
            "Ran 2 tests in 0.001s\n\nOK",
            "The suite succeeded",
        ] {
            assert!(PASSING.is_match(said), "{said}");
        }
        assert!(!PASSING.is_match("The tests failed with a bypass error"));
    }
}