use std::sync::LazyLock;
use std::time::{Duration, Instant};
use futures_util::future::BoxFuture;
use regex::Regex;
use super::gallery::mesh;
use super::media::keep;
use super::{Benchmark, Made, Run, Work, ensure, fail, nonce, number, read, short, stopped};
use crate::billing::PageQuery;
use crate::error::Result;
use crate::parrot::chat::{self, CreateChatOptions, OpenChatOptions};
use crate::parrot::generative::{Invocation, Part, Role, TextPart};
use crate::parrot::room::{ChatRoom, RoomMessage};
use crate::parrot::share::{AppendOptions, AppendResult};
use crate::private::benchmarks::BenchmarkGroup;
use crate::termite::os::computer_tool;
use crate::termite::spec::{STANDARD_SPEC, Spec};
use crate::termite::versioning::Repository;
pub const MODEL: &str = "gemini-3-flash-preview";
const BLENDER_SPEC: Spec = Spec {
vcpus: 7,
memory_bytes: 16 << 30,
storage_bytes: 32 << 30,
..STANDARD_SPEC
};
const SITE_PORT: u16 = 8125;
const SITE: &str = r#"import html, http.server, pathlib, secrets, urllib.parse
ROOT = pathlib.Path(__file__).parent
USER, PASSWORD = (ROOT / "account").read_text().split()
SESSION = secrets.token_hex(16)
STYLE = "<style>body{font:18px sans-serif;max-width:480px;margin:64px auto}form{display:grid;gap:12px}</style>"
def sign_in_page(note):
return f"""<!doctype html><title>Sign in</title>{STYLE}<h1>Sign in</h1>{note}
<form method="post" action="/signin">
<label>User name <input name="user" autocomplete="username"></label>
<label>Password <input name="password" type="password" autocomplete="current-password"></label>
<button>Sign in</button></form>"""
class Site(http.server.BaseHTTPRequestHandler):
def send(self, status, body="", headers=()):
data = body.encode()
self.send_response(status)
self.send_header("Content-Type", "text/html; charset=utf-8")
self.send_header("X-Frame-Options", "DENY")
self.send_header("Content-Security-Policy", "frame-ancestors 'none'")
for name, value in headers:
self.send_header(name, value)
self.send_header("Content-Length", str(len(data)))
self.end_headers()
self.wfile.write(data)
def signed_in(self):
cookies = dict(part.strip().split("=", 1) for part in self.headers.get("Cookie", "").split(";") if "=" in part)
return cookies.get("session") == SESSION
def do_GET(self):
if self.path.startswith("/home") and self.signed_in():
self.send(200, f'<!doctype html><title>Home</title>{STYLE}<h1>Welcome, {html.escape(USER)}</h1><p><a href="/drive">Open your Drive</a></p>')
elif self.path.startswith("/drive") and self.signed_in():
with open(ROOT / "visits.log", "a") as visits:
visits.write(f"drive {USER}\n")
files = "".join(f"<li>{html.escape(p.name)}</li>" for p in sorted((ROOT / "drive").iterdir()))
self.send(200, f"<!doctype html><title>Drive</title>{STYLE}<h1>{html.escape(USER)}'s Drive</h1><ul>{files}</ul>")
elif self.path.startswith(("/home", "/drive")):
self.send(303, headers=[("Location", "/")])
else:
self.send(200, sign_in_page(""))
def do_POST(self):
length = int(self.headers.get("Content-Length", "0"))
fields = urllib.parse.parse_qs(self.rfile.read(length).decode())
given = (fields.get("user", [""])[0], fields.get("password", [""])[0])
if given == (USER, PASSWORD):
self.send(303, headers=[("Set-Cookie", f"session={SESSION}; SameSite=Lax; Path=/; HttpOnly"), ("Location", "/home")])
else:
self.send(200, sign_in_page("<p>Wrong user name or password.</p>"))
def log_message(self, *args):
pass
http.server.ThreadingHTTPServer(("127.0.0.1", 8125), Site).serve_forever()
"#;
const SERVE_SITE: &str = "#!/usr/bin/env bash\nexec python3 /repository/site/site.py\n";
const CALC: &str = "def add(a, b):\n return a + b\n\n\ndef mean(values):\n return sum(values) / len(values)\n";
const TEST_CALC: &str = r#"import pathlib
import unittest
import calc
def setUpModule():
pathlib.Path(__file__).with_name(".tests-ran").write_text("MARKER\n")
class Calc(unittest.TestCase):
def test_add(self):
self.assertEqual(calc.add(2, 3), 5)
def test_mean(self):
self.assertEqual(calc.mean([1, 2, 3, 4]), 2.5)
if __name__ == "__main__":
unittest.main()
"#;
static PASSING: LazyLock<Regex> =
LazyLock::new(|| Regex::new(r"(?i)\bpass(ed|es|ing)?\b|\bOK\b|succe").expect("pattern"));
fn chat_work(steps: f64, turns: f64, machine_hours: f64) -> Work {
Work {
steps,
turns,
machine_hours,
dollars: 0.0,
}
}
pub(super) struct SignIn;
impl Benchmark for SignIn {
fn id(&self) -> &'static str {
"sign-in"
}
fn name(&self) -> &'static str {
"Sign in"
}
fn group(&self) -> BenchmarkGroup {
BenchmarkGroup::Product
}
fn about(&self) -> &'static str {
"A Parrot chat asked to sign in to a site its repository serves, open the Drive behind the sign in, name its files and send a photo, done with use_computer."
}
fn work(&self) -> Work {
chat_work(40.0, 3.0, 0.5)
}
fn timeout(&self) -> Duration {
Duration::from_secs(20 * 60)
}
fn run<'a>(&'a self, run: &'a Run) -> BoxFuture<'a, Result<String>> {
Box::pin(async move {
run.progress(0.0, "Seeding the repository's site");
let repository = run.repository("sign in").await?;
let user = format!("ada{}", nonce(4));
let password = nonce(12);
let file = format!("spring-plans-{}.txt", nonce(4));
repository
.write("/repository/site/site.py", SITE, None)
.await?;
repository
.write(
"/repository/site/account",
format!("{user} {password}\n"),
None,
)
.await?;
repository
.write(
&format!("/repository/site/drive/{file}"),
"Plant the tomatoes in May.\n",
None,
)
.await?;
repository
.write("/repository/entry.sh", SERVE_SITE, None)
.await?;
run.progress(0.1, "Asking the chat");
let reply = ask(
run,
&repository,
&format!(
"My computer serves a site at http://localhost:{SITE_PORT}/. Sign in there as {user} with the password {password}, open my Drive on it, and tell me which files are in it. Send me a photo of the Drive too."
),
)
.await?;
run.progress(0.85, "Saving the machine and keeping its pictures");
stopped(&run.key, repository.id).await?;
let media = keep(run, repository.id, "").await?;
let visits = read(&repository, "/repository/site/visits.log")
.await
.unwrap_or_else(|err| format!("(no visits: {err})"));
ensure(
visits.lines().any(|line| line == format!("drive {user}")),
|| {
format!(
"The site never opened its Drive for {user} signed in. The reply said: {}",
short(&reply.text)
)
},
)?;
ensure(reply.text.contains(&file), || {
format!(
"The reply didn't name {file}, the file in the Drive: {}",
short(&reply.text)
)
})?;
ensure(reply.attached("image/") || reply.attached("video/"), || {
format!(
"The reply attached no photo or recording of the Drive: {}",
short(&reply.text)
)
})?;
ensure(!media.is_empty(), || {
"The machine agent saved no photo or recording.".into()
})?;
Ok(format!(
"Signed in as {user} in a Web window and opened the Drive, and the reply named {file}."
))
})
}
}
pub(super) struct TestsVideo;
impl Benchmark for TestsVideo {
fn id(&self) -> &'static str {
"tests-video"
}
fn name(&self) -> &'static str {
"Tests on video"
}
fn group(&self) -> BenchmarkGroup {
BenchmarkGroup::Product
}
fn about(&self) -> &'static str {
"A Parrot chat asked to run its repository's tests in a terminal and show a video of them passing, with captions."
}
fn work(&self) -> Work {
chat_work(40.0, 3.0, 0.5)
}
fn timeout(&self) -> Duration {
Duration::from_secs(20 * 60)
}
fn run<'a>(&'a self, run: &'a Run) -> BoxFuture<'a, Result<String>> {
Box::pin(async move {
run.progress(0.0, "Seeding the repository's tests");
let repository = run.repository("tests").await?;
let marker = nonce(12);
repository.write("/repository/calc.py", CALC, None).await?;
repository
.write(
"/repository/test_calc.py",
TEST_CALC.replace("MARKER", &marker),
None,
)
.await?;
run.progress(0.1, "Asking the chat");
let reply = ask(
run,
&repository,
"Run this repository's tests in a terminal and show me a video of them passing.",
)
.await?;
run.progress(0.85, "Saving the machine and keeping its recording");
stopped(&run.key, repository.id).await?;
let media = keep(run, repository.id, "").await?;
let ran = read(&repository, "/repository/.tests-ran")
.await
.unwrap_or_default();
ensure(ran.trim() == marker, || {
format!(
"The tests never ran. The reply said: {}",
short(&reply.text)
)
})?;
ensure(reply.attached("video/"), || {
format!("The reply attached no video: {}", short(&reply.text))
})?;
ensure(media.iter().any(|kept| kept.captioned_video()), || {
"No recording came with captions.".into()
})?;
ensure(PASSING.is_match(&reply.text), || {
format!(
"The reply didn't say the tests passed: {}",
short(&reply.text)
)
})?;
Ok("The tests ran in a terminal, and the reply says they passed with a captioned video of it.".into())
})
}
}
pub(super) struct BlenderChat;
impl Benchmark for BlenderChat {
fn id(&self) -> &'static str {
"blender-chat"
}
fn name(&self) -> &'static str {
"Blender in a chat"
}
fn group(&self) -> BenchmarkGroup {
BenchmarkGroup::Product
}
fn about(&self) -> &'static str {
"A Parrot chat asked to install Blender, script an OBJ of a tree and show a turntable render of it, on a machine of the size it names."
}
fn work(&self) -> Work {
chat_work(200.0, 4.0, 1.0)
}
fn timeout(&self) -> Duration {
Duration::from_secs(60 * 60)
}
fn run<'a>(&'a self, run: &'a Run) -> BoxFuture<'a, Result<String>> {
Box::pin(async move {
run.progress(0.0, "Asking the chat");
let repository = run.repository("tree").await?;
let from = Instant::now();
let reply = ask(
run,
&repository,
&format!(
"Install Blender with apt (sudo fauna-install blender). Then write a Python script that makes an OBJ model of a tree, with a trunk, branches that split recursively and leaves, and save the model as tree.obj in this repository. Render a short turntable video of the tree in Blender with Cycles on the CPU, at a modest resolution and sample count, and show me the video. Use a machine with {} vCPUs, {} GiB of memory and {} GiB of disk for all of it.",
BLENDER_SPEC.vcpus,
BLENDER_SPEC.memory_bytes >> 30,
BLENDER_SPEC.storage_bytes >> 30
),
)
.await?;
run.measure(number("Reply", from.elapsed().as_secs_f64().round(), "s"));
run.progress(0.85, "Saving the machine and keeping its recording");
stopped(&run.key, repository.id).await?;
let media = keep(run, repository.id, "").await?;
let latest = repository
.revisions(PageQuery { cursor: 0, max: 1 })
.await?
.into_iter()
.next();
let Some(latest) = latest else {
return fail("The machine never saved a revision.");
};
let obj = read(&repository, "/repository/tree.obj")
.await
.unwrap_or_default();
let (vertices, faces) = mesh(&obj);
if !obj.is_empty() {
run.file("tree.obj", "model/obj", obj.into_bytes()).await?;
}
ensure(vertices >= 8 && faces >= 6, || {
format!(
"tree.obj isn't a mesh: it has {vertices} vertices and {faces} faces. The reply said: {}",
short(&reply.text)
)
})?;
ensure(
reply.attached("video/") && media.iter().any(|kept| kept.captioned_video()),
|| {
format!(
"No captioned video of the render came back: {}",
short(&reply.text)
)
},
)?;
Ok(format!(
"tree.obj has {vertices} vertices and {faces} faces, rendered in Blender with Cycles on {} vCPUs.",
latest.spec.vcpus
))
})
}
}
struct Reply {
text: String,
files: Vec<String>,
calls: Vec<Invocation>,
}
impl Reply {
fn of(messages: &[RoomMessage]) -> Reply {
let mut reply = Reply {
text: String::new(),
files: Vec::new(),
calls: Vec::new(),
};
for part in messages.iter().flat_map(|message| &message.parts) {
match part {
Part::Text(part) => {
reply.text.push_str(&part.text);
reply.text.push('\n');
}
Part::File(part) => reply.files.push(part.file.content_type.clone()),
Part::Invocation(call) => reply.calls.push(call.clone()),
Part::Html(_) | Part::Reaction(_) => {}
}
}
reply
}
fn attached(&self, kind: &str) -> bool {
self.files.iter().any(|file| file.starts_with(kind))
}
}
async fn ask(run: &Run, repository: &Repository, message: &str) -> Result<Reply> {
let options = CreateChatOptions {
repository: Some(repository.clone()),
..CreateChatOptions::default()
};
let id = chat::create_chat(&run.key, Some(&run.named("chat")), None, options).await?;
run.made(Made::Chat(id.clone()));
let opened = chat::open_chat(&run.key, &id, OpenChatOptions::default()).await?;
let room = opened.room;
let sent = room
.send_message(
Role::User,
vec![Part::Text(TextPart {
text: message.to_string(),
})],
AppendOptions {
model: Some(MODEL.to_string()),
..AppendOptions::default()
},
)
.await;
let reply = match sent {
Ok(AppendResult::Sent { id: Some(id) }) => replied(&room, &id).await,
Ok(sent) => fail(format!("The message didn't land in the chat: {sent:?}")),
Err(err) => Err(err),
};
room.disconnect();
let reply = reply?;
let computer = computer_tool(Some(repository)).name;
if reply.calls.iter().all(|call| call.name != computer) {
return fail(format!(
"The reply never used the computer: {}",
short(&reply.text)
));
}
Ok(reply)
}
async fn replied(room: &ChatRoom, asked: &str) -> Result<Reply> {
loop {
let messages = room.get_messages();
if let Some(position) = messages
.iter()
.find(|message| message.id == asked)
.map(|message| message.position)
{
let replies: Vec<RoomMessage> = messages
.into_iter()
.filter(|message| message.role == Role::Model && message.position > position)
.collect();
if !replies.is_empty() && !room.is_thinking() && room.get_queue().is_empty() {
return Ok(Reply::of(&replies));
}
}
if !room.ws().is_open() {
return fail("The chat's room closed before the reply finished.");
}
tokio::time::sleep(Duration::from_secs(2)).await;
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn hears_the_tests_passing() {
for said in [
"All 2 tests passed.",
"Ran 2 tests in 0.001s\n\nOK",
"The suite succeeded",
] {
assert!(PASSING.is_match(said), "{said}");
}
assert!(!PASSING.is_match("The tests failed with a bypass error"));
}
}