//! A live machine: one running boot of a Repository. Its page is the machine's desktop, one
//! HTML page that is the whole OS, streamed to every seat. Pointer and keyboard input, window
//! tools, file tools, exec, and stop ride the same socket.
use std::collections::HashMap;
use std::ops::Deref;
use std::sync::{Arc, Mutex, MutexGuard, PoisonError};
use std::time::Duration;
use chrono::{DateTime, Utc};
use futures_util::future::BoxFuture;
use serde::{Deserialize, Serialize};
use serde_json::{Map, Value, json};
use tokio::sync::oneshot;
use super::spec::Spec;
use super::versioning::Repository;
use crate::authentification::CredentialsOrApiKey;
use crate::billing::PageQuery;
use crate::error::{Error, Result};
use crate::groups::Group;
use crate::js::{js_number, js_string, present};
use crate::marmot::web::{HtmlPointer, WebRoom, WebRoomHooks};
use crate::parrot::generative::{Function, Tool, Usage, tool};
use crate::shared::schema::Schema;
use crate::shared::server::{Authorization, Hooks, Presence, Room, connect as connect_server};
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
#[serde(rename_all = "lowercase")]
pub enum MouseButton {
Left,
Right,
Middle,
}
/// `{ x, y }` in the TypeScript SDK.
#[derive(Debug, Clone, Copy, Default, PartialEq, Serialize, Deserialize)]
pub struct Point {
pub x: f64,
pub y: f64,
}
/// `'web' | 'files' | 'file' | 'terminal'` in the TypeScript SDK.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
#[serde(rename_all = "lowercase")]
pub enum WindowApp {
Web,
Files,
File,
Terminal,
}
/// `'files' | 'lines' | 'count'` in the TypeScript SDK.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
#[serde(rename_all = "lowercase")]
pub enum GrepOutput {
Files,
Lines,
Count,
}
/// `'cpu' | 'memory'` in the TypeScript SDK.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
#[serde(rename_all = "lowercase")]
pub enum ProcessSort {
Cpu,
Memory,
}
/// `'TERM' | 'KILL'` in the TypeScript SDK.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
#[serde(rename_all = "UPPERCASE")]
pub enum ProcessSignal {
Term,
Kill,
}
/// `'private' | 'public'` in the TypeScript SDK.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
#[serde(rename_all = "lowercase")]
pub enum InternetMode {
Private,
Public,
}
/// A key going down or up, by the name a browser's `KeyboardEvent.key` gives it ('a', 'A',
/// 'Shift', 'Enter').
#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)]
pub struct KeyStroke {
pub key: String,
pub down: bool,
}
/// Everything a seat or an agent does on a machine. Positions are desktop pixels. Paths are
/// relative to /repository, and `/` is the repository root.
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
#[serde(
tag = "kind",
rename_all = "camelCase",
rename_all_fields = "camelCase"
)]
pub enum MachineTool {
Move {
x: f64,
y: f64,
},
/// `count` clicks in a row (2 is a double click). A click on the same spot right after
/// another continues its run, the way a person's quick second click does.
Click {
#[serde(default, skip_serializing_if = "Option::is_none")]
x: Option<f64>,
#[serde(default, skip_serializing_if = "Option::is_none")]
y: Option<f64>,
#[serde(default, skip_serializing_if = "Option::is_none")]
down_ms: Option<f64>,
#[serde(default, skip_serializing_if = "Option::is_none")]
button: Option<MouseButton>,
#[serde(default, skip_serializing_if = "Option::is_none")]
count: Option<f64>,
},
Drag {
to: Point,
#[serde(default, skip_serializing_if = "Option::is_none")]
from: Option<Point>,
#[serde(default, skip_serializing_if = "Option::is_none")]
down_ms: Option<f64>,
},
Scroll {
#[serde(default, skip_serializing_if = "Option::is_none")]
delta_x: Option<f64>,
#[serde(default, skip_serializing_if = "Option::is_none")]
delta_y: Option<f64>,
},
Type {
text: String,
},
/// A person's own input, sent as it happens: a button goes down and up where the pointer
/// is, `pointer` moves it while a button is down (a drag), the wheel turns at a point, and
/// keys go down and up in order. Each lands at once; agents use the tools above.
Press {
x: f64,
y: f64,
#[serde(default, skip_serializing_if = "Option::is_none")]
button: Option<MouseButton>,
},
Pointer {
x: f64,
y: f64,
},
Release {
x: f64,
y: f64,
#[serde(default, skip_serializing_if = "Option::is_none")]
button: Option<MouseButton>,
},
Wheel {
x: f64,
y: f64,
delta_x: f64,
delta_y: f64,
},
Keys {
strokes: Vec<KeyStroke>,
},
Windows,
Focus {
window: f64,
},
Place {
window: f64,
x: f64,
y: f64,
},
Resize {
window: f64,
width: f64,
height: f64,
},
Close {
window: f64,
},
/// Load a Web window's page again, after its files change. Previews follow their file on their own.
Reload {
window: f64,
},
/// A web window at `url`; a files window at `path` (a folder of /repository); the file at
/// `path` in an editor (text and code) or a preview (images, PDFs, video, audio), which
/// follows the file as it changes; or a terminal, running `command` in view and staying open
/// at a prompt when it ends.
Open {
app: WindowApp,
#[serde(default, skip_serializing_if = "Option::is_none")]
url: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
path: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
command: Option<String>,
},
/// A chord like ['ctrl', 's']. Focuses `window` first when given.
Key {
keys: Vec<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
window: Option<f64>,
},
/// The page in Web window `window`, or the focused one: `{ url, html }` as JSON, the HTML
/// trimmed to the body without styles, scripts and SVG.
PageHtml {
#[serde(default, skip_serializing_if = "Option::is_none")]
window: Option<f64>,
},
/// Runs `script` in the page of Web window `window`, or the focused one: its result as text,
/// a string as it is and anything else as JSON.
PageScript {
script: String,
#[serde(default, skip_serializing_if = "Option::is_none")]
window: Option<f64>,
},
List {
#[serde(default, skip_serializing_if = "Option::is_none")]
path: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pattern: Option<String>,
},
Grep {
pattern: String,
#[serde(default, skip_serializing_if = "Option::is_none")]
path: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
glob: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
ignore_case: Option<bool>,
#[serde(default, skip_serializing_if = "Option::is_none")]
context: Option<f64>,
#[serde(default, skip_serializing_if = "Option::is_none")]
output: Option<GrepOutput>,
#[serde(default, skip_serializing_if = "Option::is_none")]
max: Option<f64>,
},
Read {
path: String,
#[serde(default, skip_serializing_if = "Option::is_none")]
line: Option<f64>,
#[serde(default, skip_serializing_if = "Option::is_none")]
before: Option<f64>,
#[serde(default, skip_serializing_if = "Option::is_none")]
after: Option<f64>,
},
Write {
path: String,
text: String,
},
Replace {
path: String,
old: String,
new: String,
#[serde(default, skip_serializing_if = "Option::is_none")]
all: Option<bool>,
},
Insert {
path: String,
after: f64,
text: String,
},
Remove {
path: String,
start: f64,
end: f64,
},
Processes {
#[serde(default, skip_serializing_if = "Option::is_none")]
sort: Option<ProcessSort>,
#[serde(default, skip_serializing_if = "Option::is_none")]
limit: Option<f64>,
#[serde(default, skip_serializing_if = "Option::is_none")]
filter: Option<String>,
},
/// Stops only the agent user's own processes, never the machine's or the desktop's.
Kill {
pid: f64,
#[serde(default, skip_serializing_if = "Option::is_none")]
signal: Option<ProcessSignal>,
},
/// How the machine reaches the internet: `private` only from a residential address, billed
/// per GB to whoever pays for the machine, or `public` from its own.
Internet {
mode: InternetMode,
},
/// Pause, then look again. Costs machine time but no model calls; at most
/// `WAIT_MAX_SECONDS` a call.
Wait {
seconds: f64,
},
}
impl MachineTool {
/// Pointer and keyboard input, which the desktop page itself takes.
fn is_input(&self) -> bool {
matches!(
self,
MachineTool::Move { .. }
| MachineTool::Click { .. }
| MachineTool::Drag { .. }
| MachineTool::Scroll { .. }
| MachineTool::Type { .. }
| MachineTool::Press { .. }
| MachineTool::Pointer { .. }
| MachineTool::Release { .. }
| MachineTool::Wheel { .. }
| MachineTool::Keys { .. }
)
}
}
/// The longest one `wait` pauses.
pub const WAIT_MAX_SECONDS: u64 = 600;
/// `'app' | 'web' | 'files' | 'preview' | 'terminal' | 'editor'` in the TypeScript SDK.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
#[serde(rename_all = "lowercase")]
pub enum WindowKind {
App,
Web,
Files,
Preview,
Terminal,
Editor,
}
/// Metadata for one desktop window. Its pixels live in the desktop HTML.
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct Window {
/// Same id as the element's data-window and the window tools.
pub id: u64,
pub kind: WindowKind,
/// WM_CLASS for X11 apps ('xmessage'), else the kind.
pub app: String,
pub title: String,
pub x: f64,
pub y: f64,
pub width: f64,
pub height: f64,
pub focused: bool,
}
fn truthy(value: &Value) -> bool {
match value {
Value::Null => false,
Value::Bool(value) => *value,
Value::Number(number) => number
.as_f64()
.is_some_and(|number| number != 0.0 && !number.is_nan()),
Value::String(text) => !text.is_empty(),
_ => true,
}
}
impl Window {
pub fn from_json(json: &Value) -> Window {
let kind = match json["kind"].as_str() {
Some("web") => WindowKind::Web,
Some("files") => WindowKind::Files,
Some("preview") => WindowKind::Preview,
Some("terminal") => WindowKind::Terminal,
Some("editor") => WindowKind::Editor,
_ => WindowKind::App,
};
let kind_name = match kind {
WindowKind::App => "app",
WindowKind::Web => "web",
WindowKind::Files => "files",
WindowKind::Preview => "preview",
WindowKind::Terminal => "terminal",
WindowKind::Editor => "editor",
};
Window {
id: js_number(json.get("id")) as u64,
kind,
app: js_string(json.get("app"), kind_name),
title: js_string(json.get("title"), ""),
x: json["x"].as_f64().unwrap_or_default(),
y: json["y"].as_f64().unwrap_or_default(),
width: json["width"].as_f64().unwrap_or_default(),
height: json["height"].as_f64().unwrap_or_default(),
focused: truthy(&json["focused"]),
}
}
}
/// `'list' | 'cozy'` in the TypeScript SDK.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
#[serde(rename_all = "lowercase")]
pub enum FilesView {
List,
Cozy,
}
/// The folder and view a Files window is on. The controller sets it; every seat follows.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct FilesState {
pub path: String,
pub view: FilesView,
}
impl FilesState {
pub fn from_json(json: &Value) -> FilesState {
FilesState {
path: js_string(json.get("path"), ""),
view: if json["view"] == "cozy" {
FilesView::Cozy
} else {
FilesView::List
},
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
#[serde(rename_all = "UPPERCASE")]
pub enum MachineStatus {
Booting,
Running,
Stopping,
Stopped,
Failed,
}
#[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize)]
pub struct Viewport {
pub width: f64,
pub height: f64,
}
/// `{ min, max }` in the TypeScript SDK.
#[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize)]
pub struct ViewportBounds {
pub min: Viewport,
pub max: Viewport,
}
/// The smallest and largest desktop a machine draws. The largest is the guest's X screen.
pub const VIEWPORT: ViewportBounds = ViewportBounds {
min: Viewport {
width: 640.0,
height: 400.0,
},
max: Viewport {
width: 2560.0,
height: 1600.0,
},
};
/// A desktop size inside `VIEWPORT`, in whole pixels.
pub fn clamp_viewport(size: Viewport) -> Viewport {
Viewport {
width: size
.width
.clamp(VIEWPORT.min.width, VIEWPORT.max.width)
.round(),
height: size
.height
.clamp(VIEWPORT.min.height, VIEWPORT.max.height)
.round(),
}
}
/// The first byte of an app frame.
const APP_FRAME: u8 = 1;
/// Set in an app frame's flags when it covers its whole window.
const APP_FRAME_KEY: u8 = 1;
/// An app frame's header: kind, flags, window, seq, the window's width and height, then the
/// region's x, y, width and height, all big endian.
const APP_FRAME_HEADER: usize = 22;
/// A picture of part of an X11 app window, which a machine room sends as a binary frame: a JPEG
/// to draw at (`x`, `y`) in window `window` (its element's `data-xid` in the desktop), which is
/// `window_width` by `window_height` now. Each window's frames count up in `seq`; a skipped
/// count means a stale frame was dropped, and the window needs a repaint unless the frame is a
/// `key` frame, which covers the whole window.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct AppFrame {
pub window: u32,
pub seq: u32,
pub key: bool,
pub window_width: u32,
pub window_height: u32,
pub x: u32,
pub y: u32,
pub width: u32,
pub height: u32,
pub jpeg: Vec<u8>,
}
impl AppFrame {
/// A binary frame as an app frame, or `None` when it isn't one.
pub fn from_bytes(data: &[u8]) -> Option<AppFrame> {
if data.len() < APP_FRAME_HEADER || data[0] != APP_FRAME {
return None;
}
let u32_at =
|at: usize| u32::from_be_bytes([data[at], data[at + 1], data[at + 2], data[at + 3]]);
let u16_at = |at: usize| u32::from(u16::from_be_bytes([data[at], data[at + 1]]));
Some(AppFrame {
window: u32_at(2),
seq: u32_at(6),
key: data[1] & APP_FRAME_KEY != 0,
window_width: u16_at(10),
window_height: u16_at(12),
x: u16_at(14),
y: u16_at(16),
width: u16_at(18),
height: u16_at(20),
jpeg: data[APP_FRAME_HEADER..].to_vec(),
})
}
}
/// An agent driving the machine. While one does, nobody else can act.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct RoomAgent {
pub id: String,
pub name: String,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub icon: Option<String>,
}
impl RoomAgent {
pub fn from_json(json: &Value) -> RoomAgent {
RoomAgent {
id: match &json["id"] {
Value::String(id) => id.clone(),
id => id.to_string(),
},
name: match &json["name"] {
Value::Null => "Agent".to_string(),
Value::String(name) => name.clone(),
name => name.to_string(),
},
icon: json["icon"]
.as_str()
.filter(|icon| !icon.is_empty())
.map(str::to_string),
}
}
}
/// Machine room hooks. Every method defaults to a no-op, like the optional members of the
/// TypeScript `MachineRoomHooks` object; `web` is its nested web hooks.
pub trait MachineRoomHooks: Send + Sync {
/// The desktop's pointer moved or changed shape.
fn on_cursor(&self, _room: &MachineRoom, _cursor: &HtmlPointer) {}
/// An agent, or a seat through a tool, used the pointer or the keyboard. A person's own
/// input as it happens isn't announced; their presence shows it.
fn on_tool(&self, _room: &MachineRoom, _tool: &MachineTool) {}
fn on_closed(&self, _room: &MachineRoom) {}
fn on_presence(&self, _room: &MachineRoom) {}
/// Control changed hands, or an agent took or released the machine.
fn on_control(&self, _room: &MachineRoom) {}
fn web(&self) -> Option<Arc<dyn WebRoomHooks>> {
None
}
/// A window opened, moved, resized, retitled, or changed focus.
fn on_window(&self, _room: &MachineRoom, _window: &Window) {}
fn on_window_closed(&self, _room: &MachineRoom, _id: u64) {}
/// A Files window's folder or view changed.
fn on_files(&self, _room: &MachineRoom, _window: u64, _files: &FilesState) {}
fn on_status(&self, _room: &MachineRoom) {}
/// Private turned on or off, paused while it reconnects (`private_reason` says why), or came back.
fn on_private(&self, _room: &MachineRoom) {}
/// An agent holding the machine took a step.
fn on_agent_step(&self, _room: &MachineRoom, _step: &AgentStep) {}
/// A picture of part of an X11 app window arrived.
fn on_app_frame(&self, _room: &MachineRoom, _frame: &AppFrame) {}
}
impl MachineRoomHooks for () {}
/// One tool call of an agent working the machine: the tool, with what, and what came back. A turn
/// with several calls arrives as several steps. Args and result are capped at about 2 KB; larger
/// args arrive as `{ truncated }`.
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct AgentStep {
/// The agent's id, as in the room's control state.
pub agent: String,
/// The turn, counting from 1.
pub step: u64,
/// The call's place in its turn, from 0.
pub call: u64,
pub tool: String,
#[serde(default)]
pub args: Map<String, Value>,
#[serde(default)]
pub result: String,
/// The turn's model time and usage ride on its first call, so totals add up.
#[serde(default)]
pub model_ms: f64,
#[serde(default)]
pub tool_ms: f64,
#[serde(default)]
pub usage: Usage,
}
/// A caption of a machine agent's recording, on the video's timeline, after its title card.
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct Caption {
pub start_ms: f64,
pub end_ms: f64,
pub text: String,
}
impl Caption {
pub fn from_json(json: &Value) -> Caption {
let ms = |key: &str| present(json.get(key)).map_or(0.0, |ms| js_number(Some(ms)));
Caption {
start_ms: ms("startMs"),
end_ms: ms("endMs"),
text: js_string(json.get("text"), ""),
}
}
}
/// `'video' | 'photo'` in the TypeScript SDK.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
#[serde(rename_all = "lowercase")]
pub enum MediaKind {
Video,
Photo,
}
/// A recording or photo a machine agent saved with the repository, at `path` under its owner's
/// root. A video says how long it runs and what its captions say when, which players list as
/// chapters. A chat's copy carries it on its FILE part, as `media`.
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct Media {
pub kind: MediaKind,
pub title: String,
pub path: String,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub duration_ms: Option<f64>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub captions: Option<Vec<Caption>>,
}
impl Media {
pub fn from_json(json: &Value) -> Media {
Media {
kind: if json["kind"] == "video" {
MediaKind::Video
} else {
MediaKind::Photo
},
title: js_string(json.get("title"), ""),
path: js_string(json.get("path"), ""),
duration_ms: present(json.get("durationMs")).map(|ms| js_number(Some(ms))),
captions: json["captions"]
.as_array()
.map(|captions| captions.iter().map(Caption::from_json).collect()),
}
}
}
const INPUT_MS: u64 = 30_000;
const CONTROL_MS: u64 = 30_000;
const CALL_MS: u64 = 60_000;
const EXEC_MS: u64 = 120_000;
const STOP_MS: u64 = 10 * 60_000;
/// Runs one machine tool and returns its result as text.
pub type MachineToolInvoker =
Arc<dyn Fn(MachineTool) -> BoxFuture<'static, Result<String>> + Send + Sync>;
/// Runs a shell command, with an optional timeout in milliseconds, and returns its result as text.
pub type ExecInvoker =
Arc<dyn Fn(String, Option<f64>) -> BoxFuture<'static, Result<String>> + Send + Sync>;
fn optional_text(p: &Value, key: &str) -> Option<String> {
present(p.get(key)).map(|value| js_string(Some(value), ""))
}
fn optional_number(p: &Value, key: &str) -> Option<f64> {
present(p.get(key)).map(|value| js_number(Some(value)))
}
/// Every machine tool for a model: pointer and keyboard input, windows, pages, files, processes,
/// the internet switch, wait, and exec. `invoke` runs a tool; `exec` runs a shell command and
/// returns its result as text.
pub fn parrot_tools(invoke: MachineToolInvoker, exec: ExecInvoker) -> Vec<Tool> {
let mut tools = Vec::new();
let mut add = |name: &str,
description: &str,
properties: Value,
required: &[&str],
build: fn(&Value) -> MachineTool| {
let mut parameters = json!({ "type": "OBJECT", "properties": properties });
if !required.is_empty() {
parameters["required"] = json!(required);
}
let invoke = invoke.clone();
tools.push(tool(name, description, parameters, move |params: Value| {
let params = if params.is_null() { json!({}) } else { params };
invoke(build(¶ms))
}));
};
add(
"move",
"Move the pointer to desktop coordinates. Does not click.",
json!({
"x": { "type": "NUMBER", "description": "X pixel" },
"y": { "type": "NUMBER", "description": "Y pixel" },
}),
&["x", "y"],
|p| MachineTool::Move {
x: js_number(p.get("x")),
y: js_number(p.get("y")),
},
);
add(
"click",
"Click. Optional coordinates move there first. downMs is how long the button stays down. button right opens context menus; count 2 double clicks.",
json!({
"x": { "type": "NUMBER", "description": "X pixel" },
"y": { "type": "NUMBER", "description": "Y pixel" },
"downMs": { "type": "NUMBER", "description": "Hold duration in ms" },
"button": { "type": "STRING", "enum": ["left", "right", "middle"], "description": "Mouse button, default left" },
"count": { "type": "NUMBER", "description": "Clicks in a row: 2 double clicks, 3 triple clicks. Default 1" },
}),
&[],
|p| MachineTool::Click {
x: optional_number(p, "x"),
y: optional_number(p, "y"),
down_ms: optional_number(p, "downMs"),
button: match p.get("button").and_then(Value::as_str) {
Some("right") => Some(MouseButton::Right),
Some("middle") => Some(MouseButton::Middle),
_ => None,
},
count: optional_number(p, "count"),
},
);
add(
"drag",
"Drag from an optional start point to a destination.",
json!({
"to": { "type": "OBJECT", "description": "Destination {x,y}" },
"from": { "type": "OBJECT", "description": "Start {x,y}" },
"downMs": { "type": "NUMBER", "description": "Hold at the end in ms" },
}),
&["to"],
|p| MachineTool::Drag {
to: Point {
x: js_number(p.get("to").and_then(|to| to.get("x"))),
y: js_number(p.get("to").and_then(|to| to.get("y"))),
},
from: p.get("from").filter(|from| truthy(from)).map(|from| Point {
x: js_number(from.get("x")),
y: js_number(from.get("y")),
}),
down_ms: optional_number(p, "downMs"),
},
);
add(
"scroll",
"Scroll with the wheel at the current pointer.",
json!({
"deltaX": { "type": "NUMBER", "description": "Horizontal wheel delta" },
"deltaY": { "type": "NUMBER", "description": "Vertical wheel delta" },
}),
&[],
|p| MachineTool::Scroll {
delta_x: optional_number(p, "deltaX"),
delta_y: optional_number(p, "deltaY"),
},
);
add(
"type",
"Type text at the current focus.",
json!({
"text": { "type": "STRING", "description": "Text to type" },
}),
&["text"],
|p| MachineTool::Type {
text: js_string(p.get("text"), ""),
},
);
add(
"windows",
"List every open desktop window with its id, kind, app, title, position, size, and focus.",
json!({}),
&[],
|_| MachineTool::Windows,
);
add(
"focus",
"Bring a window to the front and give it keyboard focus.",
json!({
"window": { "type": "NUMBER", "description": "Window id" },
}),
&["window"],
|p| MachineTool::Focus {
window: js_number(p.get("window")),
},
);
add(
"place",
"Move a window so its top left corner is at desktop coordinates.",
json!({
"window": { "type": "NUMBER", "description": "Window id" },
"x": { "type": "NUMBER", "description": "Desktop X pixel" },
"y": { "type": "NUMBER", "description": "Desktop Y pixel" },
}),
&["window", "x", "y"],
|p| MachineTool::Place {
window: js_number(p.get("window")),
x: js_number(p.get("x")),
y: js_number(p.get("y")),
},
);
add(
"resize",
"Resize a window. Apps redraw at the new size.",
json!({
"window": { "type": "NUMBER", "description": "Window id" },
"width": { "type": "NUMBER", "description": "Width in pixels" },
"height": { "type": "NUMBER", "description": "Height in pixels" },
}),
&["window", "width", "height"],
|p| MachineTool::Resize {
window: js_number(p.get("window")),
width: js_number(p.get("width")),
height: js_number(p.get("height")),
},
);
add(
"close",
"Close a window.",
json!({
"window": { "type": "NUMBER", "description": "Window id" },
}),
&["window"],
|p| MachineTool::Close {
window: js_number(p.get("window")),
},
);
add(
"reload",
"Reload a Web window after its page's files change, so it shows them. Previews follow their file on their own, and reload too.",
json!({
"window": { "type": "NUMBER", "description": "Window id" },
}),
&["window"],
|p| MachineTool::Reload {
window: js_number(p.get("window")),
},
);
add(
"open",
"Open a window on the desktop: web opens a Web window at url (http or https; serve local pages over HTTP first, since file:// pages don't load); files opens a Files window at a folder; file opens a file, text and code in an editor (save with ctrl s) and images, PDFs, video and audio in a preview; terminal opens a terminal, running command in view; when it ends the window stays open at a prompt, so people can read the result and you can type more. Editors and previews follow their file as it changes.",
json!({
"app": { "type": "STRING", "enum": ["web", "files", "file", "terminal"], "description": "web, files, file or terminal" },
"url": { "type": "STRING", "description": "http or https address for a web window" },
"path": { "type": "STRING", "description": "Folder for files, or the file to open, relative to /repository" },
"command": { "type": "STRING", "description": "Command for a terminal to run in view" },
}),
&["app"],
|p| MachineTool::Open {
app: match p.get("app").and_then(Value::as_str) {
Some("files") => WindowApp::Files,
Some("file") => WindowApp::File,
Some("terminal") => WindowApp::Terminal,
_ => WindowApp::Web,
},
url: optional_text(p, "url"),
path: optional_text(p, "path"),
command: optional_text(p, "command"),
},
);
add(
"key",
"Press a key chord, like ctrl and s together. Focuses the window first when given. ctrl r or f5 on a Web window reloads it.",
json!({
"keys": { "type": "ARRAY", "items": { "type": "STRING" }, "description": "Keys held together, e.g. [\"ctrl\", \"s\"]" },
"window": { "type": "NUMBER", "description": "Window id to focus first" },
}),
&["keys"],
|p| MachineTool::Key {
keys: p
.get("keys")
.and_then(Value::as_array)
.map(|keys| keys.iter().map(|key| js_string(Some(key), "")).collect())
.unwrap_or_default(),
window: optional_number(p, "window"),
},
);
add(
"pageHtml",
"Read the page in a Web window: its address and its HTML, trimmed to the body without styles, scripts and SVG. Reads the focused Web window unless window is given.",
json!({
"window": { "type": "NUMBER", "description": "Web window id" },
}),
&[],
|p| MachineTool::PageHtml {
window: optional_number(p, "window"),
},
);
add(
"pageScript",
"Run JavaScript in the page of a Web window and get its result: a string as it is, anything else as JSON. Runs in the focused Web window unless window is given.",
json!({
"script": { "type": "STRING", "description": "JavaScript to run in the page" },
"window": { "type": "NUMBER", "description": "Web window id" },
}),
&["script"],
|p| MachineTool::PageScript {
script: js_string(p.get("script"), ""),
window: optional_number(p, "window"),
},
);
add(
"list",
"List files under a folder of /repository. Folders end in a slash. With pattern (like **/*.ts) it matches recursively.",
json!({
"path": { "type": "STRING", "description": "Folder relative to /repository. Defaults to the root." },
"pattern": { "type": "STRING", "description": "Glob to match recursively" },
}),
&[],
|p| MachineTool::List {
path: optional_text(p, "path"),
pattern: optional_text(p, "pattern"),
},
);
add(
"grep",
"Search file contents in /repository with ripgrep. Output is matching files by default, or lines with context, or counts.",
json!({
"pattern": { "type": "STRING", "description": "Regular expression" },
"path": { "type": "STRING", "description": "Folder or file to search" },
"glob": { "type": "STRING", "description": "Only files matching this glob" },
"ignoreCase": { "type": "BOOLEAN", "description": "Case insensitive" },
"context": { "type": "NUMBER", "description": "Lines of context around each match" },
"output": { "type": "STRING", "enum": ["files", "lines", "count"], "description": "files, lines, or count" },
"max": { "type": "NUMBER", "description": "Result cap, default 50" },
}),
&["pattern"],
|p| MachineTool::Grep {
pattern: js_string(p.get("pattern"), ""),
path: optional_text(p, "path"),
glob: optional_text(p, "glob"),
ignore_case: present(p.get("ignoreCase")).map(truthy),
context: optional_number(p, "context"),
output: present(p.get("output"))
.and_then(|output| serde_json::from_value(output.clone()).ok()),
max: optional_number(p, "max"),
},
);
add(
"read",
"Read numbered lines of a file: from line minus before to line plus after. Defaults to the first 100 lines.",
json!({
"path": { "type": "STRING", "description": "File relative to /repository" },
"line": { "type": "NUMBER", "description": "Line to center on, default 1" },
"before": { "type": "NUMBER", "description": "Lines before, default 0" },
"after": { "type": "NUMBER", "description": "Lines after, default 100" },
}),
&["path"],
|p| MachineTool::Read {
path: js_string(p.get("path"), ""),
line: optional_number(p, "line"),
before: optional_number(p, "before"),
after: optional_number(p, "after"),
},
);
add(
"write",
"Create or overwrite a file, creating missing folders.",
json!({
"path": { "type": "STRING", "description": "File relative to /repository" },
"text": { "type": "STRING", "description": "Full file contents" },
}),
&["path", "text"],
|p| MachineTool::Write {
path: js_string(p.get("path"), ""),
text: js_string(p.get("text"), ""),
},
);
add(
"replace",
"Replace an exact string in a file. old must match exactly once unless all is true. Prefer this for edits.",
json!({
"path": { "type": "STRING", "description": "File relative to /repository" },
"old": { "type": "STRING", "description": "Exact text to find" },
"new": { "type": "STRING", "description": "Replacement text" },
"all": { "type": "BOOLEAN", "description": "Replace every match" },
}),
&["path", "old", "new"],
|p| MachineTool::Replace {
path: js_string(p.get("path"), ""),
old: js_string(p.get("old"), ""),
new: js_string(p.get("new"), ""),
all: present(p.get("all")).map(truthy),
},
);
add(
"insert",
"Insert text after a line number. 0 inserts at the top. Read the file first.",
json!({
"path": { "type": "STRING", "description": "File relative to /repository" },
"after": { "type": "NUMBER", "description": "Line to insert after" },
"text": { "type": "STRING", "description": "Text to insert" },
}),
&["path", "after", "text"],
|p| MachineTool::Insert {
path: js_string(p.get("path"), ""),
after: js_number(p.get("after")),
text: js_string(p.get("text"), ""),
},
);
add(
"remove",
"Delete lines start through end, inclusive. Read the file first.",
json!({
"path": { "type": "STRING", "description": "File relative to /repository" },
"start": { "type": "NUMBER", "description": "First line" },
"end": { "type": "NUMBER", "description": "Last line" },
}),
&["path", "start", "end"],
|p| MachineTool::Remove {
path: js_string(p.get("path"), ""),
start: js_number(p.get("start")),
end: js_number(p.get("end")),
},
);
add(
"processes",
"List running processes with their CPU right now, memory, running time, command and listening ports, plus the machine's CPU load, memory and free disk. Default: the 25 busiest by CPU.",
json!({
"sort": { "type": "STRING", "enum": ["cpu", "memory"], "description": "cpu or memory, default cpu" },
"limit": { "type": "NUMBER", "description": "How many to list, default 25" },
"filter": { "type": "STRING", "description": "Only commands containing this text" },
}),
&[],
|p| MachineTool::Processes {
sort: match p.get("sort").and_then(Value::as_str) {
Some("memory") => Some(ProcessSort::Memory),
Some("cpu") => Some(ProcessSort::Cpu),
_ => None,
},
limit: optional_number(p, "limit"),
filter: optional_text(p, "filter"),
},
);
add(
"kill",
"Stop one of your processes. TERM asks it to exit, KILL forces it. The machine's own processes, including the desktop's browser, can't be stopped.",
json!({
"pid": { "type": "NUMBER", "description": "Process id from processes" },
"signal": { "type": "STRING", "enum": ["TERM", "KILL"], "description": "TERM (default) or KILL" },
}),
&["pid"],
|p| MachineTool::Kill {
pid: js_number(p.get("pid")),
signal: match p.get("signal").and_then(Value::as_str) {
Some("KILL") => Some(ProcessSignal::Kill),
Some("TERM") => Some(ProcessSignal::Term),
_ => None,
},
},
);
add(
"internet",
"Choose how the machine reaches the internet. private sends every connection out from a residential address, billed per GB; public uses the machine's own address. While private reconnects the internet is paused and connections are refused; switch to public if the work can't wait.",
json!({
"mode": { "type": "STRING", "enum": ["private", "public"], "description": "private or public" },
}),
&["mode"],
|p| MachineTool::Internet {
mode: if p.get("mode").and_then(Value::as_str) == Some("public") {
InternetMode::Public
} else {
InternetMode::Private
},
},
);
tools.push(tool(
"wait",
&format!("Wait, then look at the screen again: for a page to load, an app to start, or a build running in a terminal to finish. Nothing watches the screen meanwhile, so wait about as long as it should take and wait again if it isn't done. At most {WAIT_MAX_SECONDS} seconds a call. To wait for a command, run it with exec instead, which returns when it ends."),
json!({
"type": "OBJECT",
"properties": {
"seconds": { "type": "NUMBER", "description": format!("Seconds to wait, at most {WAIT_MAX_SECONDS}") },
},
"required": ["seconds"],
}),
{
let invoke = invoke.clone();
move |params: Value| {
invoke(MachineTool::Wait {
seconds: js_number(params.get("seconds")),
})
}
},
));
tools.push(tool(
"exec",
"Run a Bash command as the agent user in /repository. Each call is a new shell: cd, exported variables and activated environments don't carry over, and there's no input or terminal. Output comes back when the command exits, and nothing shows on screen; to let people watch a command run, open a terminal with it instead. Start servers and GUI apps in the background with & (or nohup ... &) so they keep running; a timeout stops everything the command started. GUI apps open as desktop windows.",
json!({
"type": "OBJECT",
"properties": {
"command": { "type": "STRING", "description": "Bash command" },
"timeoutMs": { "type": "NUMBER", "description": "Timeout in ms, default 120000; pass more for installs and builds" },
},
"required": ["command"],
}),
move |params: Value| exec(js_string(params.get("command"), ""), optional_number(¶ms, "timeoutMs")),
));
tools
}
/// `use_computer`'s result when the person chose not to make a repository for it.
pub const COMPUTER_DECLINED: &str =
"They chose not to use a computer for this. Do it without one if you can.";
const COMPUTER_JOB: &str = "It's for work that needs a screen: using apps and websites, taking screenshots, recording a video. A subagent does the job on the computer and reports back. It sees nothing of this chat except goal, so goal must carry everything: what to do, where, which files, and what to capture or record (screenshots, or a video of the job). Everything it captures is attached to the chat for people to see, and comes back to you too. Slower and costlier than the other tools; use it only when the screen matters.";
/// The `use_computer` tool for a Parrot chat: a subagent does a job on the repository's machine,
/// started at `spec` when given, else at the repository's latest revision's spec (or
/// `STANDARD_SPEC` for a new one). Without a repository (`None`), the person is asked to make one
/// first; that's the shape `UI.Chat` offers as a client tool.
pub fn computer_tool(repository: Option<&Repository>) -> Function {
let place = if repository.is_some() {
"Do something on the repository's computer, a Linux machine with a desktop."
} else {
"Do something on a computer, a Linux machine with a desktop. This chat has no computer yet, so the person is asked first to make one; if they don't, do the request without a computer."
};
Function {
name: "use_computer".to_string(),
description: Some(format!("{place} {COMPUTER_JOB}")),
parameters: serde_json::from_value::<Schema>(json!({
"type": "OBJECT",
"properties": {
"goal": { "type": "STRING", "description": "Everything the subagent needs: what to do, where, which files, and what to capture or record" },
"spec": {
"type": "OBJECT",
"description": "The machine to run on, only when the job needs more than the last one, like a GPU or more memory",
"properties": {
"vcpus": { "type": "NUMBER", "description": "vCPUs" },
"memoryBytes": { "type": "NUMBER", "description": "Memory in bytes" },
"storageBytes": { "type": "NUMBER", "description": "Disk in bytes" },
"tflops": { "type": "NUMBER", "description": "GPU TFLOPS (dense BF16), 0 for no GPU" },
"gpuMemoryBytes": { "type": "NUMBER", "description": "GPU memory in bytes, 0 for no GPU" },
},
"required": ["vcpus", "memoryBytes", "storageBytes", "tflops", "gpuMemoryBytes"],
},
},
"required": ["goal"],
}))
.expect("tool schemas are valid"),
default: None,
example: None,
behavior: None,
}
}
/// One pool of machines and the nodes they share, in dollars.
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct PoolPricing {
pub name: String,
/// The Google machine type of its nodes, like n2-standard-8.
pub node_type: String,
/// What one node costs an hour, which a machine alone in the pool pays whole.
pub node_per_hour: f64,
/// What one node sells to machines.
pub node_vcpus: u32,
pub node_memory_bytes: u64,
/// The machines running in it now, and the nodes they fill.
pub machines: u32,
pub nodes: u32,
}
/// What a machine of a spec pays an hour, in dollars, in the pool it fits.
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct Quote {
pub pool: String,
/// As the only machine in its pool, paying for every node it fills.
pub alone: f64,
/// If it started now, sharing its pool's nodes with the machines running there.
pub now: f64,
}
/// What Termite charges, in dollars. Machines share what their pool's nodes cost by how much
/// of a node each takes, so a machine's price moves as others start and stop.
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct Pricing {
pub pools: Vec<PoolPricing>,
/// A repository's stored files, per GB a month.
pub storage_per_gb_month: f64,
/// What a private machine sends through the residential proxy, per GB.
pub private_per_gb: f64,
/// What a machine sends to the internet, per GB.
pub internet_per_gb: f64,
/// What the spec asked about pays an hour, or none when none was asked.
pub quote: Option<Quote>,
}
/// Termite's current prices, and with `spec`, what a machine of it pays an hour. Anyone can
/// read them, signed in or not.
pub async fn pricing(spec: Option<&Spec>) -> Result<Pricing> {
let mut request = crate::http::client().get(format!("{}/termite/pricing", super::get_url()));
if let Some(spec) = spec {
request = request.query(&[
("vcpus", spec.vcpus.to_string()),
("memoryBytes", spec.memory_bytes.to_string()),
("storageBytes", spec.storage_bytes.to_string()),
("tflops", spec.tflops.to_string()),
("gpuMemoryBytes", spec.gpu_memory_bytes.to_string()),
]);
}
let response = request.send().await?;
Ok(crate::http::ok(response).await?.json().await?)
}
/// `'called' | 'crashed' | 'skipped'` in the TypeScript SDK.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
#[serde(rename_all = "lowercase")]
pub enum WakeReason {
Called,
Crashed,
Skipped,
}
/// One time a repository woke its machine agent: its code ran wake.sh (`called`),
/// entry.sh crashed (`crashed`), or the hourly limit left it out (`skipped`).
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
pub struct Wake {
pub id: String,
pub reason: WakeReason,
/// What the agent was told: wake.sh's prompt, or how entry.sh ended.
pub prompt: String,
pub started: DateTime<Utc>,
/// When the woken run ended, or None while it works.
pub finished: Option<DateTime<Utc>>,
pub ok: bool,
/// What the agent said when it finished, or why it stopped.
pub answer: String,
}
impl Wake {
pub fn from_json(json: &Value) -> Wake {
Wake {
id: js_string(json.get("id"), "undefined"),
reason: match json["reason"].as_str() {
Some("crashed") => WakeReason::Crashed,
Some("skipped") => WakeReason::Skipped,
_ => WakeReason::Called,
},
prompt: js_string(json.get("prompt"), ""),
started: crate::tickets::parse_date(&json["started"]).unwrap_or_default(),
finished: if truthy(&json["finished"]) {
crate::tickets::parse_date(&json["finished"])
} else {
None
},
ok: truthy(&json["ok"]),
answer: js_string(json.get("answer"), ""),
}
}
}
/// The `opts` argument of the machine calls that take only a group.
#[derive(Debug, Clone, Default)]
pub struct GroupOptions {
pub group: Option<Group>,
}
fn authorized(
request: reqwest::RequestBuilder,
authorization: &CredentialsOrApiKey,
group: Option<&Group>,
) -> reqwest::RequestBuilder {
let request = request.header(
"Authorization",
match authorization {
CredentialsOrApiKey::Credentials(credentials) => format!("Token {}", credentials.token),
CredentialsOrApiKey::ApiKey(key) => format!("ApiKey {}", key.id),
},
);
match group {
Some(group) => request.header("Group", &group.id),
None => request,
}
}
/// The times a repository woke its agent, newest first.
pub async fn wakes(
authorization: &CredentialsOrApiKey,
repository: u64,
query: PageQuery,
opts: GroupOptions,
) -> Result<Vec<Wake>> {
let response = authorized(
crate::http::client().get(format!(
"{}/termite/repositories/{repository}/wakes?cursor={}&max={}",
super::get_url(),
query.cursor,
query.max
)),
authorization,
opts.group.as_ref(),
)
.send()
.await?;
let json: Value = crate::http::ok(response).await?.json().await?;
Ok(json
.as_array()
.into_iter()
.flatten()
.map(Wake::from_json)
.collect())
}
struct Machine {
cursor: HtmlPointer,
controller: Option<String>,
agent: Option<RoomAgent>,
windows: HashMap<u64, Window>,
files: HashMap<u64, FilesState>,
status: MachineStatus,
error: String,
name: String,
private: bool,
private_reason: String,
}
struct MachineState {
machine: u64,
repository: u64,
spec: Spec,
web: WebRoom,
state: Mutex<Machine>,
hooks: Mutex<Arc<dyn MachineRoomHooks>>,
calls: Mutex<HashMap<String, oneshot::Sender<Result<Value>>>>,
}
fn lock<T>(mutex: &Mutex<T>) -> MutexGuard<'_, T> {
mutex.lock().unwrap_or_else(PoisonError::into_inner)
}
/// `{ code, stdout, stderr }` in the TypeScript SDK: how a command ended.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct ExecResult {
pub code: i64,
pub stdout: String,
pub stderr: String,
}
/// The `opts` argument of `exec`.
#[derive(Debug, Clone, Copy, Default)]
pub struct ExecOptions {
pub timeout_ms: Option<u64>,
}
/// The `opts` argument of `MachineRoom::start`.
pub struct StartOptions {
pub spec: Spec,
pub group: Option<Group>,
pub hooks: Option<Arc<dyn MachineRoomHooks>>,
pub private: Option<bool>,
}
/// The `opts` argument of `MachineRoom::connect` and `MachineRoom::join`
/// (which takes no group).
#[derive(Default)]
pub struct MachineOptions {
pub group: Option<Group>,
pub hooks: Option<Arc<dyn MachineRoomHooks>>,
}
/// `{ id } | { reference }` in the TypeScript SDK: a machine by id, or by its
/// `termite:machine:{id}` reference.
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum MachineTarget {
Id { id: u64 },
Reference { reference: String },
}
/// `{ on, reason }` in the TypeScript SDK.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct PrivateState {
pub on: bool,
pub reason: String,
}
#[derive(Clone)]
pub struct MachineRoom {
room: Room,
inner: Arc<MachineState>,
}
impl Deref for MachineRoom {
type Target = Room;
fn deref(&self) -> &Room {
&self.room
}
}
impl MachineRoom {
fn create(room: &Room, hooks: Arc<dyn MachineRoomHooks>, state: &Value) -> MachineRoom {
MachineRoom {
room: room.clone(),
inner: Arc::new(MachineState {
machine: js_number(Some(&state["machine"]["id"])) as u64,
repository: js_number(Some(&state["machine"]["repository"])) as u64,
spec: Spec::from_json(&state["machine"]["spec"]),
web: WebRoom::from_parent(room, hooks.web(), state),
state: Mutex::new(Machine {
cursor: HtmlPointer::default(),
controller: None,
agent: None,
windows: HashMap::new(),
files: HashMap::new(),
status: MachineStatus::Booting,
error: String::new(),
name: String::new(),
private: false,
private_reason: String::new(),
}),
hooks: Mutex::new(hooks),
calls: Mutex::new(HashMap::new()),
}),
}
}
fn machine_state(&self) -> MutexGuard<'_, Machine> {
lock(&self.inner.state)
}
fn hooks(&self) -> Arc<dyn MachineRoomHooks> {
lock(&self.inner.hooks).clone()
}
/// The machine id, the number `connect` takes.
pub fn machine(&self) -> u64 {
self.inner.machine
}
pub fn repository(&self) -> u64 {
self.inner.repository
}
/// What the machine was granted: its vCPUs, memory, disk and GPU.
pub fn spec(&self) -> Spec {
self.inner.spec
}
/// The desktop page's HTML stream, on the same socket.
pub fn web(&self) -> &WebRoom {
&self.inner.web
}
pub fn cursor(&self) -> HtmlPointer {
self.machine_state().cursor.clone()
}
/// The seat (presence id) whose actions run.
pub fn controller(&self) -> Option<String> {
self.machine_state().controller.clone()
}
/// Set while an agent holds the machine.
pub fn agent(&self) -> Option<RoomAgent> {
self.machine_state().agent.clone()
}
pub fn windows(&self) -> HashMap<u64, Window> {
self.machine_state().windows.clone()
}
/// Each Files window's folder and view, by window id.
pub fn files(&self) -> HashMap<u64, FilesState> {
self.machine_state().files.clone()
}
pub fn status(&self) -> MachineStatus {
self.machine_state().status
}
/// Why the machine failed, when it did.
pub fn error(&self) -> String {
self.machine_state().error.clone()
}
/// The repository's name, so a guest who joined by code knows what it is.
pub fn name(&self) -> String {
self.machine_state().name.clone()
}
/// Its connections go out only from a residential address, billed per GB to whoever pays for
/// the machine. Stays true while private is paused.
pub fn private(&self) -> bool {
self.machine_state().private
}
/// Why private is paused while it reconnects, like a payer out of balance. Its internet is cut
/// until then. Empty while private carries traffic, or is off.
pub fn private_reason(&self) -> String {
self.machine_state().private_reason.clone()
}
/// Apply a connection snapshot `{ machine, control, cursor, windows, files, private }`. The
/// desktop's copy follows the connection as a bootstrap of this seat's own.
pub fn hydrate(&self, state: &Value) {
if state["control"].is_object() {
self.apply_control(&state["control"]);
}
let cursor = serde_json::from_value::<HtmlPointer>(state["cursor"].clone()).ok();
{
let mut machine = self.machine_state();
if let Ok(status) = serde_json::from_value(state["machine"]["status"].clone()) {
machine.status = status;
}
if state["machine"].is_object() {
machine.error = js_string(state["machine"].get("error"), "");
machine.name = js_string(state["machine"].get("name"), "");
}
if let Some(cursor) = &cursor {
machine.cursor = cursor.clone();
}
if let Some(windows) = state["windows"].as_array() {
machine.windows = windows
.iter()
.map(Window::from_json)
.map(|window| (window.id, window))
.collect();
}
if let Some(files) = state["files"].as_array() {
machine.files = files
.iter()
.map(|row| {
(
js_number(row.get("window")) as u64,
FilesState::from_json(row),
)
})
.collect();
}
if truthy(&state["private"]) {
machine.private = truthy(&state["private"]["on"]);
machine.private_reason = js_string(state["private"].get("reason"), "");
}
}
if let Some(cursor) = cursor {
self.hooks().on_cursor(self, &cursor);
}
}
pub fn watch(&self, hooks: Arc<dyn MachineRoomHooks>) {
if let Some(web) = hooks.web() {
self.inner.web.set_hooks(web);
}
*lock(&self.inner.hooks) = hooks;
}
/// Whether your actions run: you control it and no agent holds it.
pub fn controlling(&self, me: &Presence) -> bool {
let machine = self.machine_state();
machine.agent.is_none() && machine.controller.as_deref() == Some(me.id())
}
/// Hand control to a seat, or take it yourself without one. While an agent holds the machine this stops the agent.
pub async fn set_controller(&self, seat: Option<&str>) -> Result<()> {
let data = match seat.filter(|seat| !seat.is_empty()) {
Some(seat) => json!({ "seat": seat }),
None => json!({}),
};
self.request("control-call", data, CONTROL_MS, "Control timed out.")
.await?;
Ok(())
}
/// Presence changed: tell whoever is watching now.
pub fn emit_presence(&self) {
self.hooks().on_presence(self);
}
pub fn emit_closed(&self) {
self.hooks().on_closed(self);
}
fn apply_control(&self, control: &Value) {
let mut machine = self.machine_state();
machine.controller = control["controller"].as_str().map(str::to_string);
machine.agent = if control["agent"].is_object() {
Some(RoomAgent::from_json(&control["agent"]))
} else {
None
};
}
/// Run a tool as this seat. Only the controller's tools run. `wait` pauses here, at most
/// `WAIT_MAX_SECONDS`.
pub async fn invoke_tool(&self, tool: &MachineTool) -> Result<String> {
if let MachineTool::Wait { seconds } = tool {
let seconds = if seconds.is_nan() {
0.0
} else {
seconds.clamp(0.0, WAIT_MAX_SECONDS as f64)
};
tokio::time::sleep(Duration::from_secs_f64(seconds)).await;
return Ok(format!("Waited {seconds} seconds."));
}
let result = if tool.is_input() {
let kind = serde_json::to_value(tool)?["kind"].clone();
self.request(
"machine-tool-call",
json!({ "tool": tool }),
INPUT_MS,
&format!("Tool timed out: {}", js_string(Some(&kind), "")),
)
.await?
} else {
self.call(json!({ "kind": "tool", "tool": tool }), CALL_MS)
.await?
};
Ok(match result {
Value::Null => String::new(),
Value::String(result) => result,
result => result.to_string(),
})
}
/// Run a shell command as `agent` in /repository with DISPLAY=:0, so GUI apps open as desktop windows.
pub async fn exec(&self, command: &str, opts: ExecOptions) -> Result<ExecResult> {
let timeout_ms = opts.timeout_ms.unwrap_or(EXEC_MS);
let result = self
.call(
json!({ "kind": "exec", "command": command, "timeoutMs": timeout_ms }),
timeout_ms + 15_000,
)
.await?;
Ok(ExecResult {
code: match result.get("code") {
None | Some(Value::Null) => -1,
code => js_number(code) as i64,
},
stdout: js_string(result.get("stdout"), ""),
stderr: js_string(result.get("stderr"), ""),
})
}
/// Size the machine's desktop to this seat's screen, clamped to `VIEWPORT`. Only the controller can.
pub async fn set_viewport(&self, width: f64, height: f64) -> Result<()> {
let size = clamp_viewport(Viewport { width, height });
self.call(
json!({ "kind": "viewport", "width": size.width, "height": size.height }),
CALL_MS,
)
.await?;
Ok(())
}
/// Open a file in an app on the machine, for everyone to see: text and code in an editor
/// window, images, PDFs, video and audio in a preview, anything else with an installed
/// opener. Only the controller can.
pub async fn open(&self, path: &str) -> Result<()> {
self.call(json!({ "kind": "open", "path": path }), CALL_MS)
.await?;
Ok(())
}
/// Put a Files window on a folder and view for every seat. Only the controller can.
pub async fn set_files(&self, window: u64, files: &FilesState) -> Result<()> {
self.call(
json!({ "kind": "files", "window": window, "path": files.path, "view": files.view }),
CALL_MS,
)
.await?;
Ok(())
}
/// Save a new revision and shut down. The room closes afterwards.
pub async fn stop(&self) -> Result<()> {
match self.call(json!({ "kind": "stop" }), STOP_MS).await {
Ok(_) => Ok(()),
// The machine announces it closed before it answers the stop itself.
Err(_) if self.status() == MachineStatus::Stopped => Ok(()),
Err(error) => Err(error),
}
}
async fn call(&self, call: Value, timeout_ms: u64) -> Result<Value> {
self.request(
"machine-call",
json!({ "call": call }),
timeout_ms,
"Machine call timed out.",
)
.await
}
/// Posts `event` with a new id; its answer (`control-result`, `machine-tool-result` or
/// `machine-result`) settles it.
async fn request(
&self,
event: &str,
mut data: Value,
timeout_ms: u64,
timed_out: &str,
) -> Result<Value> {
let id = uuid::Uuid::new_v4().to_string();
let (resolve, result) = oneshot::channel();
lock(&self.inner.calls).insert(id.clone(), resolve);
data["id"] = json!(id);
self.post(&json!({ "event": event, "data": data }));
match tokio::time::timeout(Duration::from_millis(timeout_ms), result).await {
Ok(Ok(result)) => result,
Ok(Err(_)) => Err(Error::Message("Machine closed.".to_string())),
Err(_) => {
lock(&self.inner.calls).remove(&id);
Err(Error::Message(timed_out.to_string()))
}
}
}
fn settle(&self, data: &Value) {
let Some(id) = data["id"].as_str() else {
return;
};
let Some(pending) = lock(&self.inner.calls).remove(id) else {
return;
};
let _ = pending.send(if truthy(&data["error"]) {
Err(Error::Message(js_string(data.get("error"), "")))
} else {
Ok(data["result"].clone())
});
}
pub fn parrot_tools(&self) -> Vec<Tool> {
let invoking = self.clone();
let executing = self.clone();
parrot_tools(
Arc::new(move |tool| {
let room = invoking.clone();
Box::pin(async move { room.invoke_tool(&tool).await })
}),
Arc::new(move |command, timeout_ms| {
let room = executing.clone();
Box::pin(async move {
let result = room
.exec(
&command,
ExecOptions {
timeout_ms: timeout_ms.map(|ms| ms as u64),
},
)
.await?;
Ok(serde_json::to_string(&result)?)
})
}),
)
}
/// A binary frame: an app frame reaches `on_app_frame`.
pub fn handle_binary(&self, data: &[u8]) -> bool {
let Some(frame) = AppFrame::from_bytes(data) else {
return false;
};
self.hooks().on_app_frame(self, &frame);
true
}
/// Ask for a whole picture of app window `window`, after a frame of it was skipped. Anyone
/// in the room can.
pub fn repaint(&self, window: u32) {
self.post(&json!({ "event": "app-repaint", "data": { "window": window } }));
}
pub fn handle_event(&self, packet: &Value) -> bool {
if self.inner.web.handle_event(packet) {
return true;
}
let data = &packet["data"];
let hooks = self.hooks();
match packet["event"].as_str().unwrap_or_default() {
"room-control" => {
let empty = json!({});
self.apply_control(if data.is_object() { data } else { &empty });
hooks.on_control(self);
true
}
"machine-cursor" => {
if let Ok(cursor) = serde_json::from_value::<HtmlPointer>(data["cursor"].clone()) {
self.machine_state().cursor = cursor.clone();
hooks.on_cursor(self, &cursor);
}
true
}
"machine-tool-call" => {
if let Ok(tool) = serde_json::from_value::<MachineTool>(data["tool"].clone()) {
hooks.on_tool(self, &tool);
}
true
}
"control-result" | "machine-tool-result" | "machine-result" => {
self.settle(data);
true
}
"machine-status" => {
if let Ok(status) = serde_json::from_value(data["status"].clone()) {
self.machine_state().status = status;
}
hooks.on_status(self);
true
}
"machine-window" => {
if !truthy(&data["window"]) {
return true;
}
let window = Window::from_json(&data["window"]);
self.machine_state()
.windows
.insert(window.id, window.clone());
hooks.on_window(self, &window);
true
}
"machine-window-closed" => {
let id = js_number(data.get("id")) as u64;
{
let mut machine = self.machine_state();
machine.windows.remove(&id);
machine.files.remove(&id);
}
hooks.on_window_closed(self, id);
true
}
"machine-files" => {
let window = js_number(data.get("window")) as u64;
let files = FilesState::from_json(data);
self.machine_state().files.insert(window, files.clone());
hooks.on_files(self, window, &files);
true
}
"machine-private" => {
{
let mut machine = self.machine_state();
machine.private = truthy(&data["on"]);
machine.private_reason = js_string(data.get("reason"), "");
}
hooks.on_private(self);
true
}
"machine-agent-step" => {
if let Ok(step) = serde_json::from_value::<AgentStep>(data.clone()) {
hooks.on_agent_step(self, &step);
}
true
}
"machine-closed" => {
{
let mut machine = self.machine_state();
if let Ok(status) = serde_json::from_value(data["status"].clone()) {
machine.status = status;
}
if let Some(error) = data.get("error") {
machine.error = js_string(Some(error), "");
}
}
hooks.on_status(self);
for (_, pending) in lock(&self.inner.calls).drain() {
let _ = pending.send(Err(Error::Message("Machine closed.".to_string())));
}
hooks.on_closed(self);
true
}
_ => false,
}
}
/// Boot the repository's machine, or return the one already live, and connect to it.
/// `spec` is what it asks for; `room.spec()` is what it was granted, and a live machine
/// keeps its own. `private` sends its connections out from a residential address, billed
/// per GB to whoever pays for the machine; on a live machine it switches it.
pub async fn start(
authorization: &CredentialsOrApiKey,
repository: u64,
opts: StartOptions,
) -> Result<(MachineRoom, Presence)> {
let mut body = json!({ "spec": opts.spec });
if let Some(private) = opts.private {
body["private"] = json!(private);
}
let response = authorized(
crate::http::client().post(format!(
"{}/termite/repositories/{repository}/machine",
super::get_url()
)),
authorization,
opts.group.as_ref(),
)
.json(&body)
.send()
.await?;
let json: Value = crate::http::ok(response).await?.json().await?;
MachineRoom::connect(
authorization,
&MachineTarget::Id {
id: js_number(json.get("id")) as u64,
},
MachineOptions {
group: opts.group,
hooks: opts.hooks,
},
)
.await
}
/// Save and shut down a repository's machine without joining its room. For its owner or group members.
pub async fn shutdown(
authorization: &CredentialsOrApiKey,
repository: u64,
opts: GroupOptions,
) -> Result<()> {
let response = authorized(
crate::http::client().post(format!(
"{}/termite/repositories/{repository}/machine/stop",
super::get_url()
)),
authorization,
opts.group.as_ref(),
)
.json(&json!({}))
.send()
.await?;
crate::http::ok(response).await?;
Ok(())
}
/// Turn private on or off for a repository's live machine, for its owner or group members.
/// Resolves to the machine's private state and, while it's paused, why.
pub async fn set_private(
authorization: &CredentialsOrApiKey,
repository: u64,
on: bool,
opts: GroupOptions,
) -> Result<PrivateState> {
let response = authorized(
crate::http::client().post(format!(
"{}/termite/repositories/{repository}/machine/private",
super::get_url()
)),
authorization,
opts.group.as_ref(),
)
.json(&json!({ "on": on }))
.send()
.await?;
let json: Value = crate::http::ok(response).await?.json().await?;
Ok(PrivateState {
on: truthy(&json["on"]),
reason: js_string(json.get("reason"), ""),
})
}
/// Connect to a live machine by id, or by its `termite:machine:{id}` reference.
pub async fn connect(
authorization: &CredentialsOrApiKey,
target: &MachineTarget,
opts: MachineOptions,
) -> Result<(MachineRoom, Presence)> {
MachineRoom::open_room(
Authorization::Server {
authorization: authorization.clone(),
server: match target {
MachineTarget::Id { id } => format!("termite:machine:{id}"),
MachineTarget::Reference { reference } => reference.clone(),
},
group: opts.group.as_ref().map(|group| group.id.clone()),
},
opts.hooks.unwrap_or_else(|| Arc::new(())),
)
.await
}
/// Join a machine someone shared with an access code.
pub async fn join(
authorization: &CredentialsOrApiKey,
access_code: &str,
opts: MachineOptions,
) -> Result<(MachineRoom, Presence)> {
MachineRoom::open_room(
Authorization::AccessCode {
authorization: authorization.clone(),
access_code: access_code.to_string(),
},
opts.hooks.unwrap_or_else(|| Arc::new(())),
)
.await
}
async fn open_room(
auth: Authorization,
hooks: Arc<dyn MachineRoomHooks>,
) -> Result<(MachineRoom, Presence)> {
let machine_hooks = Arc::new(MachineHooks {
room: Mutex::new(None),
hooks,
});
let me = connect_server(
auth,
&format!("{}/termite/machine/live", super::get_url()),
machine_hooks.clone(),
)
.await?;
let room = machine_hooks
.room()
.ok_or_else(|| Error::Message("Room not found".to_string()))?;
room.emit_presence();
Ok((room, me))
}
}
struct MachineHooks {
room: Mutex<Option<MachineRoom>>,
hooks: Arc<dyn MachineRoomHooks>,
}
impl MachineHooks {
fn room(&self) -> Option<MachineRoom> {
lock(&self.room).clone()
}
}
impl Hooks for MachineHooks {
fn create_room(&self, room: &Room, state: &Value) {
*lock(&self.room) = Some(MachineRoom::create(room, self.hooks.clone(), state));
}
fn on_presence(&self, _room: &Room) {
if let Some(room) = self.room() {
room.emit_presence();
}
}
fn on_connection(&self, _room: &Room, state: &Value) {
if let Some(room) = self.room() {
room.hydrate(state);
}
}
fn on_event(&self, _room: &Room, packet: &Value) -> bool {
self.room().is_some_and(|room| room.handle_event(packet))
}
fn on_binary(&self, _room: &Room, data: &[u8]) {
if let Some(room) = self.room() {
room.handle_binary(data);
}
}
fn on_closed(&self, _room: &Room, _reason: Option<&str>) {
if let Some(room) = self.room() {
room.emit_closed();
}
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn reads_an_app_frames_header_and_its_jpeg() {
let mut data = vec![1u8, 1];
data.extend(0x0040_0007u32.to_be_bytes());
data.extend(9u32.to_be_bytes());
for n in [640u16, 400, 0, 0, 640, 400] {
data.extend(n.to_be_bytes());
}
data.extend([0xff, 0xd8, 0xff]);
let frame = AppFrame::from_bytes(&data).unwrap();
assert_eq!(
frame,
AppFrame {
window: 0x0040_0007,
seq: 9,
key: true,
window_width: 640,
window_height: 400,
x: 0,
y: 0,
width: 640,
height: 400,
jpeg: vec![0xff, 0xd8, 0xff],
}
);
data[1] = 0;
assert!(!AppFrame::from_bytes(&data).unwrap().key);
assert_eq!(AppFrame::from_bytes(&data[..21]), None);
data[0] = 2;
assert_eq!(AppFrame::from_bytes(&data), None);
}
#[test]
fn reads_media_with_its_captions() {
let video = Media::from_json(&json!({
"kind": "video", "title": "Signup", "path": "/termite/repositories/1/media/2.mp4",
"durationMs": 7_200, "captions": [{ "startMs": 1_200, "endMs": 3_000, "text": "Typing" }],
}));
assert_eq!(
video,
Media {
kind: MediaKind::Video,
title: "Signup".into(),
path: "/termite/repositories/1/media/2.mp4".into(),
duration_ms: Some(7_200.0),
captions: Some(vec![Caption {
start_ms: 1_200.0,
end_ms: 3_000.0,
text: "Typing".into(),
}]),
}
);
let photo =
Media::from_json(&json!({ "kind": "photo", "title": "Saved", "path": "/p.jpg" }));
assert_eq!(photo.kind, MediaKind::Photo);
assert_eq!((photo.duration_ms, photo.captions), (None, None));
}
}