use std::collections::HashMap;
use std::future::Future;
use std::sync::{Arc, Mutex};
use std::time::{Duration, Instant};
use base64::Engine;
use base64::engine::general_purpose::STANDARD;
use tokio::sync::{oneshot, watch};
use super::{fail, lock, since, tenths};
use crate::authentification::CredentialsOrApiKey;
use crate::error::{Error, Result};
use crate::marmot::web::{BootstrapMeta, WebMutation, WebNode, WebRoom, WebRoomHooks};
use crate::termite::os::{
AppFrame, ExecOptions, ExecResult, KeyStroke, MachineOptions, MachineRoom, MachineRoomHooks,
MachineStatus, MachineTool, StartOptions, Window, WindowApp, WindowKind,
};
use crate::termite::spec::Spec;
const SAMPLES: usize = 20;
const DRAG_STEP: f64 = 8.0;
const GAP: Duration = Duration::from_millis(300);
const SAMPLE_WAIT: Duration = Duration::from_secs(10);
const BOOT_WAIT: Duration = Duration::from_secs(600);
const SETTLE: Duration = Duration::from_secs(1);
const KEYS_PORT: u16 = 8124;
const APP_SIZE: (u32, u32) = (480, 360);
const DOCK: &str = r#"class="dock""#;
const TITLE_BAR: f64 = 34.0;
const WEB_BAR: f64 = 39.0;
const KEYS_PAGE: &str = r#"<!doctype html><title>Keys</title><p id="out">keys ready</p><script>
let n = 0
addEventListener('keydown', (e) => { n += 1; document.getElementById('out').textContent = `key ${n} ${e.key} seen` })
</script>"#;
const X11_APP: &str = r#"import sys, time
from Xlib import X, display
mode = sys.argv[1]
d = display.Display()
s = d.screen()
w = s.root.create_window(0, 0, 480, 360, 0, s.root_depth, X.InputOutput, X.CopyFromParent,
background_pixel=s.white_pixel, event_mask=X.ExposureMask | X.KeyPressMask)
w.set_wm_name("x11 app " + mode)
w.set_wm_class("x11app", "X11App")
w.map()
cmap = s.default_colormap
gcs = [w.create_gc(foreground=cmap.alloc_color(*rgb).pixel)
for rgb in ((65535, 0, 0), (0, 40000, 0), (0, 0, 65535), (65535, 50000, 0))]
n = 0
def paint(shift):
for band in range(-1, 9):
w.fill_rectangle(gcs[(band + n) % 4], band * 60 + shift % 60, 0, 60, 360)
d.flush()
paint(0)
shift = 0
while True:
if mode == "spin":
while d.pending_events():
d.next_event()
shift += 4
paint(shift)
time.sleep(1 / 60)
continue
e = d.next_event()
if e.type == X.KeyPress:
n += 1
paint(shift)
"#;
#[derive(Debug, Clone, Copy, PartialEq)]
pub(super) struct Gate {
pub(super) p50: f64,
pub(super) p95: f64,
}
pub(super) const DOM_GATE: Gate = Gate {
p50: 50.0,
p95: 100.0,
};
pub(super) const APP_GATE: Gate = Gate {
p50: 80.0,
p95: 150.0,
};
impl Gate {
pub(super) fn holds(&self, spread: &Spread) -> bool {
spread.p50 <= self.p50 && spread.p95 <= self.p95
}
pub(super) fn words(&self) -> String {
format!(
"{} ms at the median and {} ms at p95, past the round trip",
self.p50, self.p95
)
}
}
#[derive(Debug, Clone, Copy, PartialEq)]
pub(super) struct Booted {
pub(super) running: f64,
pub(super) desktop: f64,
}
#[derive(Debug, Clone, Copy, PartialEq)]
pub(super) struct Spread {
pub(super) p50: f64,
pub(super) p95: f64,
pub(super) n: usize,
}
impl Spread {
pub(super) fn of(mut samples: Vec<f64>) -> Spread {
samples.sort_by(f64::total_cmp);
let n = samples.len();
let rank = |p: f64| samples[((p * n as f64).ceil() as usize).clamp(1, n) - 1];
Spread {
p50: rank(0.5),
p95: rank(0.95),
n,
}
}
pub(super) fn beyond(&self, rtt: f64) -> Spread {
Spread {
p50: tenths((self.p50 - rtt).max(0.0)),
p95: tenths((self.p95 - rtt).max(0.0)),
n: self.n,
}
}
}
pub(super) fn secs(n: u64) -> Duration {
Duration::from_secs(n)
}
pub(super) async fn tool(room: &MachineRoom, tool: MachineTool) -> Result<String> {
room.invoke_tool(&tool).await
}
pub(super) async fn exec(room: &MachineRoom, command: &str) -> Result<ExecResult> {
room.exec(command, ExecOptions::default()).await
}
pub(super) async fn read(room: &MachineRoom, command: &str) -> Result<String> {
Ok(exec(room, command).await?.stdout.trim().to_string())
}
pub(super) async fn find<T, F, Fut>(mut probe: F, timeout: Duration, label: &str) -> Result<T>
where
F: FnMut() -> Fut,
Fut: Future<Output = Result<Option<T>>>,
{
let deadline = Instant::now() + timeout;
loop {
if let Some(found) = probe().await? {
return Ok(found);
}
if Instant::now() > deadline {
return fail(format!("Timed out waiting for {label}."));
}
tokio::time::sleep(Duration::from_millis(500)).await;
}
}
pub(super) fn press((x, y): (f64, f64)) -> MachineTool {
MachineTool::Press { x, y, button: None }
}
pub(super) fn release((x, y): (f64, f64)) -> MachineTool {
MachineTool::Release { x, y, button: None }
}
pub(super) fn keyed(key: &str) -> MachineTool {
let stroke = |down| KeyStroke {
key: key.to_string(),
down,
};
MachineTool::Keys {
strokes: vec![stroke(true), stroke(false)],
}
}
pub(super) async fn start_x11_app(room: &MachineRoom, mode: &str) -> Result<()> {
let started = exec(
room,
&format!(
"echo {} | base64 -d > /tmp/x11app.py && (DISPLAY=:0 nohup python3 /tmp/x11app.py {mode} >/dev/null 2>&1 &)",
STANDARD.encode(X11_APP)
),
)
.await?;
if started.code != 0 {
return fail(format!("The X11 app didn't start: {}", started.stderr));
}
Ok(())
}
fn app_origin(window: &Window) -> (f64, f64) {
(window.x + 1.0, window.y + 1.0 + TITLE_BAR)
}
fn page_origin(window: &Window) -> (f64, f64) {
(window.x + 1.0, window.y + 1.0 + TITLE_BAR + WEB_BAR)
}
pub(super) async fn boot(
authorization: &CredentialsOrApiKey,
repository: u64,
spec: Spec,
) -> Result<(MachineRoom, Booted)> {
let probe = Arc::new(Probe::new());
let options = StartOptions {
spec,
group: None,
hooks: Some(Arc::new(Seat(probe.clone()))),
private: None,
};
let from = Instant::now();
let (room, _) = MachineRoom::start(authorization, repository, options).await?;
if room.status() == MachineStatus::Running {
probe
.marks
.send_if_modified(|marks| first(&mut marks.running, Instant::now()));
}
match probe.booted(&room, from).await {
Ok(timed) => Ok((room, timed)),
Err(err) => {
if !matches!(
room.status(),
MachineStatus::Stopped | MachineStatus::Failed
) {
let _ = room.stop().await;
}
Err(err)
}
}
}
pub(super) async fn join(
authorization: &CredentialsOrApiKey,
room: &MachineRoom,
viewer: &Arc<Probe>,
) -> Result<MachineRoom> {
find(
|| async {
Ok(room
.controller()
.filter(|controller| !controller.is_empty())
.map(|_| ()))
},
secs(30),
"the owner's seat to take control",
)
.await?;
let code = room.generate_access_code().await?;
let options = MachineOptions {
hooks: Some(Arc::new(Seat(viewer.clone()))),
..MachineOptions::default()
};
Ok(MachineRoom::join(authorization, &code, options).await?.0)
}
pub(super) async fn round_trip(room: &MachineRoom) -> Result<Spread> {
sampled(move |_| async move {
let sent = Instant::now();
tool(room, MachineTool::Windows).await?;
Ok(since(sent, Instant::now()))
})
.await
}
pub(super) async fn side_by_side(room: &MachineRoom, viewer: &Probe) -> Result<[Window; 2]> {
let terminal = Window::from_json(&serde_json::from_str(
&tool(room, open(WindowApp::Terminal, None)).await?,
)?);
tool(room, open(WindowApp::Files, Some("/"))).await?;
let files = find(
|| async {
Ok(room
.windows()
.into_values()
.find(|w| w.kind == WindowKind::Files))
},
secs(30),
"a files window",
)
.await?;
let windows = [
arrange(room, terminal.id, 32.0).await?,
arrange(room, files.id, 704.0).await?,
];
for window in &windows {
find(
|| async { Ok(viewer.section(window.id)) },
secs(30),
"the viewer to see the window open",
)
.await?;
}
Ok(windows)
}
fn open(app: WindowApp, path: Option<&str>) -> MachineTool {
MachineTool::Open {
app,
url: None,
path: path.map(str::to_string),
command: None,
}
}
async fn arrange(room: &MachineRoom, id: u64, x: f64) -> Result<Window> {
let window = id as f64;
let resize = MachineTool::Resize {
window,
width: 640.0,
height: 400.0,
};
tool(room, resize).await?;
let place = MachineTool::Place { window, x, y: 48.0 };
tool(room, place).await?;
room.windows()
.remove(&id)
.ok_or_else(|| Error::Message(format!("Window {id} closed.")))
}
fn title_bar(window: &Window) -> (f64, f64) {
(window.x + window.width / 2.0, window.y + 12.0)
}
fn unseen(window: &Window) -> Error {
Error::Message(format!("The viewer never saw window {} open.", window.id))
}
pub(super) async fn focus_latency(
room: &MachineRoom,
viewer: &Probe,
pair: [&Window; 2],
) -> Result<Spread> {
let section = |window: &Window| viewer.section(window.id).ok_or_else(|| unseen(window));
let sections = [section(pair[0])?, section(pair[1])?];
let start = usize::from(room.windows().get(&pair[0].id).is_some_and(|w| w.focused));
sampled(move |i| {
let next = (start + i) % 2;
let node = sections[next];
let bar = title_bar(pair[next]);
async move {
let pressed = latency(room, viewer, press(bar), move |m| focuses(m, node)).await;
tool(room, release(bar)).await?;
pressed
}
})
.await
}
pub(super) async fn drag_latency(
room: &MachineRoom,
viewer: &Probe,
window: &Window,
) -> Result<Spread> {
let node = viewer.section(window.id).ok_or_else(|| unseen(window))?;
let left = room.windows().get(&window.id).map_or(window.x, |w| w.x);
let (x, y) = title_bar(window);
tool(room, press((x, y))).await?;
let dragged = sampled(move |i| {
let step = DRAG_STEP * (i + 1) as f64;
let moved = MachineTool::Pointer { x: x + step, y };
latency(room, viewer, moved, move |m| placed(m, node, left + step))
})
.await;
tool(room, release((x + DRAG_STEP * SAMPLES as f64, y))).await?;
dragged
}
pub(super) async fn terminal_latency(
room: &MachineRoom,
viewer: &Probe,
terminal: &Window,
) -> Result<Spread> {
let focus = MachineTool::Focus {
window: terminal.id as f64,
};
tool(room, focus).await?;
tool(room, keyed("#")).await?;
tokio::time::sleep(SETTLE).await;
sampled(move |i| {
let seen = format!("#{}", "q".repeat(i + 1));
latency(room, viewer, keyed("q"), move |m| shows(m, &seen))
})
.await
}
pub(super) async fn web_latency(room: &MachineRoom, viewer: &Probe) -> Result<Spread> {
let served = exec(
room,
&format!(
"mkdir -p /tmp/keys && echo {} | base64 -d > /tmp/keys/index.html && (cd /tmp/keys && nohup python3 -m http.server {KEYS_PORT} >/dev/null 2>&1 &) && curl -s -o /dev/null --retry 20 --retry-connrefused --retry-delay 1 http://127.0.0.1:{KEYS_PORT}/",
STANDARD.encode(KEYS_PAGE)
),
)
.await?;
if served.code != 0 {
return fail(format!("The page's server didn't start: {}", served.stderr));
}
let open = MachineTool::Open {
app: WindowApp::Web,
url: Some(format!("http://localhost:{KEYS_PORT}/")),
path: None,
command: None,
};
let window = Window::from_json(&serde_json::from_str(&tool(room, open).await?)?);
if window.title != "Keys" {
return fail(format!(
"The page didn't load: its window says {}.",
window.title
));
}
tokio::time::sleep(SETTLE).await;
let (left, top) = page_origin(&window);
let page = (left + window.width / 2.0, top + window.height / 2.0);
tool(room, press(page)).await?;
tool(room, release(page)).await?;
tokio::time::sleep(GAP).await;
sampled(move |i| {
let seen = format!("key {} k seen", i + 1);
latency(room, viewer, keyed("k"), move |m| shows(m, &seen))
})
.await
}
pub(super) async fn app_latency(room: &MachineRoom, viewer: &Probe) -> Result<Spread> {
start_x11_app(room, "keys").await?;
let window = find(
|| async { Ok(room.windows().into_values().find(|w| w.app == "x11app")) },
secs(30),
"the X11 app's window",
)
.await?;
let xid = find(
|| async { Ok(viewer.app_windows().into_iter().next()) },
secs(30),
"the viewer to get the X11 app's frames",
)
.await?;
let (left, top) = app_origin(&window);
let point = (left + 40.0, top + 40.0);
tool(room, press(point)).await?;
tool(room, release(point)).await?;
tokio::time::sleep(SETTLE).await;
sampled(move |_| async move {
let key = keyed("k");
let arrived = viewer.awaits_frame(xid);
let sent = Instant::now();
let (done, arrived) = tokio::join!(
room.invoke_tool(&key),
tokio::time::timeout(SAMPLE_WAIT, arrived)
);
done?;
let at = arrived
.map_err(|_| {
Error::Message(format!(
"The viewer got no frame within {} s.",
SAMPLE_WAIT.as_secs()
))
})?
.map_err(|_| Error::Message("The viewer went away.".into()))?;
Ok(since(sent, at))
})
.await
}
fn focuses(mutation: &WebMutation, node: u64) -> bool {
matches!(
mutation,
WebMutation::Attribute { id, name, value: Some(value) }
if *id == node && name == "class" && value.split_whitespace().any(|class| class == "focused")
)
}
fn placed(mutation: &WebMutation, node: u64, left: f64) -> bool {
let at = format!("left: {left}px;");
matches!(
mutation,
WebMutation::Attribute { id, name, value: Some(value) }
if *id == node && name == "style" && value.contains(&at)
)
}
fn shows(mutation: &WebMutation, text: &str) -> bool {
serde_json::to_string(mutation).is_ok_and(|json| json.contains(text))
}
async fn sampled<F>(mut sample: impl FnMut(usize) -> F) -> Result<Spread>
where
F: Future<Output = Result<f64>>,
{
let mut samples = Vec::with_capacity(SAMPLES);
for i in 0..SAMPLES {
samples.push(sample(i).await?);
tokio::time::sleep(GAP).await;
}
Ok(Spread::of(samples))
}
async fn latency(
room: &MachineRoom,
viewer: &Probe,
input: MachineTool,
shows: impl Fn(&WebMutation) -> bool + Send + 'static,
) -> Result<f64> {
let arrived = viewer.awaits(shows);
let sent = Instant::now();
let (done, arrived) = tokio::join!(
room.invoke_tool(&input),
tokio::time::timeout(SAMPLE_WAIT, arrived)
);
done?;
let at = arrived
.map_err(|_| {
Error::Message(format!(
"The viewer saw nothing within {} s.",
SAMPLE_WAIT.as_secs()
))
})?
.map_err(|_| Error::Message("The viewer went away.".into()))?;
Ok(since(sent, at))
}
pub(super) struct Probe {
marks: watch::Sender<Marks>,
sections: Mutex<HashMap<u64, u64>>,
awaited: Mutex<Option<Awaited>>,
apps: Mutex<Vec<u32>>,
awaited_frame: Mutex<Option<(u32, oneshot::Sender<Instant>)>>,
}
#[derive(Clone, Copy, Default)]
struct Marks {
running: Option<Instant>,
desktop: Option<Instant>,
ended: bool,
}
impl Marks {
fn done(&self) -> bool {
self.ended || (self.running.is_some() && self.desktop.is_some())
}
}
struct Awaited {
wants: Box<dyn Fn(&WebMutation) -> bool + Send>,
seen: oneshot::Sender<Instant>,
}
fn first(slot: &mut Option<Instant>, at: Instant) -> bool {
let first = slot.is_none();
if first {
*slot = Some(at);
}
first
}
impl Probe {
pub(super) fn new() -> Probe {
Probe {
marks: watch::Sender::new(Marks::default()),
sections: Mutex::new(HashMap::new()),
awaited: Mutex::new(None),
apps: Mutex::default(),
awaited_frame: Mutex::default(),
}
}
fn app_windows(&self) -> Vec<u32> {
lock(&self.apps).clone()
}
fn awaits_frame(&self, window: u32) -> oneshot::Receiver<Instant> {
let (seen, arrived) = oneshot::channel();
*lock(&self.awaited_frame) = Some((window, seen));
arrived
}
fn on_app_frame(&self, frame: &AppFrame) {
let at = Instant::now();
if (frame.window_width, frame.window_height) == APP_SIZE {
let mut apps = lock(&self.apps);
if !apps.contains(&frame.window) {
apps.push(frame.window);
}
}
let mut awaited = lock(&self.awaited_frame);
if awaited
.as_ref()
.is_some_and(|(window, _)| *window == frame.window)
&& let Some((_, seen)) = awaited.take()
{
let _ = seen.send(at);
}
}
fn section(&self, window: u64) -> Option<u64> {
lock(&self.sections).get(&window).copied()
}
fn awaits(
&self,
wants: impl Fn(&WebMutation) -> bool + Send + 'static,
) -> oneshot::Receiver<Instant> {
let (seen, arrived) = oneshot::channel();
*lock(&self.awaited) = Some(Awaited {
wants: Box::new(wants),
seen,
});
arrived
}
async fn booted(&self, room: &MachineRoom, from: Instant) -> Result<Booted> {
let mut seen = self.marks.subscribe();
let waited = tokio::time::timeout(BOOT_WAIT, seen.wait_for(Marks::done)).await;
let marks = *waited
.map_err(|_| Error::Message("Timed out waiting for the machine to boot.".into()))?
.map_err(|_| Error::Message("The machine's room went away.".into()))?;
match (marks.running, marks.desktop) {
(Some(running), Some(desktop)) if !marks.ended => Ok(Booted {
running: since(from, running),
desktop: since(from, desktop),
}),
_ => fail(format!(
"The machine ended while booting ({:?}) {}",
room.status(),
room.error()
)),
}
}
}
impl WebRoomHooks for Probe {
fn on_bootstrap(&self, _room: &WebRoom, bootstrap: &BootstrapMeta) {
let at = Instant::now();
if bootstrap
.html
.as_deref()
.is_some_and(|html| html.contains(DOCK))
{
self.marks
.send_if_modified(|marks| first(&mut marks.desktop, at));
}
}
fn on_mutation(&self, _room: &WebRoom, mutation: &WebMutation) {
let at = Instant::now();
if let WebMutation::Insert { node, subtree, .. } = mutation {
let mut sections = lock(&self.sections);
for node in std::iter::once(node).chain(subtree) {
if let WebNode::Element {
id,
tag,
attributes,
..
} = node
&& tag.eq_ignore_ascii_case("section")
&& let Some(window) = attributes.get("data-window").and_then(|w| w.parse().ok())
{
sections.insert(window, *id);
}
}
}
let mut awaited = lock(&self.awaited);
if awaited
.as_ref()
.is_some_and(|awaited| (awaited.wants)(mutation))
&& let Some(awaited) = awaited.take()
{
let _ = awaited.seen.send(at);
}
}
}
pub(super) struct Seat(pub(super) Arc<Probe>);
impl MachineRoomHooks for Seat {
fn web(&self) -> Option<Arc<dyn WebRoomHooks>> {
Some(self.0.clone())
}
fn on_app_frame(&self, _room: &MachineRoom, frame: &AppFrame) {
self.0.on_app_frame(frame);
}
fn on_status(&self, room: &MachineRoom) {
let at = Instant::now();
let marks = &self.0.marks;
match room.status() {
MachineStatus::Running => {
marks.send_if_modified(|marks| first(&mut marks.running, at));
}
MachineStatus::Stopped | MachineStatus::Failed => {
marks.send_if_modified(|marks| !std::mem::replace(&mut marks.ended, true));
}
MachineStatus::Booting | MachineStatus::Stopping => {}
}
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn takes_nearest_rank_percentiles() {
let samples: Vec<f64> = (1..=20).rev().map(f64::from).collect();
assert_eq!(
Spread::of(samples),
Spread {
p50: 10.0,
p95: 19.0,
n: 20
}
);
assert_eq!(
Spread::of(vec![7.5]),
Spread {
p50: 7.5,
p95: 7.5,
n: 1
}
);
}
#[test]
fn takes_the_round_trip_off_and_holds_it_to_its_gate() {
let felt = Spread {
p50: 92.4,
p95: 140.0,
n: 20,
};
let beyond = felt.beyond(41.3);
assert_eq!(
beyond,
Spread {
p50: 51.1,
p95: 98.7,
n: 20
}
);
assert_eq!(felt.beyond(200.0).p95, 0.0);
assert!(!DOM_GATE.holds(&beyond));
assert!(APP_GATE.holds(&beyond));
assert!(!APP_GATE.holds(&Spread {
p50: 40.0,
p95: 151.0,
n: 20,
}));
}
#[test]
fn knows_a_window_taking_focus_a_key_reaching_a_row_and_a_drag_landing() {
let attribute = |id, name: &str, value: &str| WebMutation::Attribute {
id,
name: name.into(),
value: Some(value.into()),
};
assert!(focuses(&attribute(7, "class", "window focused"), 7));
assert!(!focuses(&attribute(7, "class", "window"), 7));
assert!(!focuses(&attribute(8, "class", "window focused"), 7));
let row = WebMutation::Text {
id: 4,
text: "agent@machine:/repository$ #qq".into(),
};
assert!(shows(&row, "#qq") && !shows(&row, "#qqq") && !focuses(&row, 4));
let moved = attribute(7, "style", "left: 40px; top: 48px; width: 640px;");
assert!(placed(&moved, 7, 40.0));
assert!(!placed(&moved, 7, 4.0));
assert!(!placed(&moved, 8, 40.0));
}
}