use std::collections::HashMap;
use std::pin::Pin;
use std::sync::{Arc, Mutex};
use std::task::{Context as TaskContext, Poll, ready};
use std::time::{Duration, Instant};
use futures_util::StreamExt;
use futures_util::future::BoxFuture;
use serde_json::json;
use tokio::io::{AsyncRead, AsyncWrite, ReadBuf};
use tokio_tungstenite::tungstenite::Message;
use super::seat::{
self, APP_GATE, Booted, DOM_GATE, Gate, Probe, Spread, app_latency, drag_latency, exec, find,
focus_latency, join, read, round_trip, secs, side_by_side, start_x11_app, terminal_latency,
web_latency,
};
use super::{
Benchmark, Phases, Run, Work, ensure, fail, gated, lock, mib, number, since, stopped, tenths,
};
use crate::authentification::CredentialsOrApiKey;
use crate::error::{Error, Result};
use crate::private::benchmarks::BenchmarkGroup;
use crate::termite::os::{AppFrame, MachineRoom};
use crate::termite::spec::{STANDARD_SPEC, Spec};
const NEW_REPOSITORIES: usize = 3;
const SMALL: Spec = Spec {
vcpus: 1,
memory_bytes: 2 << 30,
..STANDARD_SPEC
};
const IDLE: Duration = Duration::from_secs(10);
const MARKER: &str = "/tmp/bench-marker";
const SLOW_LINK_BYTES: f64 = 20_000_000.0 / 8.0;
const SLOW_LINK_MBIT: f64 = 20.0;
const SLOW_RECEIVE_BUFFER: u32 = 64 * 1024;
const STREAM_WARMUP: Duration = Duration::from_secs(3);
const STREAM_FOR: Duration = Duration::from_secs(15);
const DRAIN_LIMIT_MS: f64 = 500.0;
const STEADY_FPS: f64 = 28.5;
const DRAIN_WAIT: Duration = Duration::from_secs(5);
fn machine_hours(hours: f64) -> Work {
Work {
machine_hours: hours,
..Work::default()
}
}
fn phases(run: &Run, prefix: &str, phases: &Phases) {
for (step, ms) in &phases.ms {
run.measure(number(&format!("{prefix} {step}"), *ms, "ms"));
}
}
fn booted(run: &Run, what: &str, booted: &Booted) {
run.measure(number(&format!("{what} to running"), booted.running, "ms"));
run.measure(number(
&format!("{what} to the desktop"),
booted.desktop,
"ms",
));
}
pub(super) struct Boots;
impl Benchmark for Boots {
fn id(&self) -> &'static str {
"boots"
}
fn name(&self) -> &'static str {
"Boots"
}
fn group(&self) -> BenchmarkGroup {
BenchmarkGroup::Machines
}
fn about(&self) -> &'static str {
"New repositories' first starts, to the machine running and to its desktop, and what each phase of the start took."
}
fn work(&self) -> Work {
machine_hours(0.3)
}
fn timeout(&self) -> Duration {
Duration::from_secs(20 * 60)
}
fn run<'a>(&'a self, run: &'a Run) -> BoxFuture<'a, Result<String>> {
Box::pin(async move {
let mut timed = Vec::with_capacity(NEW_REPOSITORIES);
let mut last = None;
for n in 1..=NEW_REPOSITORIES {
run.progress(
(n - 1) as f64 / NEW_REPOSITORIES as f64,
&format!("Starting new repository {n} of {NEW_REPOSITORIES}"),
);
let repository = run.repository(&format!("boot {n}")).await?;
let (room, timing) = seat::boot(&run.key, repository.id, STANDARD_SPEC).await?;
timed.push(timing);
let machine = room.machine();
let _ = room.stop().await;
last = Some(machine);
}
let median =
|mark: fn(&Booted) -> f64| Spread::of(timed.iter().map(mark).collect()).p50;
let medians = Booted {
running: median(|timing| timing.running),
desktop: median(|timing| timing.desktop),
};
booted(run, "A new repository", &medians);
let mut how = String::from("an unknown way");
if let Some(machine) = last {
let start = run.phases(machine, "start").await?;
phases(run, "Start phase", &start);
how = start.how.unwrap_or(how);
}
Ok(format!(
"New repositories ran in {:.1} s and showed their desktop in {:.1} s at the median, the last from {how}.",
medians.running / 1000.0,
medians.desktop / 1000.0
))
})
}
}
pub(super) struct Memory;
async fn idle_memory(run: &Run, repository: u64, spec: Spec) -> Result<Phases> {
let (room, _) = seat::boot(&run.key, repository, spec).await?;
let machine = room.machine();
tokio::time::sleep(IDLE).await;
room.stop().await?;
let stop = run.phases(machine, "stop").await?;
ensure(stop.rss_bytes > 0, || {
"Termite couldn't read the machine's memory.".into()
})?;
Ok(stop)
}
impl Benchmark for Memory {
fn id(&self) -> &'static str {
"memory"
}
fn name(&self) -> &'static str {
"Memory"
}
fn group(&self) -> BenchmarkGroup {
BenchmarkGroup::Machines
}
fn about(&self) -> &'static str {
"QEMU's resident memory once the desktop has idled: a standard machine, the same machine resumed from its memory, and the smallest machine people ask for."
}
fn work(&self) -> Work {
machine_hours(0.3)
}
fn timeout(&self) -> Duration {
Duration::from_secs(20 * 60)
}
fn run<'a>(&'a self, run: &'a Run) -> BoxFuture<'a, Result<String>> {
Box::pin(async move {
run.progress(0.0, "A standard machine idling");
let standard = run.repository("memory").await?;
let idle = idle_memory(run, standard.id, STANDARD_SPEC).await?;
run.measure(number("Idle", mib(idle.rss_bytes), "MiB"));
run.measure(number("Memory snapshot", mib(idle.memory_bytes), "MiB"));
run.progress(0.35, "The same machine resumed");
let resumed = idle_memory(run, standard.id, STANDARD_SPEC).await?;
run.measure(number("After a resume", mib(resumed.rss_bytes), "MiB"));
run.progress(0.7, "The smallest machine idling");
let small = run.repository("memory small").await?;
let smallest = idle_memory(run, small.id, SMALL).await?;
run.measure(number(
"Smallest machine idle",
mib(smallest.rss_bytes),
"MiB",
));
Ok(format!(
"A standard machine held {} MiB idle and {} MiB after a resume; the smallest held {} MiB.",
mib(idle.rss_bytes),
mib(resumed.rss_bytes),
mib(smallest.rss_bytes)
))
})
}
}
pub(super) struct InputLatency;
fn felt(run: &Run, what: &str, spread: &Spread, rtt: f64, gate: Option<Gate>) -> Option<String> {
let beyond = spread.beyond(rtt);
let at = |p: &str| format!("{what} at {p}");
match gate {
Some(gate) => {
let held = gate.holds(&beyond);
run.measure(gated(
&at("the median"),
beyond.p50,
"ms",
gate.words(),
held,
));
run.measure(gated(&at("p95"), beyond.p95, "ms", gate.words(), held));
(!held).then(|| {
format!(
"{what} took {} ms at the median and {} ms at p95, past {} and {}",
beyond.p50, beyond.p95, gate.p50, gate.p95
)
})
}
None => {
run.measure(number(&at("the median"), beyond.p50, "ms"));
run.measure(number(&at("p95"), beyond.p95, "ms"));
None
}
}
}
impl Benchmark for InputLatency {
fn id(&self) -> &'static str {
"input-latency"
}
fn name(&self) -> &'static str {
"Input latency"
}
fn group(&self) -> BenchmarkGroup {
BenchmarkGroup::Machines
}
fn about(&self) -> &'static str {
"Input from the controlling seat reaching a second seat that watches, past the network round trip: a window taking focus, a drag, a Web tab, the terminal and an X11 app."
}
fn work(&self) -> Work {
machine_hours(0.5)
}
fn timeout(&self) -> Duration {
Duration::from_secs(20 * 60)
}
fn run<'a>(&'a self, run: &'a Run) -> BoxFuture<'a, Result<String>> {
Box::pin(async move {
run.progress(0.0, "Starting the machine");
let repository = run.repository("latency").await?;
let (room, _) = seat::boot(&run.key, repository.id, STANDARD_SPEC).await?;
let measured = measure_latency(run, &room).await;
let _ = room.stop().await;
let (rtt, missed) = measured?;
ensure(missed.is_empty(), || missed.join("; "))?;
Ok(format!(
"Every kind of window held its gate past a {} ms round trip.",
rtt.p50
))
})
}
}
async fn measure_latency(run: &Run, room: &MachineRoom) -> Result<(Spread, Vec<String>)> {
let viewer = Arc::new(Probe::new());
let watcher = join(&run.key, room, &viewer).await?;
let measured = async {
run.progress(0.1, "The room's round trip");
let rtt = round_trip(room).await?;
run.measure(number("Round trip at the median", rtt.p50, "ms"));
run.measure(number("Round trip at p95", rtt.p95, "ms"));
let mut missed = Vec::new();
run.progress(0.25, "A window taking focus");
let [terminal, files] = side_by_side(room, &viewer).await?;
let desktop = focus_latency(room, &viewer, [&terminal, &files]).await?;
missed.extend(felt(run, "The desktop", &desktop, rtt.p50, Some(DOM_GATE)));
run.progress(0.4, "A dragged window");
let dragged = drag_latency(room, &viewer, &terminal).await?;
felt(run, "A drag", &dragged, rtt.p50, None);
run.progress(0.55, "Typing in the terminal");
let typed = terminal_latency(room, &viewer, &terminal).await?;
missed.extend(felt(run, "The terminal", &typed, rtt.p50, Some(DOM_GATE)));
run.progress(0.7, "Typing in a Web tab");
let web = web_latency(room, &viewer).await?;
missed.extend(felt(run, "A Web tab", &web, rtt.p50, Some(DOM_GATE)));
run.progress(0.85, "Typing in an X11 app");
let app = app_latency(room, &viewer).await?;
missed.extend(felt(run, "An X11 app", &app, rtt.p50, Some(APP_GATE)));
Ok((rtt, missed))
}
.await;
watcher.disconnect();
measured
}
pub(super) struct AppStream;
impl Benchmark for AppStream {
fn id(&self) -> &'static str {
"app-stream"
}
fn name(&self) -> &'static str {
"App stream"
}
fn group(&self) -> BenchmarkGroup {
BenchmarkGroup::Machines
}
fn about(&self) -> &'static str {
"An X11 app animating at 60 frames a second, watched over a 20 Mbit link: the frames a second it gets, how long frames keep coming once the app ends, and termited's share of a CPU."
}
fn work(&self) -> Work {
machine_hours(0.3)
}
fn timeout(&self) -> Duration {
Duration::from_secs(15 * 60)
}
fn run<'a>(&'a self, run: &'a Run) -> BoxFuture<'a, Result<String>> {
Box::pin(async move {
run.progress(0.0, "Starting the machine");
let repository = run.repository("app stream").await?;
let (room, _) = seat::boot(&run.key, repository.id, STANDARD_SPEC).await?;
run.progress(0.3, "Streaming the animated app over a slow link");
let streamed = app_stream(&run.key, &room).await;
let _ = room.stop().await;
let stream = streamed?;
let steady = stream.fps >= STEADY_FPS;
let drained = stream.drain_ms <= DRAIN_LIMIT_MS;
run.measure(gated(
"Frames a second",
stream.fps,
"fps",
format!("{STEADY_FPS} or more over {SLOW_LINK_MBIT} Mbit"),
steady,
));
run.measure(gated(
"Frames after the app ended",
stream.drain_ms,
"ms",
format!("{DRAIN_LIMIT_MS} ms at most, so no queue grows"),
drained,
));
run.measure(number("Frames", stream.frames as f64, "frames"));
run.measure(number(
"termited's CPU",
stream.termited_cpu_percent,
"% of a CPU",
));
ensure(steady && drained, || {
format!(
"The slow viewer got {} frames a second and frames kept arriving {} ms after the app ended",
stream.fps, stream.drain_ms
)
})?;
Ok(format!(
"{} frames a second over {SLOW_LINK_MBIT} Mbit, the last {} ms after the app ended, with termited at {}% of a CPU.",
stream.fps, stream.drain_ms, stream.termited_cpu_percent
))
})
}
}
struct Streamed {
fps: f64,
frames: usize,
drain_ms: f64,
termited_cpu_percent: f64,
}
async fn app_stream(key: &CredentialsOrApiKey, room: &MachineRoom) -> Result<Streamed> {
seat::find(
|| async {
Ok(room
.controller()
.filter(|controller| !controller.is_empty())
.map(|_| ()))
},
secs(30),
"the owner's seat to take control",
)
.await?;
let code = room.generate_access_code().await?;
let slow = SlowViewer::join(key, &code).await?;
start_x11_app(room, "spin").await?;
let spin = find(
|| async { Ok(slow.windows().into_iter().next()) },
secs(30),
"the slow viewer to get the animated app's frames",
)
.await?;
tokio::time::sleep(STREAM_WARMUP).await;
let ticks = || {
read(
room,
"cat /proc/$(pgrep -o -f termited.py)/stat | awk '{print $14 + $15}'",
)
};
let parsed = |text: String| {
text.parse::<f64>()
.map_err(|_| Error::Message(format!("Not a number: {text}")))
};
let cpu_from = parsed(ticks().await?)?;
let counted = slow.count(spin);
let from = Instant::now();
tokio::time::sleep(STREAM_FOR).await;
let frames = slow.count(spin) - counted;
let seconds = from.elapsed().as_secs_f64();
let cpu_to = parsed(ticks().await?)?;
let per_second = parsed(read(room, "getconf CLK_TCK").await?)?;
exec(room, "pkill -f 'x11app.py spin'").await?;
let killed = Instant::now();
tokio::time::sleep(DRAIN_WAIT).await;
let drain_ms = slow.last(spin).map_or(0.0, |at| since(killed, at));
slow.close();
Ok(Streamed {
fps: tenths(frames as f64 / seconds),
frames,
drain_ms,
termited_cpu_percent: tenths((cpu_to - cpu_from) / per_second / seconds * 100.0),
})
}
struct Throttled<S> {
inner: S,
rate: f64,
tokens: f64,
at: Instant,
wait: Option<Pin<Box<tokio::time::Sleep>>>,
}
impl<S> Throttled<S> {
fn new(inner: S, rate: f64) -> Throttled<S> {
Throttled {
inner,
rate,
tokens: 0.0,
at: Instant::now(),
wait: None,
}
}
}
impl<S: AsyncRead + Unpin> AsyncRead for Throttled<S> {
fn poll_read(
self: Pin<&mut Self>,
cx: &mut TaskContext<'_>,
buf: &mut ReadBuf<'_>,
) -> Poll<std::io::Result<()>> {
let me = self.get_mut();
loop {
if let Some(wait) = me.wait.as_mut() {
ready!(wait.as_mut().poll(cx));
me.wait = None;
}
let now = Instant::now();
me.tokens =
(me.tokens + now.duration_since(me.at).as_secs_f64() * me.rate).min(me.rate / 10.0);
me.at = now;
if me.tokens >= 1.0 {
break;
}
let short = Duration::from_secs_f64((1.0 - me.tokens) / me.rate);
me.wait = Some(Box::pin(tokio::time::sleep(
short.max(Duration::from_millis(1)),
)));
}
let allowed = (me.tokens as usize).min(buf.remaining());
let mut limited = buf.take(allowed);
ready!(Pin::new(&mut me.inner).poll_read(cx, &mut limited))?;
let n = limited.filled().len();
unsafe { buf.assume_init(n) };
buf.advance(n);
me.tokens -= n as f64;
Poll::Ready(Ok(()))
}
}
impl<S: AsyncWrite + Unpin> AsyncWrite for Throttled<S> {
fn poll_write(
self: Pin<&mut Self>,
cx: &mut TaskContext<'_>,
data: &[u8],
) -> Poll<std::io::Result<usize>> {
Pin::new(&mut self.get_mut().inner).poll_write(cx, data)
}
fn poll_flush(self: Pin<&mut Self>, cx: &mut TaskContext<'_>) -> Poll<std::io::Result<()>> {
Pin::new(&mut self.get_mut().inner).poll_flush(cx)
}
fn poll_shutdown(self: Pin<&mut Self>, cx: &mut TaskContext<'_>) -> Poll<std::io::Result<()>> {
Pin::new(&mut self.get_mut().inner).poll_shutdown(cx)
}
}
struct SlowViewer {
seen: Arc<Mutex<HashMap<u32, (usize, Instant)>>>,
reading: tokio::task::JoinHandle<()>,
}
impl SlowViewer {
async fn join(authorization: &CredentialsOrApiKey, code: &str) -> Result<SlowViewer> {
let base = reqwest::Url::parse(&crate::termite::get_url())
.map_err(|err| Error::Message(err.to_string()))?;
let host = base
.host_str()
.ok_or_else(|| Error::Message(format!("No host in {base}")))?
.to_string();
let port = base
.port_or_known_default()
.ok_or_else(|| Error::Message(format!("No port in {base}")))?;
let header = match authorization {
CredentialsOrApiKey::Credentials(credentials) => format!("Token {}", credentials.token),
CredentialsOrApiKey::ApiKey(key) => format!("ApiKey {}", key.id),
};
let packed = json!({ "Authorization": header, "AccessCode": code }).to_string();
let scheme = if base.scheme() == "https" {
"wss"
} else {
"ws"
};
let url = format!(
"{scheme}://{host}:{port}/termite/machine/live?authorization={}",
crate::http::encode(&packed)
);
let address = tokio::net::lookup_host((host.as_str(), port))
.await
.map_err(|err| Error::Message(err.to_string()))?
.next()
.ok_or_else(|| Error::Message(format!("{host} has no address")))?;
let socket = if address.is_ipv4() {
tokio::net::TcpSocket::new_v4()
} else {
tokio::net::TcpSocket::new_v6()
}
.map_err(|err| Error::Message(err.to_string()))?;
socket
.set_recv_buffer_size(SLOW_RECEIVE_BUFFER)
.map_err(|err| Error::Message(err.to_string()))?;
let stream = socket
.connect(address)
.await
.map_err(|err| Error::Message(err.to_string()))?;
let (mut ws, _) =
tokio_tungstenite::client_async_tls(url, Throttled::new(stream, SLOW_LINK_BYTES))
.await?;
let seen: Arc<Mutex<HashMap<u32, (usize, Instant)>>> = Arc::default();
let into = seen.clone();
let reading = tokio::spawn(async move {
while let Some(Ok(message)) = ws.next().await {
let Message::Binary(data) = message else {
continue;
};
if let Some(frame) = AppFrame::from_bytes(&data) {
let at = Instant::now();
let mut seen = lock(&into);
let entry = seen.entry(frame.window).or_insert((0, at));
*entry = (entry.0 + 1, at);
}
}
});
Ok(SlowViewer { seen, reading })
}
fn windows(&self) -> Vec<u32> {
lock(&self.seen).keys().copied().collect()
}
fn count(&self, window: u32) -> usize {
lock(&self.seen).get(&window).map_or(0, |(n, _)| *n)
}
fn last(&self, window: u32) -> Option<Instant> {
lock(&self.seen).get(&window).map(|(_, at)| *at)
}
fn close(self) {
self.reading.abort();
}
}
pub(super) struct Saving;
impl Benchmark for Saving {
fn id(&self) -> &'static str {
"saving"
}
fn name(&self) -> &'static str {
"Saving"
}
fn group(&self) -> BenchmarkGroup {
BenchmarkGroup::Machines
}
fn about(&self) -> &'static str {
"A stop that saves a revision with the machine's memory, then a resume from it: how long each took, the snapshot's size, and whether a file in memory lived through it."
}
fn work(&self) -> Work {
machine_hours(0.2)
}
fn timeout(&self) -> Duration {
Duration::from_secs(15 * 60)
}
fn run<'a>(&'a self, run: &'a Run) -> BoxFuture<'a, Result<String>> {
Box::pin(async move {
run.progress(0.0, "Starting the machine");
let repository = run.repository("saving").await?;
let (room, _) = seat::boot(&run.key, repository.id, STANDARD_SPEC).await?;
let marker = uuid::Uuid::new_v4().simple().to_string();
let wrote = exec(&room, &format!("echo {marker} > {MARKER}")).await?;
if wrote.code != 0 {
let _ = room.stop().await;
return fail(format!("Writing the marker failed: {}", wrote.stderr));
}
run.progress(0.3, "Stopping, which saves the machine's memory");
let machine = room.machine();
let from = Instant::now();
room.stop().await?;
let stop_ms = since(from, Instant::now());
run.measure(number("Stop and save", stop_ms, "ms"));
let stop = run.phases(machine, "stop").await?;
run.measure(number("Memory snapshot", mib(stop.memory_bytes), "MiB"));
phases(run, "Stop phase", &stop);
stopped(&run.key, repository.id).await?;
run.progress(0.6, "Resuming from the snapshot");
let (room, resume) = seat::boot(&run.key, repository.id, STANDARD_SPEC).await?;
booted(run, "The resume", &resume);
let start = run.phases(room.machine(), "start").await?;
phases(run, "Resume phase", &start);
let read = exec(&room, &format!("cat {MARKER}")).await;
let _ = room.stop().await;
let lived = read?.stdout.trim() == marker;
run.measure(gated(
"A file in /tmp lived",
if lived { 1.0 } else { 0.0 },
"",
"The resume keeps what memory held".into(),
lived,
));
ensure(lived, || {
"The file written to /tmp before the stop was gone after the resume.".into()
})?;
Ok(format!(
"Saved in {:.1} s with a {} MiB memory snapshot, and resumed to the desktop in {:.1} s {}.",
stop_ms / 1000.0,
mib(stop.memory_bytes),
resume.desktop / 1000.0,
start
.how
.map_or(String::new(), |how| format!("from a {how}"))
))
})
}
}