mod agent;
mod ant;
mod chat;
mod gallery;
mod machines;
mod marmot;
mod media;
mod seat;
use std::cmp::Reverse;
use std::collections::BTreeMap;
use std::fmt;
use std::sync::{Arc, Mutex, MutexGuard, PoisonError};
use std::time::{Duration, Instant};
use futures_util::future::BoxFuture;
use super::{BenchmarkGroup, BenchmarkMeasurement};
use crate::ant::timeline::{self, GroupOptions as TimelineOptions};
use crate::authentification::CredentialsOrApiKey;
use crate::error::{Error, Result};
use crate::marmot::research::{self, GroupOptions as DatasetOptions};
use crate::marmot::workspace::{self, WorkspaceOptions};
use crate::parrot::chat as chats;
use crate::termite::os::{GroupOptions, MachineRoom};
use crate::termite::versioning::{self, CreateOptions, FileOrFiles, Repository};
pub use chat::MODEL;
const STOP_WAIT: Duration = Duration::from_secs(600);
const PHASES_WAIT: Duration = Duration::from_secs(30);
pub trait Benchmark: Send + Sync {
fn id(&self) -> &'static str;
fn name(&self) -> &'static str;
fn group(&self) -> BenchmarkGroup;
fn about(&self) -> &'static str;
fn work(&self) -> Work;
fn timeout(&self) -> Duration;
fn run<'a>(&'a self, run: &'a Run) -> BoxFuture<'a, Result<String>>;
}
pub fn catalog() -> Vec<Arc<dyn Benchmark>> {
vec![
Arc::new(machines::Boots),
Arc::new(machines::Memory),
Arc::new(machines::InputLatency),
Arc::new(machines::AppStream),
Arc::new(machines::Saving),
Arc::new(gallery::Gallery),
Arc::new(gallery::Blender),
Arc::new(chat::SignIn),
Arc::new(chat::TestsVideo),
Arc::new(chat::BlenderChat),
Arc::new(ant::Simulation),
Arc::new(marmot::Research),
]
}
#[derive(Debug, Clone, Copy, Default, PartialEq)]
pub struct Work {
pub steps: f64,
pub turns: f64,
pub machine_hours: f64,
pub dollars: f64,
}
#[derive(Debug, Clone, Default, PartialEq)]
pub struct Phases {
pub how: Option<String>,
pub ms: BTreeMap<String, f64>,
pub rss_bytes: u64,
pub memory_bytes: u64,
}
pub trait Sink: Send + Sync {
fn progress(&self, fraction: f64, note: &str);
fn measure(&self, measurement: BenchmarkMeasurement);
fn file<'a>(
&'a self,
name: &'a str,
content_type: &'a str,
bytes: Vec<u8>,
) -> BoxFuture<'a, Result<()>>;
fn phases<'a>(&'a self, machine: u64, event: &'a str) -> BoxFuture<'a, Option<Phases>>;
}
pub struct Run {
pub key: CredentialsOrApiKey,
tag: String,
sink: Arc<dyn Sink>,
made: Ledger,
}
impl Run {
pub fn new(key: CredentialsOrApiKey, tag: String, sink: Arc<dyn Sink>) -> Run {
Run {
made: Ledger::new(key.clone()),
key,
tag,
sink,
}
}
pub fn named(&self, what: &str) -> String {
format!("Benchmark {} {what}", self.tag)
}
pub fn progress(&self, fraction: f64, note: &str) {
self.sink.progress(fraction.clamp(0.0, 1.0), note);
}
pub fn measure(&self, measurement: BenchmarkMeasurement) {
self.sink.measure(measurement);
}
pub async fn file(&self, name: &str, content_type: &str, bytes: Vec<u8>) -> Result<()> {
self.sink.file(name, content_type, bytes).await
}
pub async fn phases(&self, machine: u64, event: &str) -> Result<Phases> {
let deadline = Instant::now() + PHASES_WAIT;
loop {
if let Some(phases) = self.sink.phases(machine, event).await {
return Ok(phases);
}
if Instant::now() > deadline {
return fail(format!(
"Termite recorded no {event} of machine {machine} within {} s.",
PHASES_WAIT.as_secs()
));
}
tokio::time::sleep(Duration::from_secs(1)).await;
}
}
pub async fn repository(&self, what: &str) -> Result<Repository> {
let options = CreateOptions {
name: Some(self.named(what)),
group: None,
};
let repository = versioning::create(&self.key, options).await?;
self.made.add(Made::Repository(repository.id));
Ok(repository)
}
pub(crate) fn made(&self, made: Made) {
self.made.add(made);
}
pub async fn stop_machines(&self) {
self.made.stop_machines().await;
}
pub async fn clear(&self) -> Vec<String> {
self.made.clear().await
}
}
#[derive(Debug, Clone, PartialEq)]
pub(crate) enum Made {
Chat(String),
Timeline(String),
Dataset { workspace: String, id: String },
Workspace(String),
Repository(u64),
}
impl Made {
fn rank(&self) -> u8 {
match self {
Made::Chat(_) => 0,
Made::Timeline(_) => 1,
Made::Dataset { .. } => 2,
Made::Workspace(_) => 3,
Made::Repository(_) => 4,
}
}
async fn delete(&self, key: &CredentialsOrApiKey) -> Result<()> {
match self {
Made::Chat(id) => chats::delete_chat(key, id, None).await?,
Made::Timeline(id) => {
timeline::delete_timeline(key, id, TimelineOptions::default()).await?
}
Made::Dataset { workspace, id } => {
research::list_datasets(key, workspace, DatasetOptions::default())
.await?
.into_iter()
.find(|dataset| &dataset.id == id)
.ok_or_else(|| Error::Message("it isn't in its workspace".into()))?
.remove()
.await?
}
Made::Workspace(id) => {
workspace::remove_workspace(key, id, WorkspaceOptions::default()).await?
}
Made::Repository(id) => {
stopped(key, *id).await?;
versioning::get(key, *id, versioning::GroupOptions::default())
.await?
.delete()
.await?
}
}
Ok(())
}
}
impl fmt::Display for Made {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
Made::Chat(id) => write!(f, "chat {id}"),
Made::Timeline(id) => write!(f, "Ant timeline {id}"),
Made::Dataset { workspace, id } => {
write!(f, "Marmot dataset {id} in workspace {workspace}")
}
Made::Workspace(id) => write!(f, "Marmot workspace {id}"),
Made::Repository(id) => write!(f, "repository {id}"),
}
}
}
fn teardown(made: Vec<Made>) -> Vec<Made> {
let mut ordered: Vec<(usize, Made)> = made.into_iter().enumerate().collect();
ordered.sort_by_key(|(at, made)| (made.rank(), Reverse(*at)));
ordered.into_iter().map(|(_, made)| made).collect()
}
struct Ledger {
key: CredentialsOrApiKey,
made: Mutex<Vec<Made>>,
}
impl Ledger {
fn new(key: CredentialsOrApiKey) -> Ledger {
Ledger {
key,
made: Mutex::default(),
}
}
fn add(&self, made: Made) {
lock(&self.made).push(made);
}
async fn stop_machines(&self) {
let repositories: Vec<u64> = lock(&self.made)
.iter()
.filter_map(|made| match made {
Made::Repository(id) => Some(*id),
_ => None,
})
.collect();
for id in repositories {
let _ = stopped(&self.key, id).await;
}
}
async fn clear(&self) -> Vec<String> {
let made = std::mem::take(&mut *lock(&self.made));
let mut left = Vec::new();
for made in teardown(made) {
if let Err(err) = made.delete(&self.key).await {
left.push(format!("{made}: {err}"));
}
}
left
}
}
pub(crate) fn lock<T>(mutex: &Mutex<T>) -> MutexGuard<'_, T> {
mutex.lock().unwrap_or_else(PoisonError::into_inner)
}
pub(crate) fn fail<T>(text: impl Into<String>) -> Result<T> {
Err(Error::Message(text.into()))
}
pub(crate) fn ensure(holds: bool, text: impl FnOnce() -> String) -> Result<()> {
if holds { Ok(()) } else { fail(text()) }
}
pub(crate) async fn stopped(key: &CredentialsOrApiKey, repository: u64) -> Result<()> {
let _ = MachineRoom::shutdown(key, repository, GroupOptions::default()).await;
let deadline = Instant::now() + STOP_WAIT;
loop {
let found = versioning::get(key, repository, versioning::GroupOptions::default()).await?;
if found.machine.is_none() {
return Ok(());
}
if Instant::now() > deadline {
return fail(format!(
"Repository {repository}'s machine didn't stop within {} minutes.",
STOP_WAIT.as_secs() / 60
));
}
tokio::time::sleep(Duration::from_secs(5)).await;
}
}
pub(crate) async fn read(repository: &Repository, path: &str) -> Result<String> {
match repository.open(Some(path)).await? {
FileOrFiles::File(mut file) => Ok(file.text().await?),
FileOrFiles::Files(_) => fail(format!("{path} is a folder.")),
}
}
pub(crate) async fn bytes(repository: &Repository, path: &str) -> Result<Vec<u8>> {
match repository.open(Some(path)).await? {
FileOrFiles::File(mut file) => file.download(false, None).await,
FileOrFiles::Files(_) => fail(format!("{path} is a folder.")),
}
}
pub(crate) fn nonce(n: usize) -> String {
uuid::Uuid::new_v4().simple().to_string()[..n].to_string()
}
pub(crate) fn short(text: &str) -> String {
let mut chars = text.trim().chars();
let head: String = chars.by_ref().take(400).collect();
if chars.next().is_some() {
format!("{head}...")
} else {
head
}
}
pub(crate) fn tenths(n: f64) -> f64 {
(n * 10.0).round() / 10.0
}
pub(crate) fn since(from: Instant, at: Instant) -> f64 {
tenths(at.saturating_duration_since(from).as_secs_f64() * 1000.0)
}
pub(crate) fn number(name: &str, value: f64, unit: &str) -> BenchmarkMeasurement {
BenchmarkMeasurement {
name: name.to_string(),
value,
unit: unit.to_string(),
gate: None,
passed: None,
}
}
pub(crate) fn gated(
name: &str,
value: f64,
unit: &str,
gate: String,
passed: bool,
) -> BenchmarkMeasurement {
BenchmarkMeasurement {
gate: Some(gate),
passed: Some(passed),
..number(name, value, unit)
}
}
pub(crate) fn mib(bytes: u64) -> f64 {
tenths(bytes as f64 / f64::from(1 << 20))
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn deletes_what_uses_a_repository_before_the_repository() {
let made = vec![
Made::Repository(1),
Made::Chat("10".into()),
Made::Repository(2),
Made::Chat("11".into()),
Made::Workspace("w".into()),
Made::Dataset {
workspace: "w".into(),
id: "d".into(),
},
Made::Repository(3),
Made::Timeline("t".into()),
];
assert_eq!(
teardown(made),
[
Made::Chat("11".into()),
Made::Chat("10".into()),
Made::Timeline("t".into()),
Made::Dataset {
workspace: "w".into(),
id: "d".into(),
},
Made::Workspace("w".into()),
Made::Repository(3),
Made::Repository(2),
Made::Repository(1),
]
);
assert_eq!(
Made::Dataset {
workspace: "w".into(),
id: "d".into()
}
.to_string(),
"Marmot dataset d in workspace w"
);
}
#[test]
fn every_benchmark_has_its_own_id() {
let ids: Vec<&str> = catalog().iter().map(|b| b.id()).collect();
let mut unique = ids.clone();
unique.sort();
unique.dedup();
assert_eq!(ids.len(), unique.len(), "{ids:?}");
assert!(!ids.contains(&"live"), "live is the run's socket");
}
#[test]
fn shortens_long_text_and_rounds_to_tenths() {
assert_eq!(short(" done "), "done");
assert_eq!(short(&"a".repeat(500)), format!("{}...", "a".repeat(400)));
assert_eq!(nonce(6).len(), 6);
assert_eq!(tenths(1.26), 1.3);
assert_eq!(mib(3 << 20), 3.0);
}
}