use std::sync::Mutex;
use std::time::{Duration, Instant};
use futures_util::future::BoxFuture;
use serde_json::{Map, json};
use super::{Benchmark, Made, Run, Work, ensure, fail, lock, number, stopped};
use crate::error::{Error, Result};
use crate::marmot::media::DocumentMode;
use crate::marmot::research::{
self, CancelReason, Candidate, CreateDatasetOptions, FormatOrSchemaLeafOrChildren,
GroupOptions, MachineFrame, ResearchCallbacks, ResearchOptions, Schema, SchemaJSON, SchemaLeaf,
Snapshot, Termination,
};
use crate::marmot::workspace::{self, PartialWorkspace, WorkspaceOptions};
use crate::parrot::format::{Format, UtteranceFormat};
use crate::private::benchmarks::BenchmarkGroup;
const CAP: f64 = 1.0;
const BLOG: &str = "https://blog.rust-lang.org/";
const RESEARCH_WAIT: Duration = Duration::from_secs(25 * 60);
pub(super) struct Research;
impl Benchmark for Research {
fn id(&self) -> &'static str {
"marmot"
}
fn name(&self) -> &'static str {
"Marmot research"
}
fn group(&self) -> BenchmarkGroup {
BenchmarkGroup::Product
}
fn about(&self) -> &'static str {
"Marmot research on a blog's newest post that reads pages only by rendering them on the workspace's computer, capped at a dollar."
}
fn work(&self) -> Work {
Work {
machine_hours: 0.5,
dollars: CAP * 1.5,
..Work::default()
}
}
fn timeout(&self) -> Duration {
Duration::from_secs(30 * 60)
}
fn run<'a>(&'a self, run: &'a Run) -> BoxFuture<'a, Result<String>> {
Box::pin(async move {
run.progress(0.0, "Making the workspace and its computer");
let repository = run.repository("research computer").await?;
let id = workspace::create_workspace(
&run.key,
&run.named("research"),
None,
WorkspaceOptions::default(),
)
.await?;
run.made(Made::Workspace(id.clone()));
let computer = PartialWorkspace {
repository_id: Some(repository.id),
..PartialWorkspace::default()
};
workspace::update_workspace(&run.key, &id, &computer, WorkspaceOptions::default())
.await?;
let options = CreateDatasetOptions {
name: run.named("newest post"),
schema: newest_post(),
termination: Some(vec![Termination {
r#type: "cost".into(),
fields: Map::from_iter([("max".to_string(), json!(CAP))]),
}]),
content_types: None,
proxy: None,
document_modes: Some(vec![DocumentMode::Render, DocumentMode::Virtualized]),
};
let dataset =
research::create_dataset(&run.key, &id, &options, GroupOptions::default()).await?;
run.made(Made::Dataset {
workspace: id.clone(),
id: dataset.clone(),
});
let dataset = research::list_datasets(&run.key, &id, GroupOptions::default())
.await?
.into_iter()
.find(|found| found.id == dataset)
.ok_or_else(|| Error::Message("The new dataset isn't in its workspace.".into()))?;
run.progress(0.1, "Researching");
let watched = Watched::default();
let from = Instant::now();
let researched = tokio::time::timeout(
RESEARCH_WAIT,
dataset.research(&watched, ResearchOptions::default()),
)
.await;
if researched.is_err() {
let _ = dataset.cancel().await;
}
run.measure(number(
"Research",
from.elapsed().as_secs_f64().round(),
"s",
));
run.progress(0.9, "Saving the computer");
stopped(&run.key, repository.id).await?;
match researched {
Err(_) => {
return fail(format!(
"Research didn't end within {} minutes.",
RESEARCH_WAIT.as_secs() / 60
));
}
Ok(researched) => researched?,
};
let seen = lock(&watched.seen);
run.measure(number("Pages on the computer", seen.pages.len() as f64, ""));
let ended = seen.ended.as_deref().unwrap_or("ended without saying how");
ensure(!seen.pages.is_empty(), || {
format!("No page went to the workspace's computer. The run {ended}.")
})?;
ensure(seen.finished, || {
format!(
"{} pages went to the computer, but the run {ended}.",
seen.pages.len()
)
})?;
Ok(format!(
"{} pages went to the workspace's computer ({}), and the run {ended}.",
seen.pages.len(),
seen.pages.join(", ")
))
})
}
}
fn newest_post() -> Schema {
let field = |instruction: &str| SchemaLeaf {
value: FormatOrSchemaLeafOrChildren::Format(Format::Utterance(UtteranceFormat::new(
instruction,
None,
))),
prompt: None,
sources: Some(1),
optional: None,
array: None,
max_iterations: None,
filters: None,
};
Schema::Json(SchemaJSON::new(SchemaLeaf {
value: FormatOrSchemaLeafOrChildren::Children(vec![
("title".into(), field("The newest post's title.")),
("published".into(), field("The day it was published.")),
]),
prompt: Some(format!(
"The newest post on {BLOG}, read from the blog itself."
)),
sources: None,
optional: None,
array: None,
max_iterations: None,
filters: None,
}))
}
#[derive(Default)]
struct Seen {
pages: Vec<String>,
finished: bool,
ended: Option<String>,
}
#[derive(Default)]
struct Watched {
seen: Mutex<Seen>,
}
impl Watched {
fn ended(&self, finished: bool, how: String) {
let mut seen = lock(&self.seen);
seen.finished = finished;
seen.ended = Some(how);
}
}
impl ResearchCallbacks for Watched {
fn on_machine(&self, frame: &MachineFrame) {
lock(&self.seen).pages.push(frame.url.clone());
}
fn on_done(&self, _snapshot: &Snapshot) {
self.ended(true, "finished with a snapshot".into());
}
fn on_cancelled(&self, reason: CancelReason) {
self.ended(reason == CancelReason::Cost, format!("stopped: {reason:?}"));
}
fn on_convergence_failed(&self, field: &str, reason: &str, _candidates: &[Candidate]) {
self.ended(false, format!("couldn't settle {field}: {reason}"));
}
fn on_error(&self, message: &str) {
self.ended(false, format!("failed: {message}"));
}
}