fauna-api 1.0.861

The Fauna SDK for Rust: accounts, Drive, groups, calendar, chats, payments, Parrot and live rooms.
Documentation
//! A Marmot research run that renders and investigates pages on machines: a workspace whose
//! computer is a repository of the run's, and a small dataset that reads documents only by
//! rendering them there or having the machine agent work them, never by extracting them.

use std::sync::Mutex;
use std::time::{Duration, Instant};

use futures_util::future::BoxFuture;
use serde_json::{Map, json};

use super::{Benchmark, Made, Run, Work, ensure, fail, lock, number, stopped};
use crate::error::{Error, Result};
use crate::marmot::media::DocumentMode;
use crate::marmot::research::{
    self, CancelReason, Candidate, CreateDatasetOptions, FormatOrSchemaLeafOrChildren,
    GroupOptions, MachineFrame, ResearchCallbacks, ResearchOptions, Schema, SchemaJSON, SchemaLeaf,
    Snapshot, Termination,
};
use crate::marmot::workspace::{self, PartialWorkspace, WorkspaceOptions};
use crate::parrot::format::{Format, UtteranceFormat};
use crate::private::benchmarks::BenchmarkGroup;

/// What research may spend on models, search and the machine agent, in dollars.
const CAP: f64 = 1.0;
const BLOG: &str = "https://blog.rust-lang.org/";
/// Less than the benchmark's timeout, so research that hangs is cancelled here first.
const RESEARCH_WAIT: Duration = Duration::from_secs(25 * 60);

/// Research stops itself at `CAP`, which a last step can overshoot.
pub(super) struct Research;

impl Benchmark for Research {
    fn id(&self) -> &'static str {
        "marmot"
    }
    fn name(&self) -> &'static str {
        "Marmot research"
    }
    fn group(&self) -> BenchmarkGroup {
        BenchmarkGroup::Product
    }
    fn about(&self) -> &'static str {
        "Marmot research on a blog's newest post that reads pages only by rendering them on the workspace's computer, capped at a dollar."
    }
    fn work(&self) -> Work {
        Work {
            machine_hours: 0.5,
            dollars: CAP * 1.5,
            ..Work::default()
        }
    }
    fn timeout(&self) -> Duration {
        Duration::from_secs(30 * 60)
    }
    fn run<'a>(&'a self, run: &'a Run) -> BoxFuture<'a, Result<String>> {
        Box::pin(async move {
            run.progress(0.0, "Making the workspace and its computer");
            let repository = run.repository("research computer").await?;
            let id = workspace::create_workspace(
                &run.key,
                &run.named("research"),
                None,
                WorkspaceOptions::default(),
            )
            .await?;
            run.made(Made::Workspace(id.clone()));
            let computer = PartialWorkspace {
                repository_id: Some(repository.id),
                ..PartialWorkspace::default()
            };
            workspace::update_workspace(&run.key, &id, &computer, WorkspaceOptions::default())
                .await?;

            let options = CreateDatasetOptions {
                name: run.named("newest post"),
                schema: newest_post(),
                termination: Some(vec![Termination {
                    r#type: "cost".into(),
                    fields: Map::from_iter([("max".to_string(), json!(CAP))]),
                }]),
                content_types: None,
                proxy: None,
                document_modes: Some(vec![DocumentMode::Render, DocumentMode::Virtualized]),
            };
            let dataset =
                research::create_dataset(&run.key, &id, &options, GroupOptions::default()).await?;
            run.made(Made::Dataset {
                workspace: id.clone(),
                id: dataset.clone(),
            });
            let dataset = research::list_datasets(&run.key, &id, GroupOptions::default())
                .await?
                .into_iter()
                .find(|found| found.id == dataset)
                .ok_or_else(|| Error::Message("The new dataset isn't in its workspace.".into()))?;

            run.progress(0.1, "Researching");
            let watched = Watched::default();
            let from = Instant::now();
            let researched = tokio::time::timeout(
                RESEARCH_WAIT,
                dataset.research(&watched, ResearchOptions::default()),
            )
            .await;
            if researched.is_err() {
                let _ = dataset.cancel().await;
            }
            run.measure(number(
                "Research",
                from.elapsed().as_secs_f64().round(),
                "s",
            ));
            run.progress(0.9, "Saving the computer");
            stopped(&run.key, repository.id).await?;
            match researched {
                Err(_) => {
                    return fail(format!(
                        "Research didn't end within {} minutes.",
                        RESEARCH_WAIT.as_secs() / 60
                    ));
                }
                Ok(researched) => researched?,
            };

            let seen = lock(&watched.seen);
            run.measure(number("Pages on the computer", seen.pages.len() as f64, ""));
            let ended = seen.ended.as_deref().unwrap_or("ended without saying how");
            ensure(!seen.pages.is_empty(), || {
                format!("No page went to the workspace's computer. The run {ended}.")
            })?;
            ensure(seen.finished, || {
                format!(
                    "{} pages went to the computer, but the run {ended}.",
                    seen.pages.len()
                )
            })?;
            Ok(format!(
                "{} pages went to the workspace's computer ({}), and the run {ended}.",
                seen.pages.len(),
                seen.pages.join(", ")
            ))
        })
    }
}

/// The blog's newest post: its title and the day it was published.
fn newest_post() -> Schema {
    let field = |instruction: &str| SchemaLeaf {
        value: FormatOrSchemaLeafOrChildren::Format(Format::Utterance(UtteranceFormat::new(
            instruction,
            None,
        ))),
        prompt: None,
        sources: Some(1),
        optional: None,
        array: None,
        max_iterations: None,
        filters: None,
    };
    Schema::Json(SchemaJSON::new(SchemaLeaf {
        value: FormatOrSchemaLeafOrChildren::Children(vec![
            ("title".into(), field("The newest post's title.")),
            ("published".into(), field("The day it was published.")),
        ]),
        prompt: Some(format!(
            "The newest post on {BLOG}, read from the blog itself."
        )),
        sources: None,
        optional: None,
        array: None,
        max_iterations: None,
        filters: None,
    }))
}

#[derive(Default)]
struct Seen {
    /// The pages that went to the computer, in order.
    pages: Vec<String>,
    /// Whether it finished with a snapshot, or stopped at its cost limit.
    finished: bool,
    ended: Option<String>,
}

#[derive(Default)]
struct Watched {
    seen: Mutex<Seen>,
}

impl Watched {
    fn ended(&self, finished: bool, how: String) {
        let mut seen = lock(&self.seen);
        seen.finished = finished;
        seen.ended = Some(how);
    }
}

impl ResearchCallbacks for Watched {
    fn on_machine(&self, frame: &MachineFrame) {
        lock(&self.seen).pages.push(frame.url.clone());
    }

    fn on_done(&self, _snapshot: &Snapshot) {
        self.ended(true, "finished with a snapshot".into());
    }

    fn on_cancelled(&self, reason: CancelReason) {
        self.ended(reason == CancelReason::Cost, format!("stopped: {reason:?}"));
    }

    fn on_convergence_failed(&self, field: &str, reason: &str, _candidates: &[Candidate]) {
        self.ended(false, format!("couldn't settle {field}: {reason}"));
    }

    fn on_error(&self, message: &str) {
        self.ended(false, format!("failed: {message}"));
    }
}