use std::panic::{AssertUnwindSafe, catch_unwind};
use std::sync::Arc;
use std::sync::atomic::{AtomicBool, Ordering};
use durable::{
ConversationId, ConversationOwnership, Doc, Owner, ROOT_CONVERSATION_ID, Session, SessionEnd, SessionError,
Storage, SubmissionId, SubmissionQuery, SubmissionRecord, SubmissionStatus, TaskId, TaskRecord, Tx, TxScope,
};
use serde_json::Value;
use super::cancel::Ctx;
use super::registry::RegistryReader;
use super::{
BlockedReason, Clock, HarnessError, Report, Result, Scheduler, SchedulerOptions, SettleOutcome, WithdrawInputs,
commit_on,
};
use crate::chat::inbox::withdraw_queued_inputs;
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum AbortResult {
Marked,
Terminal,
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum Scheduling {
Paused,
Running,
Closing,
}
#[derive(Clone, Debug, PartialEq)]
pub enum TaskInspectionState {
Running,
Completing,
Waiting {
on: Vec<TaskId>,
},
Blocked {
reason: BlockedReason,
error: Option<HarnessError>,
},
Ready {
migrates: bool,
},
}
#[derive(Clone, Debug, PartialEq)]
pub struct TaskInspection {
pub record: TaskRecord,
pub state: TaskInspectionState,
}
#[derive(Clone, Debug, PartialEq)]
pub struct Inspection {
pub scheduling: Scheduling,
pub tasks: Vec<TaskInspection>,
pub submissions: Vec<SubmissionRecord>,
}
pub(crate) type FallibleClock = Arc<dyn Fn() -> std::result::Result<u64, HarnessError> + Send + Sync>;
#[derive(Clone)]
pub struct HarnessOptions {
pub registry: Arc<dyn RegistryReader>,
pub now: Option<FallibleClock>,
pub on_report: Option<Report>,
pub settle_outcome: Option<SettleOutcome>,
pub withdraw_inputs: Option<WithdrawInputs>,
}
impl HarnessOptions {
pub fn new(registry: Arc<dyn RegistryReader>) -> HarnessOptions {
HarnessOptions {
registry,
now: None,
on_report: None,
settle_outcome: None,
withdraw_inputs: None,
}
}
}
fn wall_clock() -> u64 {
jsrt::date::now_ms() as u64
}
fn safe_report(on_report: Option<Report>) -> Report {
Arc::new(move |error| {
if let Some(on_report) = &on_report {
let _ = catch_unwind(AssertUnwindSafe(|| on_report(error)));
}
})
}
pub fn safe_now(now: Option<FallibleClock>, report: Report) -> Clock {
let Some(now) = now else {
return Arc::new(wall_clock);
};
let reported = AtomicBool::new(false);
Arc::new(move || {
let read =
catch_unwind(AssertUnwindSafe(|| now())).unwrap_or_else(|_| Err(HarnessError::msg("clock panicked")));
match read {
Ok(value) => value,
Err(error) => {
if !reported.swap(true, Ordering::SeqCst) {
report(&error);
}
wall_clock()
}
}
})
}
#[derive(Clone)]
pub struct Harness {
session: Session,
scheduler: Arc<Scheduler>,
}
impl Harness {
pub fn open(storage: impl Storage + Send + 'static, options: HarnessOptions) -> Result<Harness> {
let report = safe_report(options.on_report);
let now = safe_now(options.now, report.clone());
let clock = now.clone();
let session = Session::with_clock(storage, move || clock());
let scheduler = Scheduler::new(SchedulerOptions {
session: session.clone(),
registry: options.registry,
now,
report,
settle_outcome: options.settle_outcome.unwrap_or_else(|| Arc::new(|_, _, _| Ok(()))),
withdraw_inputs: options
.withdraw_inputs
.unwrap_or_else(|| Arc::new(withdraw_queued_inputs)),
});
if let Err(error) = scheduler.open() {
let _ = session.close();
scheduler.join();
return Err(error);
}
Ok(Harness { session, scheduler })
}
pub fn reopen(&self, storage: impl Storage + Send + 'static, options: HarnessOptions) -> Result<Harness> {
let _ = self.close();
Harness::open(storage, options)
}
pub fn session(&self) -> &Session {
&self.session
}
fn assert_usable(&self) -> Result<()> {
if let Some(failure) = self.session.failure() {
return Err(HarnessError::Session(SessionError::Failed(Box::new(failure))));
}
if self.scheduler.is_closing() {
return Err(HarnessError::Session(SessionError::Closed));
}
Ok(())
}
pub fn resume(&self) -> Result<()> {
self.assert_usable()?;
self.scheduler.resume();
Ok(())
}
pub fn commit<T>(&self, change: impl FnOnce(&mut Tx<'_>) -> Result<T>) -> Result<T> {
commit_on(&self.session, TxScope::default(), change)
}
pub fn root(&self) -> Result<Conversation> {
self.assert_usable()?;
let id = self.commit(|tx| {
if tx.conversation(ROOT_CONVERSATION_ID)?.is_some() {
return Ok(ROOT_CONVERSATION_ID);
}
Ok(tx.create_root_conversation()?.id)
})?;
Ok(self.handle(id))
}
pub fn create_conversation(&self, ownership: ConversationOwnership) -> Result<Conversation> {
self.assert_usable()?;
let id = self.commit(|tx| Ok(tx.create_conversation(ownership)?.id))?;
Ok(self.handle(id))
}
pub fn conversation(&self, id: ConversationId) -> Result<Option<Conversation>> {
self.assert_usable()?;
let record = self.commit(|tx| Ok(tx.conversation(id)?))?;
Ok(record.map(|record| self.handle(record.id)))
}
fn handle(&self, id: ConversationId) -> Conversation {
Conversation {
id,
harness: self.clone(),
}
}
pub fn get_task(&self, id: TaskId) -> Result<Option<TaskRecord>> {
self.commit(|tx| Ok(tx.task(id)?))
}
pub fn submission(&self, id: SubmissionId) -> Result<Option<SubmissionRecord>> {
self.commit(|tx| Ok(tx.submission(id)?))
}
pub fn wait_for_task(&self, id: TaskId, ctx: &Ctx) -> Result<TaskRecord> {
self.scheduler.resume();
self.scheduler.wait_for_task(id, ctx)
}
pub fn abort_task(&self, id: TaskId, ctx: &Ctx) -> Result<AbortResult> {
self.scheduler.abort(id, ctx, false)
}
pub fn wait_for_idle(&self, ctx: &Ctx) -> Result<()> {
self.scheduler.resume();
self.scheduler.wait_for_idle(None, ctx)
}
pub fn inspect(&self) -> Result<Inspection> {
let snapshot = self.scheduler.registry().snapshot();
self.commit(|tx| {
let (scheduling, tasks) = self.scheduler.inspect(tx, &snapshot)?;
let mut submissions = Vec::new();
for status in [SubmissionStatus::Queued, SubmissionStatus::Placed] {
let query = SubmissionQuery {
status: Some(status),
..SubmissionQuery::default()
};
let mut cursor = None;
loop {
let page = tx.committed(|storage| {
storage.scan_submissions(&query, super::SCAN_PAGE_SIZE, cursor.as_ref())
})?;
submissions.extend(page.items);
match page.next {
Some(next) => cursor = Some(next),
None => break,
}
}
}
submissions.sort_by_key(|submission| submission.id);
Ok(Inspection {
scheduling,
tasks,
submissions,
})
})
}
pub fn snapshot(&self, doc: &Doc, owner: Owner) -> Result<Option<Arc<Value>>> {
Ok(self.session.snapshot(doc, owner)?)
}
pub fn close(&self) -> Result<()> {
let result = self.session.close();
self.scheduler.join();
self.scheduler.report_failure();
result.map_err(HarnessError::Session)
}
pub fn line_len(&self) -> u64 {
self.session.line_len()
}
pub fn closed(&self) -> SessionEnd {
let end = self.session.wait_closed();
self.scheduler.join();
self.scheduler.report_failure();
end
}
}
#[derive(Clone)]
pub struct Conversation {
pub id: ConversationId,
harness: Harness,
}
impl Conversation {
pub fn commit<T>(&self, change: impl FnOnce(&mut Tx<'_>) -> Result<T>) -> Result<T> {
let scope = TxScope {
conversation_id: Some(self.id),
task_id: None,
};
commit_on(&self.harness.session, scope, change)
}
pub fn abort(&self, background: bool, ctx: &Ctx) -> Result<()> {
self.harness.scheduler.resume();
self.harness.scheduler.abort_conversation(self.id, background, ctx)
}
pub fn wait_for_idle(&self, ctx: &Ctx) -> Result<()> {
self.harness.scheduler.resume();
self.harness.scheduler.wait_for_idle(Some(self.id), ctx)
}
}