use std::sync::{Arc, LazyLock, Mutex};
use std::time::{Duration, Instant};
use ai::{AssistantContent, Message, Tool};
use durable::{
ConversationId, EntryDraft, EntryId, EntryRecord, TaskDefinition, TaskId, TaskOutcome, TaskOutcomeError,
TaskRecord, TaskState, Tx,
};
use serde::{Deserialize, Serialize};
use serde_json::{Value, json};
use super::Services;
use super::entries;
use super::generation::Control;
use super::live::{CELL_KIND, clear_progress, finish_slot, live, slot, slot_index, slot_path};
use super::throttle::Throttle;
use super::usage::{Bucket, record_usage};
use super::util::{d, from_value, strict, to_value};
use crate::scheduler::cancel::lock;
use crate::scheduler::{Cancel, HarnessError, NextState, Result, Task, TaskRuntime};
pub const DEFAULT_MAX_BYTES: usize = 50 * 1024;
pub const DEFAULT_MAX_LINES: usize = 2000;
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub enum Replay {
Safe,
#[default]
Unsafe,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct Diagnostic {
pub severity: String,
pub code: String,
pub message: String,
}
impl Diagnostic {
pub fn error(code: &str, message: &str) -> Diagnostic {
Diagnostic {
severity: "error".into(),
code: code.into(),
message: message.into(),
}
}
fn warn(code: &str, message: &str) -> Diagnostic {
Diagnostic {
severity: "warn".into(),
code: code.into(),
message: message.into(),
}
}
}
#[derive(Clone, Debug, Default, PartialEq)]
pub struct CellResult {
pub output: Option<Vec<Value>>,
pub is_error: bool,
pub details: Option<Value>,
pub usage: Option<Value>,
pub diagnostics: Vec<Diagnostic>,
pub control: Option<Control>,
}
impl CellResult {
pub fn text(text: impl Into<String>) -> CellResult {
CellResult {
output: Some(vec![json!({ "type": "text", "text": text.into() })]),
..CellResult::default()
}
}
pub fn error(code: &str, message: &str) -> CellResult {
CellResult {
output: Some(Vec::new()),
is_error: true,
diagnostics: vec![Diagnostic::error(code, message)],
..CellResult::default()
}
}
}
#[derive(Clone, Debug, PartialEq)]
pub struct CellCall {
pub conversation_id: ConversationId,
pub task_id: TaskId,
pub call_id: String,
pub name: String,
pub arguments: Value,
}
impl CellCall {
pub fn code(&self) -> &str {
self.arguments["code"].as_str().unwrap_or_default()
}
pub fn lane(&self) -> &str {
self.arguments["lane"].as_str().unwrap_or("main")
}
}
pub trait CellExecutor: Send + Sync {
fn tool(&self) -> Tool;
fn prepare(&self, arguments: &Value) -> std::result::Result<Value, String> {
if !arguments["code"].is_string() {
return Err("rlm needs a `code` string".into());
}
if arguments.get("lane").is_some_and(|lane| !lane.is_string()) {
return Err("rlm's `lane` must be a string".into());
}
Ok(arguments.clone())
}
fn replay(&self, call: &CellCall) -> Replay {
let _ = call;
Replay::Unsafe
}
fn execute(
&self,
call: &CellCall,
progress: &CellProgress,
signal: &Cancel,
) -> std::result::Result<CellResult, String>;
}
#[derive(Clone, Debug, Default, PartialEq)]
struct Retained {
text: String,
lines: usize,
dropped_bytes: usize,
dropped_lines: usize,
}
impl Retained {
fn push(&mut self, chunk: &str) {
if self.dropped_bytes > 0 {
self.dropped_bytes += chunk.len();
self.dropped_lines += chunk.matches('\n').count();
return;
}
let mut taken = 0;
for (index, character) in chunk.char_indices() {
let end = index + character.len_utf8();
let lines = self.lines + usize::from(character == '\n');
if self.text.len() + end > DEFAULT_MAX_BYTES || lines > DEFAULT_MAX_LINES {
break;
}
taken = end;
self.lines = lines;
}
self.text.push_str(&chunk[..taken]);
let rest = &chunk[taken..];
if !rest.is_empty() {
self.dropped_bytes += rest.len();
self.dropped_lines += rest.matches('\n').count().max(1);
}
}
}
type Published = (Retained, Option<Value>);
#[derive(Clone)]
pub struct CellProgress {
retained: Arc<Mutex<(Retained, Option<Value>)>>,
throttle: Arc<Mutex<Option<Throttle<Published>>>>,
}
impl CellProgress {
fn new(runtime: &TaskRuntime, interval: Duration) -> CellProgress {
let committer = runtime.clone();
let throttle = Throttle::new(interval, move |(retained, details): (Retained, Option<Value>)| {
let conversation_id = committer.conversation_id();
let task_id = committer.task_id();
let result = committer.commit(&committer.ctx(), |tx, _| {
let live = live(tx, conversation_id)?;
let Some(index) = slot_index(&live, task_id)? else {
return Ok(None);
};
let current = d(live.get(&slot_path(index, Some("output"))))?;
if current.as_ref().and_then(Value::as_str).unwrap_or_default() != retained.text {
d(live.set(&slot_path(index, Some("output")), json!(retained.text)))?;
}
if retained.dropped_bytes > 0 {
d(live.set(&slot_path(index, Some("droppedBytes")), json!(retained.dropped_bytes)))?;
d(live.set(&slot_path(index, Some("droppedLines")), json!(retained.dropped_lines)))?;
}
if let Some(details) = details {
d(live.set(&slot_path(index, Some("details")), details))?;
}
Ok(None)
});
if let Err(error) = result
&& !committer.signal().is_cancelled()
{
let _ = committer.report(&error);
}
});
CellProgress {
retained: Arc::new(Mutex::new((Retained::default(), None))),
throttle: Arc::new(Mutex::new(Some(throttle))),
}
}
pub fn detached() -> CellProgress {
CellProgress {
retained: Arc::new(Mutex::new((Retained::default(), None))),
throttle: Arc::new(Mutex::new(None)),
}
}
pub fn output(&self, chunk: &str) {
let snapshot = {
let mut state = lock(&self.retained);
state.0.push(chunk);
state.clone()
};
if let Some(throttle) = lock(&self.throttle).as_ref() {
throttle.mark(snapshot);
}
}
pub fn details(&self, value: Value) {
let snapshot = {
let mut state = lock(&self.retained);
state.1 = Some(strict(value));
state.clone()
};
if let Some(throttle) = lock(&self.throttle).as_ref() {
throttle.mark(snapshot);
}
}
fn stop(self) -> (Retained, Option<Value>) {
let throttle = lock(&self.throttle).take();
if let Some(throttle) = throttle {
throttle.stop();
}
lock(&self.retained).clone()
}
}
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
#[serde(tag = "phase", rename_all = "camelCase")]
pub(crate) enum CellCheckpoint {
Call,
Execute {
arguments: Value,
replay: Replay,
},
}
static CELL_DEFINITION: LazyLock<TaskDefinition> =
LazyLock::new(|| TaskDefinition::new(CELL_KIND, 2, |_| to_value(&CellCheckpoint::Call)));
pub(crate) fn cell_definition() -> &'static TaskDefinition {
&CELL_DEFINITION
}
pub(crate) fn cell_task(services: Arc<Services>) -> Task {
let (call, execute, abort) = (services.clone(), services.clone(), services);
Task::define(CELL_KIND, 2, |input| (CELL_DEFINITION.initial)(input))
.migrate(|input, checkpoint, _| {
let mut input = input.clone();
if input.get("kind").is_none() {
input["kind"] = json!("model");
}
Ok((input, checkpoint.clone()))
})
.phase("call", move |task, runtime| call_phase(&call, task, runtime))
.phase("execute", move |task, runtime| execute_phase(&execute, task, runtime))
.abort(move |task, runtime| abort_handler(&abort, task, runtime))
.build()
}
fn checkpoint_of(task: &TaskRecord) -> Result<CellCheckpoint> {
match &task.state {
TaskState::Pending { checkpoint }
| TaskState::Running { checkpoint }
| TaskState::Waiting { checkpoint, .. } => from_value(checkpoint.clone(), "cell checkpoint"),
_ => Err(HarnessError::msg("A terminal cell has no checkpoint")),
}
}
fn read_call(runtime: &TaskRuntime, task: &TaskRecord) -> Result<(String, String, Value)> {
let assistant = EntryId(task.input["assistant"].as_u64().unwrap_or_default());
let call_id = task.input["callId"].as_str().unwrap_or_default().to_owned();
let entry = runtime.read(|tx| Ok(tx.entry_of(entries::ASSISTANT, assistant)?))?;
let message = entry
.and_then(|entry| entry.model)
.and_then(|model| model.into_iter().next());
if let Some(message) = message
&& let Message::Assistant(message) = super::context::message_of(message)?
{
for content in message.content {
if let AssistantContent::ToolCall(call) = content
&& call.id == call_id
{
return Ok((call.id, call.name, call.arguments));
}
}
}
Err(HarnessError::Error(format!(
"Entry {} has no tool call {call_id}",
assistant.0
)))
}
enum Ending {
Completed,
Aborted,
Failed(String),
}
fn call_phase(services: &Services, task: &TaskRecord, runtime: &TaskRuntime) -> Result<()> {
let (call_id, name, arguments) = read_call(runtime, task)?;
let tool = services.cells.tool();
if name != tool.name {
let error = CellResult::error("tool_unavailable", &format!("Tool {name} is not available"));
return settle(
runtime,
task,
&call_id,
&name,
Ending::Completed,
|_| error.clone(),
None,
);
}
let arguments = match services.cells.prepare(&arguments) {
Ok(arguments) => strict(arguments),
Err(message) => {
let invalid = CellResult::error("invalid_arguments", &message);
return settle(
runtime,
task,
&call_id,
&name,
Ending::Completed,
|_| invalid.clone(),
None,
);
}
};
let call = CellCall {
conversation_id: runtime.conversation_id(),
task_id: runtime.task_id(),
call_id: call_id.clone(),
name: name.clone(),
arguments: arguments.clone(),
};
if let Some(gate) = &services.hooks.gate_tool {
let run = runtime.read(|tx| super::gate::gate_run(tx, call.conversation_id))?;
if let Some(block) = gate(&run, &call) {
let blocked = CellResult {
output: Some(vec![json!({ "type": "text", "text": block.reason })]),
is_error: true,
control: block.terminate.then_some(Control {
terminate: Some(true),
handoff: None,
}),
..CellResult::default()
};
return settle(
runtime,
task,
&call_id,
&name,
Ending::Completed,
|_| blocked.clone(),
None,
);
}
}
let replay = services.cells.replay(&call);
let conversation_id = runtime.conversation_id();
let task_id = runtime.task_id();
runtime.commit(&runtime.ctx(), |tx, _| {
let live = live(tx, conversation_id)?;
if let Some(index) = slot_index(&live, task_id)? {
d(live.set(&slot_path(index, Some("status")), json!("running")))?;
}
Ok(Some(NextState::Running {
checkpoint: to_value(&CellCheckpoint::Execute {
arguments: arguments.clone(),
replay,
}),
}))
})?;
run(services, runtime, task, call)
}
fn execute_phase(services: &Services, task: &TaskRecord, runtime: &TaskRuntime) -> Result<()> {
let CellCheckpoint::Execute { arguments, replay } = checkpoint_of(task)? else {
return Err(HarnessError::msg("execute phase without intent"));
};
let (call_id, name, _) = read_call(runtime, task)?;
let call = CellCall {
conversation_id: runtime.conversation_id(),
task_id: runtime.task_id(),
call_id: call_id.clone(),
name: name.clone(),
arguments,
};
if replay == Replay::Safe && name == services.cells.tool().name && services.cells.replay(&call) == Replay::Safe {
let conversation_id = runtime.conversation_id();
let task_id = runtime.task_id();
runtime.commit(&runtime.ctx(), |tx, _| {
let live = live(tx, conversation_id)?;
if let Some(index) = slot_index(&live, task_id)? {
clear_progress(&live, index)?;
}
Ok(None)
})?;
return run(services, runtime, task, call);
}
let message = interrupted_message(&name);
settle(
runtime,
task,
&call_id,
&name,
Ending::Failed(message.clone()),
|slot| from_slot(slot, "interrupted", &message),
None,
)
}
pub fn interrupted_message(name: &str) -> String {
format!(
"Tool {name} was interrupted by a restart; effect unknown: it may have partially run and was not replayed. \
Check what it changed before you run it again"
)
}
fn abort_handler(_services: &Services, task: &TaskRecord, runtime: &TaskRuntime) -> Result<()> {
let (call_id, name, _) = read_call(runtime, task)?;
let executing = matches!(checkpoint_of(task)?, CellCheckpoint::Execute { .. });
let (code, message) = if task.abort_reason.is_none() {
("aborted", format!("Tool {name} was aborted"))
} else if executing {
("interrupted", interrupted_message(&name))
} else {
(
"abandoned",
format!("Tool {name} was not started: its caller ended with a restart"),
)
};
settle(
runtime,
task,
&call_id,
&name,
Ending::Aborted,
|slot| from_slot(slot, code, &message),
None,
)
}
fn run(services: &Services, runtime: &TaskRuntime, task: &TaskRecord, call: CellCall) -> Result<()> {
let interval = Duration::from_millis((services.settings)().output_interval_ms);
let progress = CellProgress::new(runtime, interval);
let started = Instant::now();
let executed = services.cells.execute(&call, &progress, runtime.signal());
let duration = started.elapsed().as_millis() as u64;
let (retained, details) = progress.stop();
let (mut result, ending) = match executed {
Ok(result) => (result, Ending::Completed),
Err(message) => {
runtime.signal().check()?;
let result = CellResult {
output: None,
is_error: true,
diagnostics: vec![Diagnostic::error("tool_error", &message)],
..CellResult::default()
};
(result, Ending::Failed(format!("Tool {} threw", call.name)))
}
};
if result.output.is_none() {
result.output = Some(if retained.text.is_empty() {
Vec::new()
} else {
vec![json!({ "type": "text", "text": retained.text })]
});
if retained.dropped_bytes > 0 {
result.diagnostics.insert(0, truncated(&retained, Some("head")));
}
}
if result.details.is_none() {
result.details = details;
}
settle(
runtime,
task,
&call.call_id,
&call.name,
ending,
|_| result.clone(),
Some(duration),
)
}
fn truncated(retained: &Retained, retain: Option<&str>) -> Diagnostic {
let kept = match retain {
Some("head") => " to its beginning",
Some(_) => " to its end",
None => "",
};
Diagnostic::warn(
"truncated",
&format!(
"Output truncated{kept}: {} lines, {} bytes dropped",
retained.dropped_lines, retained.dropped_bytes
),
)
}
fn from_slot(slot: Option<&Value>, code: &str, message: &str) -> CellResult {
let mut diagnostics: Vec<Diagnostic> = slot
.and_then(|slot| slot.get("diagnostics"))
.and_then(|value| serde_json::from_value(value.clone()).ok())
.unwrap_or_default();
let dropped_bytes = slot.and_then(|slot| slot["droppedBytes"].as_u64()).unwrap_or(0) as usize;
if dropped_bytes > 0 {
let dropped_lines = slot.and_then(|slot| slot["droppedLines"].as_u64()).unwrap_or(0) as usize;
diagnostics.push(truncated(
&Retained {
dropped_bytes,
dropped_lines,
..Retained::default()
},
None,
));
}
diagnostics.push(Diagnostic::error(code, message));
let output = slot
.and_then(|slot| slot["output"].as_str())
.filter(|text| !text.is_empty())
.map(|text| vec![json!({ "type": "text", "text": text })])
.unwrap_or_default();
CellResult {
output: Some(output),
is_error: true,
details: slot.and_then(|slot| slot.get("details").cloned()),
diagnostics,
..CellResult::default()
}
}
fn settle(
runtime: &TaskRuntime,
_task: &TaskRecord,
call_id: &str,
name: &str,
ending: Ending,
build: impl FnOnce(Option<&Value>) -> CellResult,
duration_ms: Option<u64>,
) -> Result<()> {
let conversation_id = runtime.conversation_id();
let task_id = runtime.task_id();
runtime.commit(&runtime.ctx(), |tx, _| {
let live = live(tx, conversation_id)?;
let current = slot(&live, task_id)?;
let result = build(current.as_ref());
let entry = append_tool_result(tx, conversation_id, call_id, name, &result, runtime.now()?, duration_ms)?;
if let Some(index) = slot_index(&live, task_id)? {
finish_slot(&live, index)?;
d(live.set(&slot_path(index, Some("entry")), json!(entry.id.0)))?;
}
let mut settled = json!({ "kind": "model", "entryId": entry.id.0 });
if let (Ending::Completed, Some(control)) = (&ending, &result.control) {
settled["control"] = strict(to_value(control));
}
let outcome = match ending {
Ending::Completed => TaskOutcome::Completed { result: settled },
Ending::Aborted => TaskOutcome::Aborted {
reason: None,
result: Some(settled),
},
Ending::Failed(message) => TaskOutcome::Failed {
error: TaskOutcomeError { message, detail: None },
result: Some(settled),
},
};
Ok(Some(NextState::Terminal { outcome }))
})
}
fn render_diagnostics(diagnostics: &[Diagnostic]) -> String {
let lines: Vec<String> = diagnostics
.iter()
.map(|diagnostic| format!("[{}] {}", diagnostic.severity, diagnostic.message))
.collect();
format!("<harness>\n{}\n</harness>", lines.join("\n"))
}
pub(crate) fn append_tool_result(
tx: &mut Tx<'_>,
conversation_id: ConversationId,
call_id: &str,
name: &str,
result: &CellResult,
timestamp: u64,
duration_ms: Option<u64>,
) -> Result<EntryRecord> {
let mut content = result.output.clone().unwrap_or_default();
if !result.diagnostics.is_empty() {
content.push(json!({ "type": "text", "text": render_diagnostics(&result.diagnostics) }));
}
let mut message = json!({
"role": "toolResult",
"toolCallId": call_id,
"toolName": name,
"content": content,
});
if let Some(details) = &result.details {
message["details"] = details.clone();
}
if let Some(usage) = &result.usage {
message["usage"] = usage.clone();
}
message["isError"] = json!(result.is_error);
if let Some(duration) = duration_ms {
message["durationMs"] = json!(duration);
}
message["timestamp"] = json!(timestamp);
if let Some(usage) = &result.usage {
record_usage(tx, conversation_id, Bucket::Tools, name, usage)?;
}
Ok(tx.append_entry(
conversation_id,
EntryDraft {
model: Some(vec![message]),
data: Some(json!({ "diagnostics": result.diagnostics })),
..EntryDraft::new(entries::TOOL_RESULT)
},
)?)
}