use crate::durable_store::Observer;
use agent::{AgentEvent, AgentTool, AgentToolResult};
use ai::{AbortSignal, AssistantContent, AssistantMessage, AssistantMessageEvent, Message, Model, StopReason, Tool};
use durable::{ROOT_CONVERSATION_ID, StorageWrite};
use harness::chat::{CellCall, CellExecutor, CellProgress, CellResult, Models, StreamRequest};
use harness::scheduler::{Cancel, HarnessError};
use jsrt::sync;
use serde_json::{Value, json};
use std::collections::HashMap;
use std::sync::{Arc, Mutex};
pub struct GenerationStart {
pub messages: Vec<Message>,
}
pub struct Answer<T> {
value: Arc<Mutex<Option<T>>>,
ready: Cancel,
}
impl<T> Clone for Answer<T> {
fn clone(&self) -> Self {
Answer {
value: self.value.clone(),
ready: self.ready.clone(),
}
}
}
impl<T> Default for Answer<T> {
fn default() -> Self {
Answer {
value: Arc::new(Mutex::new(None)),
ready: Cancel::new(),
}
}
}
impl<T> Answer<T> {
pub fn give(&self, value: T) {
*sync::lock(&self.value) = Some(value);
self.ready.abort();
}
fn wait(&self, signal: &Cancel) -> Option<T> {
let linked = Cancel::any(&[signal, &self.ready]);
linked.signal().wait();
sync::lock(&self.value).take()
}
}
pub enum Request {
Generation {
users: Vec<Message>,
answer: Answer<Result<GenerationStart, StartError>>,
},
Event(Box<AgentEvent>),
Overflow {
message: Box<AssistantMessage>,
answer: Answer<Settled>,
},
RoundEnded {
calls: usize,
failed: usize,
answer: Answer<()>,
},
}
pub enum StartError {
Request(String),
Checkpoint(String),
}
pub enum Settled {
Answer(Box<AssistantMessage>),
Retry,
Failed(String),
}
pub type ToActor = Arc<dyn Fn(Request) + Send + Sync>;
#[derive(Clone)]
struct Reported {
name: String,
result: AgentToolResult,
is_error: bool,
}
#[derive(Default)]
struct Round {
calls: Vec<(String, String)>,
done: HashMap<String, Reported>,
}
pub struct Bridge {
to_actor: ToActor,
results: Mutex<HashMap<String, Reported>>,
round: Mutex<Round>,
user_abort: Mutex<bool>,
}
impl Bridge {
pub fn new(to_actor: ToActor) -> Arc<Bridge> {
Arc::new(Bridge {
to_actor,
results: Mutex::new(HashMap::new()),
round: Mutex::new(Round::default()),
user_abort: Mutex::new(false),
})
}
pub fn request(&self, request: Request) {
(self.to_actor)(request);
}
fn send(&self, event: AgentEvent) {
(self.to_actor)(Request::Event(Box::new(event)));
}
pub fn set_user_abort(&self, aborted: bool) {
*sync::lock(&self.user_abort) = aborted;
}
pub fn clear(&self) {
*sync::lock(&self.round) = Round::default();
sync::lock(&self.results).clear();
}
pub fn answer_committed(&self, message: AssistantMessage) {
let calls: Vec<(String, String, Value)> = message
.content
.iter()
.filter_map(|content| match content {
AssistantContent::ToolCall(call) => Some((call.id.clone(), call.name.clone(), call.arguments.clone())),
_ => None,
})
.collect();
let round = message.stop_reason == StopReason::ToolUse && !calls.is_empty();
self.send(AgentEvent::MessageEnd {
message: Message::Assistant(message),
});
if !round {
return;
}
*sync::lock(&self.round) = Round {
calls: calls.iter().map(|(id, name, _)| (id.clone(), name.clone())).collect(),
done: HashMap::new(),
};
for (id, name, args) in calls {
self.send(AgentEvent::ToolExecutionStart {
tool_call_id: id,
tool_name: name,
args,
});
}
}
fn round_complete(&self) -> Option<(usize, usize)> {
let round = sync::lock(&self.round);
let results = sync::lock(&self.results);
if round.calls.is_empty() {
return None;
}
let mut failed = 0;
for (id, _) in &round.calls {
let reported = results.get(id).or_else(|| round.done.get(id))?;
failed += usize::from(reported.is_error);
}
Some((round.calls.len(), failed))
}
pub fn resume_round(&self, calls: Vec<(String, String)>, done: Vec<(String, AgentToolResult, bool)>) {
let mut round = Round {
calls,
done: HashMap::new(),
};
for (id, result, is_error) in done {
let name = round
.calls
.iter()
.find(|(each, _)| *each == id)
.map(|(_, name)| name.clone())
.unwrap_or_default();
round.done.insert(id, Reported { name, result, is_error });
}
*sync::lock(&self.round) = round;
}
pub fn result_committed(&self, message: &Value) {
let call_id = message["toolCallId"].as_str().unwrap_or_default().to_string();
let reported = sync::lock(&self.results).remove(&call_id).unwrap_or_else(|| Reported {
name: message["toolName"].as_str().unwrap_or("rlm").to_string(),
result: AgentToolResult {
content: serde_json::from_value(message["content"].clone()).unwrap_or_default(),
details: message.get("details").cloned().unwrap_or_else(|| json!({})),
usage: serde_json::from_value(message["usage"].clone()).ok(),
terminate: None,
},
is_error: message["isError"] == true,
});
self.send(AgentEvent::ToolExecutionEnd {
tool_call_id: call_id.clone(),
tool_name: reported.name.clone(),
result: reported.result.clone(),
is_error: reported.is_error,
});
let finished = {
let mut round = sync::lock(&self.round);
round.done.insert(call_id, reported);
if round.calls.iter().all(|(id, _)| round.done.contains_key(id)) {
let round = std::mem::take(&mut *round);
Some(
round
.calls
.iter()
.filter_map(|(id, _)| round.done.get(id).map(|r| (id.clone(), r.clone())))
.collect::<Vec<_>>(),
)
} else {
None
}
};
for (id, reported) in finished.into_iter().flatten() {
let message = Message::ToolResult(ai::ToolResultMessage {
tool_call_id: id,
tool_name: reported.name,
content: reported.result.content,
details: Some(reported.result.details).filter(|d| !d.is_null()),
usage: reported.result.usage,
is_error: reported.is_error,
timestamp: jsrt::date::now_ms(),
});
self.send(AgentEvent::MessageStart {
message: message.clone(),
});
self.send(AgentEvent::MessageEnd { message });
}
}
}
pub const DURABLE_CRASH_ENV: &str = "__ULTRON_DURABLE_CRASH";
fn crash_point() -> Option<(String, std::path::PathBuf)> {
let spec = std::env::var(DURABLE_CRASH_ENV).ok()?;
let (kind, marker) = spec.split_once(':')?;
Some((kind.to_string(), std::path::PathBuf::from(marker)))
}
fn maybe_crash(point: &Option<(String, std::path::PathBuf)>, kind: &str) {
if let Some((wanted, marker)) = point
&& wanted == kind
&& std::fs::OpenOptions::new()
.write(true)
.create_new(true)
.open(marker)
.is_ok()
{
std::process::abort();
}
}
pub type Committed = Arc<dyn Fn(&durable::EntryRecord) + Send + Sync>;
pub fn observer(bridge: Arc<Bridge>, committed: Committed) -> Observer {
let crash = crash_point();
Arc::new(move |writes: &[StorageWrite]| {
for write in writes {
let StorageWrite::Entry { value } = write else { continue };
if value.conversation_id != ROOT_CONVERSATION_ID {
committed(value);
continue;
}
let Some(message) = value.model.as_ref().and_then(|model| model.first()) else {
continue;
};
match value.kind.as_str() {
harness::chat::entries::ASSISTANT => maybe_crash(&crash, "answer"),
harness::chat::entries::TOOL_RESULT => maybe_crash(&crash, "result"),
_ => {}
}
match value.kind.as_str() {
harness::chat::entries::ASSISTANT => {
if let Ok(message) = serde_json::from_value::<AssistantMessage>(message.clone()) {
bridge.answer_committed(message);
}
}
harness::chat::entries::TOOL_RESULT => bridge.result_committed(message),
_ => {}
}
}
})
}
struct Bridged {
abort: AbortSignal,
done: Cancel,
watcher: Option<std::thread::JoinHandle<()>>,
}
impl Bridged {
fn new(signal: &Cancel) -> Bridged {
let abort = AbortSignal::new();
let done = Cancel::new();
let (watched, finished, target) = (signal.clone(), done.clone(), abort.clone());
let watcher = std::thread::spawn(move || {
let linked = Cancel::any(&[&watched, &finished]);
linked.signal().wait();
if watched.is_cancelled() {
target.abort();
}
});
Bridged {
abort,
done,
watcher: Some(watcher),
}
}
}
impl Drop for Bridged {
fn drop(&mut self) {
self.done.abort();
if let Some(watcher) = self.watcher.take() {
let _ = watcher.join();
}
}
}
fn runtime() -> Result<tokio::runtime::Runtime, HarnessError> {
tokio::runtime::Builder::new_current_thread()
.enable_all()
.build()
.map_err(|e| HarnessError::Error(e.to_string()))
}
pub type ResolveModel = Arc<dyn Fn(&str, &str) -> Option<Model> + Send + Sync>;
pub type ProviderKey = Arc<dyn Fn(&str) -> Option<String> + Send + Sync>;
pub struct TurnModels {
pub bridge: Arc<Bridge>,
pub provider: Arc<dyn agent::StreamFn>,
pub resolve: ResolveModel,
pub api_key: ProviderKey,
pub root_session: Arc<std::sync::OnceLock<String>>,
pub charge: Option<ChargeTurn>,
pub transform_child: Option<TransformChild>,
pub trace: Option<(Arc<crate::request_trace::RequestTrace>, String)>,
}
pub type TransformChild = Arc<dyn Fn(durable::ConversationId, &mut Vec<Message>) + Send + Sync>;
pub type ChargeTurn = Arc<dyn Fn(durable::ConversationId, &AssistantMessage) + Send + Sync>;
impl TurnModels {
fn stream_child(
&self,
model: &Model,
context: &ai::TranscriptContext,
request: &StreamRequest,
signal: &Cancel,
on_event: &mut (dyn FnMut(&AssistantMessageEvent) + Send),
) -> harness::scheduler::Result<AssistantMessage> {
let bridged = Bridged::new(signal);
let mut options = request.simple_options();
options.base.api_key = (self.api_key)(&model.provider);
options.base.signal = Some(bridged.abort.clone());
let mut sink = |event: &AssistantMessageEvent| on_event(event);
let transformed;
let context = match (&self.transform_child, request.conversation_id, &request.final_note) {
(None, _, None) | (_, None, None) => context,
(transform, conversation, note) => {
let mut messages = context.messages.clone();
if let (Some(transform), Some(conversation)) = (transform, conversation) {
transform(conversation, &mut messages);
}
if let Some(note) = note {
harness::chat::gate::apply_final(note, &mut messages, &mut options, jsrt::date::now_ms());
}
transformed = ai::TranscriptContext { messages };
&transformed
}
};
let message = runtime()?.block_on(self.provider.stream(model, context, &options, &mut sink));
drop(bridged);
if let Some((trace, session)) = &self.trace {
let lane = request
.conversation_id
.map_or_else(|| "child".to_string(), |id| format!("conversation:{}", id.0));
trace.record(session, &lane, model, context, &message);
}
signal.check()?;
if let (Some(charge), Some(conversation)) = (&self.charge, request.conversation_id) {
charge(conversation, &message);
}
Ok(message)
}
}
fn trailing_users(messages: &[Message]) -> Vec<Message> {
let start = messages
.iter()
.rposition(|m| matches!(m, Message::Assistant(_) | Message::ToolResult(_)))
.map_or(0, |i| i + 1);
messages[start..]
.iter()
.filter(|m| matches!(m, Message::User(_)))
.cloned()
.collect()
}
impl Models for TurnModels {
fn get_model(&self, provider: &str, model_id: &str) -> Option<Model> {
(self.resolve)(provider, model_id)
}
fn stream(
&self,
model: &Model,
context: &ai::TranscriptContext,
request: &StreamRequest,
signal: &Cancel,
on_event: &mut (dyn FnMut(&AssistantMessageEvent) + Send),
) -> harness::scheduler::Result<AssistantMessage> {
if self.root_session.get().map(String::as_str) != request.session_id.as_deref() {
return self.stream_child(model, context, request, signal, on_event);
}
loop {
let message = self.stream_root(model, context, request, signal, on_event)?;
if !crate::auto_compaction::overflowed(&message, model.context_window, model.max_tokens) {
return Ok(message);
}
let answer = Answer::default();
(self.bridge.to_actor)(Request::Overflow {
message: Box::new(message),
answer: answer.clone(),
});
match answer.wait(signal) {
Some(Settled::Retry) => continue,
Some(Settled::Answer(message)) => return Ok(*message),
Some(Settled::Failed(error)) => return Err(HarnessError::Error(error)),
None => return Err(signal.reason().unwrap_or_else(|| HarnessError::msg("cancelled"))),
}
}
}
}
impl TurnModels {
fn stream_root(
&self,
model: &Model,
context: &ai::TranscriptContext,
request: &StreamRequest,
signal: &Cancel,
on_event: &mut (dyn FnMut(&AssistantMessageEvent) + Send),
) -> harness::scheduler::Result<AssistantMessage> {
let answer = Answer::default();
(self.bridge.to_actor)(Request::Generation {
users: trailing_users(&context.messages),
answer: answer.clone(),
});
let start = match answer.wait(signal) {
Some(Ok(start)) => start,
Some(Err(StartError::Request(error))) => return Ok(crate::lane::error_message(model, error)),
Some(Err(StartError::Checkpoint(error))) => return Err(HarnessError::Error(error)),
None => return Err(signal.reason().unwrap_or_else(|| HarnessError::msg("cancelled"))),
};
let bridged = Bridged::new(signal);
let mut options = request.simple_options();
options.base.api_key = (self.api_key)(&model.provider);
options.base.signal = Some(bridged.abort.clone());
let mut messages = start.messages;
if let Some(note) = &request.final_note {
harness::chat::gate::apply_final(note, &mut messages, &mut options, jsrt::date::now_ms());
}
let llm = ai::TranscriptContext { messages };
let bridge = self.bridge.clone();
let mut started = false;
let mut sink = |event: &AssistantMessageEvent| {
on_event(event);
match event {
AssistantMessageEvent::Start { partial } => {
started = true;
bridge.send(AgentEvent::MessageStart {
message: Message::Assistant(partial.clone()),
});
}
AssistantMessageEvent::Done { .. } | AssistantMessageEvent::Error { .. } => {}
AssistantMessageEvent::TextStart { partial, .. }
| AssistantMessageEvent::TextDelta { partial, .. }
| AssistantMessageEvent::TextEnd { partial, .. }
| AssistantMessageEvent::ThinkingStart { partial, .. }
| AssistantMessageEvent::ThinkingDelta { partial, .. }
| AssistantMessageEvent::ThinkingEnd { partial, .. }
| AssistantMessageEvent::ToolCallStart { partial, .. }
| AssistantMessageEvent::ToolCallDelta { partial, .. }
| AssistantMessageEvent::ToolCallEnd { partial, .. } => {
if started {
bridge.send(AgentEvent::MessageUpdate {
message: Message::Assistant(partial.clone()),
assistant_message_event: event.clone(),
});
}
}
}
};
let message = runtime()?.block_on(self.provider.stream(model, &llm, &options, &mut sink));
drop(bridged);
if let Some((trace, session)) = &self.trace {
trace.record(session, session_journal::run::MAIN_LANE, model, &llm, &message);
}
if (!signal.is_cancelled() || *sync::lock(&self.bridge.user_abort))
&& let Some(charge) = &self.charge
{
charge(ROOT_CONVERSATION_ID, &message);
}
if !started && message.stop_reason != StopReason::Error {
self.bridge.send(AgentEvent::MessageStart {
message: Message::Assistant(message.clone()),
});
}
if signal.is_cancelled() {
if *sync::lock(&self.bridge.user_abort) {
self.bridge.send(AgentEvent::MessageEnd {
message: Message::Assistant(message.clone()),
});
}
return Err(signal.reason().unwrap_or_else(|| HarnessError::msg("cancelled")));
}
Ok(message)
}
}
pub const CELL_OUTPUT: &str = "cell.output";
pub fn publish_output(progress: &Mutex<Option<CellProgress>>, text: &str, mask: &crate::async_events::MaskText) {
if text.is_empty() {
return;
}
let masked = mask(text);
if let Some(progress) = sync::lock(progress).as_ref() {
progress.output(&masked);
}
}
pub struct TurnCells {
pub bridge: Arc<Bridge>,
pub tool: Arc<dyn AgentTool>,
pub progress: Arc<Mutex<Option<CellProgress>>>,
pub stack: Arc<crate::compose::Stack>,
pub children: Arc<crate::subagents::DurableChildren>,
}
fn run_tool(tool: &dyn AgentTool, call: &CellCall, abort: &AbortSignal) -> Result<(AgentToolResult, bool), String> {
let update = |_: AgentToolResult| {};
let executed = runtime().map_err(|e| e.to_string())?.block_on(tool.execute(
&call.call_id,
call.arguments.clone(),
abort,
&update,
));
Ok(match executed {
Ok(result) => (result, false),
Err(message) => (
AgentToolResult {
content: vec![crate::lane::text_block(message)],
details: json!({}),
usage: None,
terminate: None,
},
true,
),
})
}
fn cell_result(result: AgentToolResult, is_error: bool) -> CellResult {
CellResult {
output: Some(
result
.content
.iter()
.filter_map(|c| serde_json::to_value(c).ok())
.collect(),
),
is_error,
details: Some(result.details).filter(|d| !d.is_null()),
usage: result.usage.and_then(|u| serde_json::to_value(u).ok()),
diagnostics: Vec::new(),
control: None,
}
}
impl CellExecutor for TurnCells {
fn tool(&self) -> Tool {
self.tool.tool().clone()
}
fn prepare(&self, arguments: &Value) -> Result<Value, String> {
let tool = self.tool.tool();
jsonschema::validate_tool_arguments(&tool.name, &tool.parameters, arguments)
}
fn execute(&self, call: &CellCall, progress: &CellProgress, signal: &Cancel) -> Result<CellResult, String> {
if call.conversation_id != ROOT_CONVERSATION_ID {
let lane = self
.children
.lane_of(call.conversation_id)
.ok_or("This conversation has no REPL lane")?;
let tool = self.stack.rlm(&lane)?;
let bridged = Bridged::new(signal);
let (result, is_error) = run_tool(&tool, call, &bridged.abort)?;
drop(bridged);
if signal.is_cancelled() {
return Err(signal
.reason()
.map_or_else(|| "cancelled".to_string(), |r| r.to_string()));
}
return Ok(cell_result(result, is_error));
}
let bridged = Bridged::new(signal);
*sync::lock(&self.progress) = Some(progress.clone());
let context = self.stack.context.get().cloned();
if let Some(context) = &context {
context.tool_start(session_journal::run::MAIN_LANE, &call.name, &call.call_id);
}
let executed = run_tool(self.tool.as_ref(), call, &bridged.abort);
if let Some(context) = &context {
context.tool_end(session_journal::run::MAIN_LANE, &call.call_id);
}
*sync::lock(&self.progress) = None;
drop(bridged);
let (result, is_error) = executed?;
sync::lock(&self.bridge.results).insert(
call.call_id.clone(),
Reported {
name: call.name.clone(),
result: result.clone(),
is_error,
},
);
if signal.is_cancelled() {
return Err(signal
.reason()
.map_or_else(|| "cancelled".to_string(), |r| r.to_string()));
}
if let Some((calls, failed)) = self.bridge.round_complete() {
let answer = Answer::default();
self.bridge.request(Request::RoundEnded {
calls,
failed,
answer: answer.clone(),
});
let _ = answer.wait(signal);
}
Ok(cell_result(result, is_error))
}
}