1use crate::durable_store::Observer;
19use agent::{AgentEvent, AgentTool, AgentToolResult};
20use ai::{AbortSignal, AssistantContent, AssistantMessage, AssistantMessageEvent, Message, Model, StopReason, Tool};
21use durable::{ROOT_CONVERSATION_ID, StorageWrite};
22use harness::chat::{CellCall, CellExecutor, CellProgress, CellResult, Models, StreamRequest};
23use harness::scheduler::{Cancel, HarnessError};
24use jsrt::sync;
25use serde_json::{Value, json};
26use std::collections::HashMap;
27use std::sync::{Arc, Mutex};
28
29pub struct GenerationStart {
32 pub messages: Vec<Message>,
33}
34
35pub struct Answer<T> {
37 value: Arc<Mutex<Option<T>>>,
38 ready: Cancel,
39}
40
41impl<T> Clone for Answer<T> {
42 fn clone(&self) -> Self {
43 Answer {
44 value: self.value.clone(),
45 ready: self.ready.clone(),
46 }
47 }
48}
49
50impl<T> Default for Answer<T> {
51 fn default() -> Self {
52 Answer {
53 value: Arc::new(Mutex::new(None)),
54 ready: Cancel::new(),
55 }
56 }
57}
58
59impl<T> Answer<T> {
60 pub fn give(&self, value: T) {
61 *sync::lock(&self.value) = Some(value);
62 self.ready.abort();
63 }
64
65 fn wait(&self, signal: &Cancel) -> Option<T> {
67 let linked = Cancel::any(&[signal, &self.ready]);
68 linked.signal().wait();
69 sync::lock(&self.value).take()
70 }
71}
72
73pub enum Request {
75 Generation {
77 users: Vec<Message>,
78 answer: Answer<Result<GenerationStart, StartError>>,
79 },
80 Event(Box<AgentEvent>),
82 Overflow {
85 message: Box<AssistantMessage>,
86 answer: Answer<Settled>,
87 },
88 RoundEnded {
91 calls: usize,
92 failed: usize,
93 answer: Answer<()>,
94 },
95}
96
97pub enum StartError {
99 Request(String),
101 Checkpoint(String),
103}
104
105pub enum Settled {
107 Answer(Box<AssistantMessage>),
109 Retry,
111 Failed(String),
113}
114
115pub type ToActor = Arc<dyn Fn(Request) + Send + Sync>;
117
118#[derive(Clone)]
120struct Reported {
121 name: String,
122 result: AgentToolResult,
123 is_error: bool,
124}
125
126#[derive(Default)]
127struct Round {
128 calls: Vec<(String, String)>,
130 done: HashMap<String, Reported>,
131}
132
133pub struct Bridge {
135 to_actor: ToActor,
136 results: Mutex<HashMap<String, Reported>>,
138 round: Mutex<Round>,
139 user_abort: Mutex<bool>,
142}
143
144impl Bridge {
145 pub fn new(to_actor: ToActor) -> Arc<Bridge> {
146 Arc::new(Bridge {
147 to_actor,
148 results: Mutex::new(HashMap::new()),
149 round: Mutex::new(Round::default()),
150 user_abort: Mutex::new(false),
151 })
152 }
153
154 pub fn request(&self, request: Request) {
156 (self.to_actor)(request);
157 }
158
159 fn send(&self, event: AgentEvent) {
160 (self.to_actor)(Request::Event(Box::new(event)));
161 }
162
163 pub fn set_user_abort(&self, aborted: bool) {
164 *sync::lock(&self.user_abort) = aborted;
165 }
166
167 pub fn clear(&self) {
169 *sync::lock(&self.round) = Round::default();
170 sync::lock(&self.results).clear();
171 }
172
173 pub fn answer_committed(&self, message: AssistantMessage) {
176 let calls: Vec<(String, String, Value)> = message
177 .content
178 .iter()
179 .filter_map(|content| match content {
180 AssistantContent::ToolCall(call) => Some((call.id.clone(), call.name.clone(), call.arguments.clone())),
181 _ => None,
182 })
183 .collect();
184 let round = message.stop_reason == StopReason::ToolUse && !calls.is_empty();
185 self.send(AgentEvent::MessageEnd {
186 message: Message::Assistant(message),
187 });
188 if !round {
189 return;
190 }
191 *sync::lock(&self.round) = Round {
192 calls: calls.iter().map(|(id, name, _)| (id.clone(), name.clone())).collect(),
193 done: HashMap::new(),
194 };
195 for (id, name, args) in calls {
196 self.send(AgentEvent::ToolExecutionStart {
197 tool_call_id: id,
198 tool_name: name,
199 args,
200 });
201 }
202 }
203
204 fn round_complete(&self) -> Option<(usize, usize)> {
206 let round = sync::lock(&self.round);
207 let results = sync::lock(&self.results);
208 if round.calls.is_empty() {
209 return None;
210 }
211 let mut failed = 0;
212 for (id, _) in &round.calls {
213 let reported = results.get(id).or_else(|| round.done.get(id))?;
214 failed += usize::from(reported.is_error);
215 }
216 Some((round.calls.len(), failed))
217 }
218
219 pub fn resume_round(&self, calls: Vec<(String, String)>, done: Vec<(String, AgentToolResult, bool)>) {
221 let mut round = Round {
222 calls,
223 done: HashMap::new(),
224 };
225 for (id, result, is_error) in done {
226 let name = round
227 .calls
228 .iter()
229 .find(|(each, _)| *each == id)
230 .map(|(_, name)| name.clone())
231 .unwrap_or_default();
232 round.done.insert(id, Reported { name, result, is_error });
233 }
234 *sync::lock(&self.round) = round;
235 }
236
237 pub fn result_committed(&self, message: &Value) {
240 let call_id = message["toolCallId"].as_str().unwrap_or_default().to_string();
241 let reported = sync::lock(&self.results).remove(&call_id).unwrap_or_else(|| Reported {
242 name: message["toolName"].as_str().unwrap_or("rlm").to_string(),
243 result: AgentToolResult {
244 content: serde_json::from_value(message["content"].clone()).unwrap_or_default(),
245 details: message.get("details").cloned().unwrap_or_else(|| json!({})),
246 usage: serde_json::from_value(message["usage"].clone()).ok(),
247 terminate: None,
248 },
249 is_error: message["isError"] == true,
250 });
251 self.send(AgentEvent::ToolExecutionEnd {
252 tool_call_id: call_id.clone(),
253 tool_name: reported.name.clone(),
254 result: reported.result.clone(),
255 is_error: reported.is_error,
256 });
257 let finished = {
258 let mut round = sync::lock(&self.round);
259 round.done.insert(call_id, reported);
260 if round.calls.iter().all(|(id, _)| round.done.contains_key(id)) {
261 let round = std::mem::take(&mut *round);
262 Some(
263 round
264 .calls
265 .iter()
266 .filter_map(|(id, _)| round.done.get(id).map(|r| (id.clone(), r.clone())))
267 .collect::<Vec<_>>(),
268 )
269 } else {
270 None
271 }
272 };
273 for (id, reported) in finished.into_iter().flatten() {
274 let message = Message::ToolResult(ai::ToolResultMessage {
275 tool_call_id: id,
276 tool_name: reported.name,
277 content: reported.result.content,
278 details: Some(reported.result.details).filter(|d| !d.is_null()),
279 usage: reported.result.usage,
280 is_error: reported.is_error,
281 timestamp: jsrt::date::now_ms(),
282 });
283 self.send(AgentEvent::MessageStart {
284 message: message.clone(),
285 });
286 self.send(AgentEvent::MessageEnd { message });
287 }
288 }
289}
290
291pub const DURABLE_CRASH_ENV: &str = "__ULTRON_DURABLE_CRASH";
295
296fn crash_point() -> Option<(String, std::path::PathBuf)> {
297 let spec = std::env::var(DURABLE_CRASH_ENV).ok()?;
298 let (kind, marker) = spec.split_once(':')?;
299 Some((kind.to_string(), std::path::PathBuf::from(marker)))
300}
301
302fn maybe_crash(point: &Option<(String, std::path::PathBuf)>, kind: &str) {
303 if let Some((wanted, marker)) = point
304 && wanted == kind
305 && std::fs::OpenOptions::new()
306 .write(true)
307 .create_new(true)
308 .open(marker)
309 .is_ok()
310 {
311 std::process::abort();
312 }
313}
314
315pub type Committed = Arc<dyn Fn(&durable::EntryRecord) + Send + Sync>;
317
318pub fn observer(bridge: Arc<Bridge>, committed: Committed) -> Observer {
321 let crash = crash_point();
322 Arc::new(move |writes: &[StorageWrite]| {
323 for write in writes {
324 let StorageWrite::Entry { value } = write else { continue };
325 if value.conversation_id != ROOT_CONVERSATION_ID {
326 committed(value);
327 continue;
328 }
329 let Some(message) = value.model.as_ref().and_then(|model| model.first()) else {
330 continue;
331 };
332 match value.kind.as_str() {
333 harness::chat::entries::ASSISTANT => maybe_crash(&crash, "answer"),
334 harness::chat::entries::TOOL_RESULT => maybe_crash(&crash, "result"),
335 _ => {}
336 }
337 match value.kind.as_str() {
338 harness::chat::entries::ASSISTANT => {
339 if let Ok(message) = serde_json::from_value::<AssistantMessage>(message.clone()) {
340 bridge.answer_committed(message);
341 }
342 }
343 harness::chat::entries::TOOL_RESULT => bridge.result_committed(message),
344 _ => {}
345 }
346 }
347 })
348}
349
350struct Bridged {
352 abort: AbortSignal,
353 done: Cancel,
354 watcher: Option<std::thread::JoinHandle<()>>,
355}
356
357impl Bridged {
358 fn new(signal: &Cancel) -> Bridged {
359 let abort = AbortSignal::new();
360 let done = Cancel::new();
361 let (watched, finished, target) = (signal.clone(), done.clone(), abort.clone());
362 let watcher = std::thread::spawn(move || {
363 let linked = Cancel::any(&[&watched, &finished]);
364 linked.signal().wait();
365 if watched.is_cancelled() {
366 target.abort();
367 }
368 });
369 Bridged {
370 abort,
371 done,
372 watcher: Some(watcher),
373 }
374 }
375}
376
377impl Drop for Bridged {
378 fn drop(&mut self) {
379 self.done.abort();
380 if let Some(watcher) = self.watcher.take() {
381 let _ = watcher.join();
382 }
383 }
384}
385
386fn runtime() -> Result<tokio::runtime::Runtime, HarnessError> {
387 tokio::runtime::Builder::new_current_thread()
388 .enable_all()
389 .build()
390 .map_err(|e| HarnessError::Error(e.to_string()))
391}
392
393pub type ResolveModel = Arc<dyn Fn(&str, &str) -> Option<Model> + Send + Sync>;
395pub type ProviderKey = Arc<dyn Fn(&str) -> Option<String> + Send + Sync>;
396
397pub struct TurnModels {
399 pub bridge: Arc<Bridge>,
400 pub provider: Arc<dyn agent::StreamFn>,
401 pub resolve: ResolveModel,
402 pub api_key: ProviderKey,
405 pub root_session: Arc<std::sync::OnceLock<String>>,
408 pub charge: Option<ChargeTurn>,
411 pub transform_child: Option<TransformChild>,
414 pub trace: Option<(Arc<crate::request_trace::RequestTrace>, String)>,
416}
417
418pub type TransformChild = Arc<dyn Fn(durable::ConversationId, &mut Vec<Message>) + Send + Sync>;
420
421pub type ChargeTurn = Arc<dyn Fn(durable::ConversationId, &AssistantMessage) + Send + Sync>;
423
424impl TurnModels {
425 fn stream_child(
427 &self,
428 model: &Model,
429 context: &ai::TranscriptContext,
430 request: &StreamRequest,
431 signal: &Cancel,
432 on_event: &mut (dyn FnMut(&AssistantMessageEvent) + Send),
433 ) -> harness::scheduler::Result<AssistantMessage> {
434 let bridged = Bridged::new(signal);
435 let mut options = request.simple_options();
438 options.base.api_key = (self.api_key)(&model.provider);
439 options.base.signal = Some(bridged.abort.clone());
440 let mut sink = |event: &AssistantMessageEvent| on_event(event);
441 let transformed;
442 let context = match (&self.transform_child, request.conversation_id, &request.final_note) {
443 (None, _, None) | (_, None, None) => context,
444 (transform, conversation, note) => {
445 let mut messages = context.messages.clone();
446 if let (Some(transform), Some(conversation)) = (transform, conversation) {
447 transform(conversation, &mut messages);
448 }
449 if let Some(note) = note {
450 harness::chat::gate::apply_final(note, &mut messages, &mut options, jsrt::date::now_ms());
451 }
452 transformed = ai::TranscriptContext { messages };
453 &transformed
454 }
455 };
456 let message = runtime()?.block_on(self.provider.stream(model, context, &options, &mut sink));
457 drop(bridged);
458 if let Some((trace, session)) = &self.trace {
459 let lane = request
460 .conversation_id
461 .map_or_else(|| "child".to_string(), |id| format!("conversation:{}", id.0));
462 trace.record(session, &lane, model, context, &message);
463 }
464 signal.check()?;
465 if let (Some(charge), Some(conversation)) = (&self.charge, request.conversation_id) {
466 charge(conversation, &message);
467 }
468 Ok(message)
469 }
470}
471
472fn trailing_users(messages: &[Message]) -> Vec<Message> {
474 let start = messages
475 .iter()
476 .rposition(|m| matches!(m, Message::Assistant(_) | Message::ToolResult(_)))
477 .map_or(0, |i| i + 1);
478 messages[start..]
479 .iter()
480 .filter(|m| matches!(m, Message::User(_)))
481 .cloned()
482 .collect()
483}
484
485impl Models for TurnModels {
486 fn get_model(&self, provider: &str, model_id: &str) -> Option<Model> {
487 (self.resolve)(provider, model_id)
488 }
489
490 fn stream(
491 &self,
492 model: &Model,
493 context: &ai::TranscriptContext,
494 request: &StreamRequest,
495 signal: &Cancel,
496 on_event: &mut (dyn FnMut(&AssistantMessageEvent) + Send),
497 ) -> harness::scheduler::Result<AssistantMessage> {
498 if self.root_session.get().map(String::as_str) != request.session_id.as_deref() {
499 return self.stream_child(model, context, request, signal, on_event);
500 }
501 loop {
502 let message = self.stream_root(model, context, request, signal, on_event)?;
503 if !crate::auto_compaction::overflowed(&message, model.context_window, model.max_tokens) {
505 return Ok(message);
506 }
507 let answer = Answer::default();
508 (self.bridge.to_actor)(Request::Overflow {
509 message: Box::new(message),
510 answer: answer.clone(),
511 });
512 match answer.wait(signal) {
513 Some(Settled::Retry) => continue,
514 Some(Settled::Answer(message)) => return Ok(*message),
515 Some(Settled::Failed(error)) => return Err(HarnessError::Error(error)),
516 None => return Err(signal.reason().unwrap_or_else(|| HarnessError::msg("cancelled"))),
517 }
518 }
519 }
520}
521
522impl TurnModels {
523 fn stream_root(
525 &self,
526 model: &Model,
527 context: &ai::TranscriptContext,
528 request: &StreamRequest,
529 signal: &Cancel,
530 on_event: &mut (dyn FnMut(&AssistantMessageEvent) + Send),
531 ) -> harness::scheduler::Result<AssistantMessage> {
532 let answer = Answer::default();
533 (self.bridge.to_actor)(Request::Generation {
534 users: trailing_users(&context.messages),
535 answer: answer.clone(),
536 });
537 let start = match answer.wait(signal) {
538 Some(Ok(start)) => start,
539 Some(Err(StartError::Request(error))) => return Ok(crate::lane::error_message(model, error)),
540 Some(Err(StartError::Checkpoint(error))) => return Err(HarnessError::Error(error)),
541 None => return Err(signal.reason().unwrap_or_else(|| HarnessError::msg("cancelled"))),
542 };
543 let bridged = Bridged::new(signal);
544 let mut options = request.simple_options();
545 options.base.api_key = (self.api_key)(&model.provider);
546 options.base.signal = Some(bridged.abort.clone());
547 let mut messages = start.messages;
548 if let Some(note) = &request.final_note {
550 harness::chat::gate::apply_final(note, &mut messages, &mut options, jsrt::date::now_ms());
551 }
552 let llm = ai::TranscriptContext { messages };
553 let bridge = self.bridge.clone();
554 let mut started = false;
555 let mut sink = |event: &AssistantMessageEvent| {
556 on_event(event);
557 match event {
558 AssistantMessageEvent::Start { partial } => {
559 started = true;
560 bridge.send(AgentEvent::MessageStart {
561 message: Message::Assistant(partial.clone()),
562 });
563 }
564 AssistantMessageEvent::Done { .. } | AssistantMessageEvent::Error { .. } => {}
565 AssistantMessageEvent::TextStart { partial, .. }
566 | AssistantMessageEvent::TextDelta { partial, .. }
567 | AssistantMessageEvent::TextEnd { partial, .. }
568 | AssistantMessageEvent::ThinkingStart { partial, .. }
569 | AssistantMessageEvent::ThinkingDelta { partial, .. }
570 | AssistantMessageEvent::ThinkingEnd { partial, .. }
571 | AssistantMessageEvent::ToolCallStart { partial, .. }
572 | AssistantMessageEvent::ToolCallDelta { partial, .. }
573 | AssistantMessageEvent::ToolCallEnd { partial, .. } => {
574 if started {
575 bridge.send(AgentEvent::MessageUpdate {
576 message: Message::Assistant(partial.clone()),
577 assistant_message_event: event.clone(),
578 });
579 }
580 }
581 }
582 };
583 let message = runtime()?.block_on(self.provider.stream(model, &llm, &options, &mut sink));
584 drop(bridged);
585 if let Some((trace, session)) = &self.trace {
586 trace.record(session, session_journal::run::MAIN_LANE, model, &llm, &message);
587 }
588 if (!signal.is_cancelled() || *sync::lock(&self.bridge.user_abort))
592 && let Some(charge) = &self.charge
593 {
594 charge(ROOT_CONVERSATION_ID, &message);
595 }
596 if !started && message.stop_reason != StopReason::Error {
599 self.bridge.send(AgentEvent::MessageStart {
600 message: Message::Assistant(message.clone()),
601 });
602 }
603 if signal.is_cancelled() {
604 if *sync::lock(&self.bridge.user_abort) {
607 self.bridge.send(AgentEvent::MessageEnd {
608 message: Message::Assistant(message.clone()),
609 });
610 }
611 return Err(signal.reason().unwrap_or_else(|| HarnessError::msg("cancelled")));
612 }
613 Ok(message)
614 }
615}
616
617pub const CELL_OUTPUT: &str = "cell.output";
620
621pub fn publish_output(progress: &Mutex<Option<CellProgress>>, text: &str, mask: &crate::async_events::MaskText) {
624 if text.is_empty() {
625 return;
626 }
627 let masked = mask(text);
628 if let Some(progress) = sync::lock(progress).as_ref() {
629 progress.output(&masked);
630 }
631}
632
633pub struct TurnCells {
635 pub bridge: Arc<Bridge>,
636 pub tool: Arc<dyn AgentTool>,
637 pub progress: Arc<Mutex<Option<CellProgress>>>,
639 pub stack: Arc<crate::compose::Stack>,
640 pub children: Arc<crate::subagents::DurableChildren>,
641}
642
643fn run_tool(tool: &dyn AgentTool, call: &CellCall, abort: &AbortSignal) -> Result<(AgentToolResult, bool), String> {
645 let update = |_: AgentToolResult| {};
646 let executed = runtime().map_err(|e| e.to_string())?.block_on(tool.execute(
647 &call.call_id,
648 call.arguments.clone(),
649 abort,
650 &update,
651 ));
652 Ok(match executed {
653 Ok(result) => (result, false),
654 Err(message) => (
655 AgentToolResult {
656 content: vec![crate::lane::text_block(message)],
657 details: json!({}),
658 usage: None,
659 terminate: None,
660 },
661 true,
662 ),
663 })
664}
665
666fn cell_result(result: AgentToolResult, is_error: bool) -> CellResult {
667 CellResult {
668 output: Some(
669 result
670 .content
671 .iter()
672 .filter_map(|c| serde_json::to_value(c).ok())
673 .collect(),
674 ),
675 is_error,
676 details: Some(result.details).filter(|d| !d.is_null()),
677 usage: result.usage.and_then(|u| serde_json::to_value(u).ok()),
678 diagnostics: Vec::new(),
679 control: None,
680 }
681}
682
683impl CellExecutor for TurnCells {
684 fn tool(&self) -> Tool {
685 self.tool.tool().clone()
686 }
687
688 fn prepare(&self, arguments: &Value) -> Result<Value, String> {
689 let tool = self.tool.tool();
692 jsonschema::validate_tool_arguments(&tool.name, &tool.parameters, arguments)
693 }
694
695 fn execute(&self, call: &CellCall, progress: &CellProgress, signal: &Cancel) -> Result<CellResult, String> {
696 if call.conversation_id != ROOT_CONVERSATION_ID {
697 let lane = self
699 .children
700 .lane_of(call.conversation_id)
701 .ok_or("This conversation has no REPL lane")?;
702 let tool = self.stack.rlm(&lane)?;
703 let bridged = Bridged::new(signal);
704 let (result, is_error) = run_tool(&tool, call, &bridged.abort)?;
705 drop(bridged);
706 if signal.is_cancelled() {
707 return Err(signal
708 .reason()
709 .map_or_else(|| "cancelled".to_string(), |r| r.to_string()));
710 }
711 return Ok(cell_result(result, is_error));
712 }
713 let bridged = Bridged::new(signal);
714 *sync::lock(&self.progress) = Some(progress.clone());
715 let context = self.stack.context.get().cloned();
718 if let Some(context) = &context {
719 context.tool_start(session_journal::run::MAIN_LANE, &call.name, &call.call_id);
720 }
721 let executed = run_tool(self.tool.as_ref(), call, &bridged.abort);
722 if let Some(context) = &context {
723 context.tool_end(session_journal::run::MAIN_LANE, &call.call_id);
724 }
725 *sync::lock(&self.progress) = None;
726 drop(bridged);
727 let (result, is_error) = executed?;
728 sync::lock(&self.bridge.results).insert(
729 call.call_id.clone(),
730 Reported {
731 name: call.name.clone(),
732 result: result.clone(),
733 is_error,
734 },
735 );
736 if signal.is_cancelled() {
737 return Err(signal
738 .reason()
739 .map_or_else(|| "cancelled".to_string(), |r| r.to_string()));
740 }
741 if let Some((calls, failed)) = self.bridge.round_complete() {
743 let answer = Answer::default();
744 self.bridge.request(Request::RoundEnded {
745 calls,
746 failed,
747 answer: answer.clone(),
748 });
749 let _ = answer.wait(signal);
750 }
751 Ok(cell_result(result, is_error))
752 }
753}