1use std::sync::{Arc, LazyLock, Mutex};
12use std::time::{Duration, Instant};
13
14use ai::{AssistantContent, Message, Tool};
15use durable::{
16 ConversationId, EntryDraft, EntryId, EntryRecord, TaskDefinition, TaskId, TaskOutcome, TaskOutcomeError,
17 TaskRecord, TaskState, Tx,
18};
19use serde::{Deserialize, Serialize};
20use serde_json::{Value, json};
21
22use super::Services;
23use super::entries;
24use super::generation::Control;
25use super::live::{CELL_KIND, clear_progress, finish_slot, live, slot, slot_index, slot_path};
26use super::throttle::Throttle;
27use super::usage::{Bucket, record_usage};
28use super::util::{d, from_value, strict, to_value};
29use crate::scheduler::cancel::lock;
30use crate::scheduler::{Cancel, HarnessError, NextState, Result, Task, TaskRuntime};
31
32pub const DEFAULT_MAX_BYTES: usize = 50 * 1024;
34pub const DEFAULT_MAX_LINES: usize = 2000;
36
37#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)]
39#[serde(rename_all = "camelCase")]
40pub enum Replay {
41 Safe,
42 #[default]
43 Unsafe,
44}
45
46#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
48pub struct Diagnostic {
49 pub severity: String,
50 pub code: String,
51 pub message: String,
52}
53
54impl Diagnostic {
55 pub fn error(code: &str, message: &str) -> Diagnostic {
56 Diagnostic {
57 severity: "error".into(),
58 code: code.into(),
59 message: message.into(),
60 }
61 }
62
63 fn warn(code: &str, message: &str) -> Diagnostic {
64 Diagnostic {
65 severity: "warn".into(),
66 code: code.into(),
67 message: message.into(),
68 }
69 }
70}
71
72#[derive(Clone, Debug, Default, PartialEq)]
74pub struct CellResult {
75 pub output: Option<Vec<Value>>,
77 pub is_error: bool,
78 pub details: Option<Value>,
79 pub usage: Option<Value>,
81 pub diagnostics: Vec<Diagnostic>,
82 pub control: Option<Control>,
84}
85
86impl CellResult {
87 pub fn text(text: impl Into<String>) -> CellResult {
88 CellResult {
89 output: Some(vec![json!({ "type": "text", "text": text.into() })]),
90 ..CellResult::default()
91 }
92 }
93
94 pub fn error(code: &str, message: &str) -> CellResult {
96 CellResult {
97 output: Some(Vec::new()),
98 is_error: true,
99 diagnostics: vec![Diagnostic::error(code, message)],
100 ..CellResult::default()
101 }
102 }
103}
104
105#[derive(Clone, Debug, PartialEq)]
107pub struct CellCall {
108 pub conversation_id: ConversationId,
109 pub task_id: TaskId,
110 pub call_id: String,
111 pub name: String,
112 pub arguments: Value,
114}
115
116impl CellCall {
117 pub fn code(&self) -> &str {
118 self.arguments["code"].as_str().unwrap_or_default()
119 }
120
121 pub fn lane(&self) -> &str {
123 self.arguments["lane"].as_str().unwrap_or("main")
124 }
125}
126
127pub trait CellExecutor: Send + Sync {
129 fn tool(&self) -> Tool;
131
132 fn prepare(&self, arguments: &Value) -> std::result::Result<Value, String> {
135 if !arguments["code"].is_string() {
136 return Err("rlm needs a `code` string".into());
137 }
138 if arguments.get("lane").is_some_and(|lane| !lane.is_string()) {
139 return Err("rlm's `lane` must be a string".into());
140 }
141 Ok(arguments.clone())
142 }
143
144 fn replay(&self, call: &CellCall) -> Replay {
146 let _ = call;
147 Replay::Unsafe
148 }
149
150 fn execute(
153 &self,
154 call: &CellCall,
155 progress: &CellProgress,
156 signal: &Cancel,
157 ) -> std::result::Result<CellResult, String>;
158}
159
160#[derive(Clone, Debug, Default, PartialEq)]
162struct Retained {
163 text: String,
164 lines: usize,
165 dropped_bytes: usize,
166 dropped_lines: usize,
167}
168
169impl Retained {
170 fn push(&mut self, chunk: &str) {
171 if self.dropped_bytes > 0 {
172 self.dropped_bytes += chunk.len();
173 self.dropped_lines += chunk.matches('\n').count();
174 return;
175 }
176 let mut taken = 0;
177 for (index, character) in chunk.char_indices() {
178 let end = index + character.len_utf8();
179 let lines = self.lines + usize::from(character == '\n');
180 if self.text.len() + end > DEFAULT_MAX_BYTES || lines > DEFAULT_MAX_LINES {
181 break;
182 }
183 taken = end;
184 self.lines = lines;
185 }
186 self.text.push_str(&chunk[..taken]);
187 let rest = &chunk[taken..];
188 if !rest.is_empty() {
189 self.dropped_bytes += rest.len();
190 self.dropped_lines += rest.matches('\n').count().max(1);
191 }
192 }
193}
194
195type Published = (Retained, Option<Value>);
197
198#[derive(Clone)]
201pub struct CellProgress {
202 retained: Arc<Mutex<(Retained, Option<Value>)>>,
203 throttle: Arc<Mutex<Option<Throttle<Published>>>>,
204}
205
206impl CellProgress {
207 fn new(runtime: &TaskRuntime, interval: Duration) -> CellProgress {
208 let committer = runtime.clone();
209 let throttle = Throttle::new(interval, move |(retained, details): (Retained, Option<Value>)| {
210 let conversation_id = committer.conversation_id();
211 let task_id = committer.task_id();
212 let result = committer.commit(&committer.ctx(), |tx, _| {
213 let live = live(tx, conversation_id)?;
214 let Some(index) = slot_index(&live, task_id)? else {
215 return Ok(None);
216 };
217 let current = d(live.get(&slot_path(index, Some("output"))))?;
218 if current.as_ref().and_then(Value::as_str).unwrap_or_default() != retained.text {
219 d(live.set(&slot_path(index, Some("output")), json!(retained.text)))?;
220 }
221 if retained.dropped_bytes > 0 {
222 d(live.set(&slot_path(index, Some("droppedBytes")), json!(retained.dropped_bytes)))?;
223 d(live.set(&slot_path(index, Some("droppedLines")), json!(retained.dropped_lines)))?;
224 }
225 if let Some(details) = details {
226 d(live.set(&slot_path(index, Some("details")), details))?;
227 }
228 Ok(None)
229 });
230 if let Err(error) = result
231 && !committer.signal().is_cancelled()
232 {
233 let _ = committer.report(&error);
234 }
235 });
236 CellProgress {
237 retained: Arc::new(Mutex::new((Retained::default(), None))),
238 throttle: Arc::new(Mutex::new(Some(throttle))),
239 }
240 }
241
242 pub fn detached() -> CellProgress {
244 CellProgress {
245 retained: Arc::new(Mutex::new((Retained::default(), None))),
246 throttle: Arc::new(Mutex::new(None)),
247 }
248 }
249
250 pub fn output(&self, chunk: &str) {
251 let snapshot = {
252 let mut state = lock(&self.retained);
253 state.0.push(chunk);
254 state.clone()
255 };
256 if let Some(throttle) = lock(&self.throttle).as_ref() {
257 throttle.mark(snapshot);
258 }
259 }
260
261 pub fn details(&self, value: Value) {
263 let snapshot = {
264 let mut state = lock(&self.retained);
265 state.1 = Some(strict(value));
266 state.clone()
267 };
268 if let Some(throttle) = lock(&self.throttle).as_ref() {
269 throttle.mark(snapshot);
270 }
271 }
272
273 fn stop(self) -> (Retained, Option<Value>) {
275 let throttle = lock(&self.throttle).take();
276 if let Some(throttle) = throttle {
277 throttle.stop();
278 }
279 lock(&self.retained).clone()
280 }
281}
282
283#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
285#[serde(tag = "phase", rename_all = "camelCase")]
286pub(crate) enum CellCheckpoint {
287 Call,
288 Execute {
290 arguments: Value,
291 replay: Replay,
292 },
293}
294
295static CELL_DEFINITION: LazyLock<TaskDefinition> =
296 LazyLock::new(|| TaskDefinition::new(CELL_KIND, 2, |_| to_value(&CellCheckpoint::Call)));
297
298pub(crate) fn cell_definition() -> &'static TaskDefinition {
299 &CELL_DEFINITION
300}
301
302pub(crate) fn cell_task(services: Arc<Services>) -> Task {
303 let (call, execute, abort) = (services.clone(), services.clone(), services);
304 Task::define(CELL_KIND, 2, |input| (CELL_DEFINITION.initial)(input))
305 .migrate(|input, checkpoint, _| {
307 let mut input = input.clone();
308 if input.get("kind").is_none() {
309 input["kind"] = json!("model");
310 }
311 Ok((input, checkpoint.clone()))
312 })
313 .phase("call", move |task, runtime| call_phase(&call, task, runtime))
314 .phase("execute", move |task, runtime| execute_phase(&execute, task, runtime))
315 .abort(move |task, runtime| abort_handler(&abort, task, runtime))
316 .build()
317}
318
319fn checkpoint_of(task: &TaskRecord) -> Result<CellCheckpoint> {
320 match &task.state {
321 TaskState::Pending { checkpoint }
322 | TaskState::Running { checkpoint }
323 | TaskState::Waiting { checkpoint, .. } => from_value(checkpoint.clone(), "cell checkpoint"),
324 _ => Err(HarnessError::msg("A terminal cell has no checkpoint")),
325 }
326}
327
328fn read_call(runtime: &TaskRuntime, task: &TaskRecord) -> Result<(String, String, Value)> {
330 let assistant = EntryId(task.input["assistant"].as_u64().unwrap_or_default());
331 let call_id = task.input["callId"].as_str().unwrap_or_default().to_owned();
332 let entry = runtime.read(|tx| Ok(tx.entry_of(entries::ASSISTANT, assistant)?))?;
333 let message = entry
334 .and_then(|entry| entry.model)
335 .and_then(|model| model.into_iter().next());
336 if let Some(message) = message
337 && let Message::Assistant(message) = super::context::message_of(message)?
338 {
339 for content in message.content {
340 if let AssistantContent::ToolCall(call) = content
341 && call.id == call_id
342 {
343 return Ok((call.id, call.name, call.arguments));
344 }
345 }
346 }
347 Err(HarnessError::Error(format!(
348 "Entry {} has no tool call {call_id}",
349 assistant.0
350 )))
351}
352
353enum Ending {
356 Completed,
357 Aborted,
358 Failed(String),
359}
360
361fn call_phase(services: &Services, task: &TaskRecord, runtime: &TaskRuntime) -> Result<()> {
362 let (call_id, name, arguments) = read_call(runtime, task)?;
363 let tool = services.cells.tool();
364 if name != tool.name {
365 let error = CellResult::error("tool_unavailable", &format!("Tool {name} is not available"));
366 return settle(
367 runtime,
368 task,
369 &call_id,
370 &name,
371 Ending::Completed,
372 |_| error.clone(),
373 None,
374 );
375 }
376 let arguments = match services.cells.prepare(&arguments) {
377 Ok(arguments) => strict(arguments),
378 Err(message) => {
379 let invalid = CellResult::error("invalid_arguments", &message);
380 return settle(
381 runtime,
382 task,
383 &call_id,
384 &name,
385 Ending::Completed,
386 |_| invalid.clone(),
387 None,
388 );
389 }
390 };
391 let call = CellCall {
392 conversation_id: runtime.conversation_id(),
393 task_id: runtime.task_id(),
394 call_id: call_id.clone(),
395 name: name.clone(),
396 arguments: arguments.clone(),
397 };
398 if let Some(gate) = &services.hooks.gate_tool {
401 let run = runtime.read(|tx| super::gate::gate_run(tx, call.conversation_id))?;
402 if let Some(block) = gate(&run, &call) {
403 let blocked = CellResult {
404 output: Some(vec![json!({ "type": "text", "text": block.reason })]),
405 is_error: true,
406 control: block.terminate.then_some(Control {
407 terminate: Some(true),
408 handoff: None,
409 }),
410 ..CellResult::default()
411 };
412 return settle(
413 runtime,
414 task,
415 &call_id,
416 &name,
417 Ending::Completed,
418 |_| blocked.clone(),
419 None,
420 );
421 }
422 }
423 let replay = services.cells.replay(&call);
424 let conversation_id = runtime.conversation_id();
425 let task_id = runtime.task_id();
426 runtime.commit(&runtime.ctx(), |tx, _| {
427 let live = live(tx, conversation_id)?;
428 if let Some(index) = slot_index(&live, task_id)? {
429 d(live.set(&slot_path(index, Some("status")), json!("running")))?;
430 }
431 Ok(Some(NextState::Running {
432 checkpoint: to_value(&CellCheckpoint::Execute {
433 arguments: arguments.clone(),
434 replay,
435 }),
436 }))
437 })?;
438 run(services, runtime, task, call)
439}
440
441fn execute_phase(services: &Services, task: &TaskRecord, runtime: &TaskRuntime) -> Result<()> {
443 let CellCheckpoint::Execute { arguments, replay } = checkpoint_of(task)? else {
444 return Err(HarnessError::msg("execute phase without intent"));
445 };
446 let (call_id, name, _) = read_call(runtime, task)?;
447 let call = CellCall {
448 conversation_id: runtime.conversation_id(),
449 task_id: runtime.task_id(),
450 call_id: call_id.clone(),
451 name: name.clone(),
452 arguments,
453 };
454 if replay == Replay::Safe && name == services.cells.tool().name && services.cells.replay(&call) == Replay::Safe {
455 let conversation_id = runtime.conversation_id();
457 let task_id = runtime.task_id();
458 runtime.commit(&runtime.ctx(), |tx, _| {
459 let live = live(tx, conversation_id)?;
460 if let Some(index) = slot_index(&live, task_id)? {
461 clear_progress(&live, index)?;
462 }
463 Ok(None)
464 })?;
465 return run(services, runtime, task, call);
466 }
467 let message = interrupted_message(&name);
468 settle(
469 runtime,
470 task,
471 &call_id,
472 &name,
473 Ending::Failed(message.clone()),
474 |slot| from_slot(slot, "interrupted", &message),
475 None,
476 )
477}
478
479pub fn interrupted_message(name: &str) -> String {
481 format!(
482 "Tool {name} was interrupted by a restart; effect unknown: it may have partially run and was not replayed. \
483 Check what it changed before you run it again"
484 )
485}
486
487fn abort_handler(_services: &Services, task: &TaskRecord, runtime: &TaskRuntime) -> Result<()> {
488 let (call_id, name, _) = read_call(runtime, task)?;
489 let executing = matches!(checkpoint_of(task)?, CellCheckpoint::Execute { .. });
490 let (code, message) = if task.abort_reason.is_none() {
491 ("aborted", format!("Tool {name} was aborted"))
492 } else if executing {
493 ("interrupted", interrupted_message(&name))
494 } else {
495 (
496 "abandoned",
497 format!("Tool {name} was not started: its caller ended with a restart"),
498 )
499 };
500 settle(
501 runtime,
502 task,
503 &call_id,
504 &name,
505 Ending::Aborted,
506 |slot| from_slot(slot, code, &message),
507 None,
508 )
509}
510
511fn run(services: &Services, runtime: &TaskRuntime, task: &TaskRecord, call: CellCall) -> Result<()> {
512 let interval = Duration::from_millis((services.settings)().output_interval_ms);
513 let progress = CellProgress::new(runtime, interval);
514 let started = Instant::now();
515 let executed = services.cells.execute(&call, &progress, runtime.signal());
516 let duration = started.elapsed().as_millis() as u64;
517 let (retained, details) = progress.stop();
518 let (mut result, ending) = match executed {
519 Ok(result) => (result, Ending::Completed),
520 Err(message) => {
521 runtime.signal().check()?;
522 let result = CellResult {
523 output: None,
524 is_error: true,
525 diagnostics: vec![Diagnostic::error("tool_error", &message)],
526 ..CellResult::default()
527 };
528 (result, Ending::Failed(format!("Tool {} threw", call.name)))
529 }
530 };
531 if result.output.is_none() {
532 result.output = Some(if retained.text.is_empty() {
533 Vec::new()
534 } else {
535 vec![json!({ "type": "text", "text": retained.text })]
536 });
537 if retained.dropped_bytes > 0 {
538 result.diagnostics.insert(0, truncated(&retained, Some("head")));
539 }
540 }
541 if result.details.is_none() {
542 result.details = details;
543 }
544 settle(
545 runtime,
546 task,
547 &call.call_id,
548 &call.name,
549 ending,
550 |_| result.clone(),
551 Some(duration),
552 )
553}
554
555fn truncated(retained: &Retained, retain: Option<&str>) -> Diagnostic {
556 let kept = match retain {
557 Some("head") => " to its beginning",
558 Some(_) => " to its end",
559 None => "",
560 };
561 Diagnostic::warn(
562 "truncated",
563 &format!(
564 "Output truncated{kept}: {} lines, {} bytes dropped",
565 retained.dropped_lines, retained.dropped_bytes
566 ),
567 )
568}
569
570fn from_slot(slot: Option<&Value>, code: &str, message: &str) -> CellResult {
572 let mut diagnostics: Vec<Diagnostic> = slot
573 .and_then(|slot| slot.get("diagnostics"))
574 .and_then(|value| serde_json::from_value(value.clone()).ok())
575 .unwrap_or_default();
576 let dropped_bytes = slot.and_then(|slot| slot["droppedBytes"].as_u64()).unwrap_or(0) as usize;
577 if dropped_bytes > 0 {
578 let dropped_lines = slot.and_then(|slot| slot["droppedLines"].as_u64()).unwrap_or(0) as usize;
579 diagnostics.push(truncated(
580 &Retained {
581 dropped_bytes,
582 dropped_lines,
583 ..Retained::default()
584 },
585 None,
586 ));
587 }
588 diagnostics.push(Diagnostic::error(code, message));
589 let output = slot
590 .and_then(|slot| slot["output"].as_str())
591 .filter(|text| !text.is_empty())
592 .map(|text| vec![json!({ "type": "text", "text": text })])
593 .unwrap_or_default();
594 CellResult {
595 output: Some(output),
596 is_error: true,
597 details: slot.and_then(|slot| slot.get("details").cloned()),
598 diagnostics,
599 ..CellResult::default()
600 }
601}
602
603fn settle(
607 runtime: &TaskRuntime,
608 _task: &TaskRecord,
609 call_id: &str,
610 name: &str,
611 ending: Ending,
612 build: impl FnOnce(Option<&Value>) -> CellResult,
613 duration_ms: Option<u64>,
614) -> Result<()> {
615 let conversation_id = runtime.conversation_id();
616 let task_id = runtime.task_id();
617 runtime.commit(&runtime.ctx(), |tx, _| {
618 let live = live(tx, conversation_id)?;
619 let current = slot(&live, task_id)?;
620 let result = build(current.as_ref());
621 let entry = append_tool_result(tx, conversation_id, call_id, name, &result, runtime.now()?, duration_ms)?;
622 if let Some(index) = slot_index(&live, task_id)? {
623 finish_slot(&live, index)?;
624 d(live.set(&slot_path(index, Some("entry")), json!(entry.id.0)))?;
625 }
626 let mut settled = json!({ "kind": "model", "entryId": entry.id.0 });
627 if let (Ending::Completed, Some(control)) = (&ending, &result.control) {
628 settled["control"] = strict(to_value(control));
629 }
630 let outcome = match ending {
631 Ending::Completed => TaskOutcome::Completed { result: settled },
632 Ending::Aborted => TaskOutcome::Aborted {
633 reason: None,
634 result: Some(settled),
635 },
636 Ending::Failed(message) => TaskOutcome::Failed {
637 error: TaskOutcomeError { message, detail: None },
638 result: Some(settled),
639 },
640 };
641 Ok(Some(NextState::Terminal { outcome }))
642 })
643}
644
645fn render_diagnostics(diagnostics: &[Diagnostic]) -> String {
646 let lines: Vec<String> = diagnostics
647 .iter()
648 .map(|diagnostic| format!("[{}] {}", diagnostic.severity, diagnostic.message))
649 .collect();
650 format!("<harness>\n{}\n</harness>", lines.join("\n"))
651}
652
653pub(crate) fn append_tool_result(
656 tx: &mut Tx<'_>,
657 conversation_id: ConversationId,
658 call_id: &str,
659 name: &str,
660 result: &CellResult,
661 timestamp: u64,
662 duration_ms: Option<u64>,
663) -> Result<EntryRecord> {
664 let mut content = result.output.clone().unwrap_or_default();
665 if !result.diagnostics.is_empty() {
666 content.push(json!({ "type": "text", "text": render_diagnostics(&result.diagnostics) }));
667 }
668 let mut message = json!({
669 "role": "toolResult",
670 "toolCallId": call_id,
671 "toolName": name,
672 "content": content,
673 });
674 if let Some(details) = &result.details {
675 message["details"] = details.clone();
676 }
677 if let Some(usage) = &result.usage {
678 message["usage"] = usage.clone();
679 }
680 message["isError"] = json!(result.is_error);
681 if let Some(duration) = duration_ms {
682 message["durationMs"] = json!(duration);
683 }
684 message["timestamp"] = json!(timestamp);
685 if let Some(usage) = &result.usage {
686 record_usage(tx, conversation_id, Bucket::Tools, name, usage)?;
687 }
688 Ok(tx.append_entry(
689 conversation_id,
690 EntryDraft {
691 model: Some(vec![message]),
692 data: Some(json!({ "diagnostics": result.diagnostics })),
693 ..EntryDraft::new(entries::TOOL_RESULT)
694 },
695 )?)
696}