1use super::{
13 Engine, Kind, LAST_OUTPUTS, Ledger, MAX_DEPTH, MIN_TRANCHE, OUTPUT_PREVIEW_CHARS, Options, Outcome,
14 REQUEST_OVERHEAD_TOKENS, View, bounded, charged_tokens, frame_prompt, frame_system, lock, parse_reply,
15 repair_prompt, trace,
16};
17use crate::trace::{Budget, Frame, Spec, TraceStore, Traces};
18use agent::{Agent, AgentOptions, AgentTool, StreamFn};
19use ai::{AssistantContent, AssistantMessage, Message, StopReason};
20use kernel::Signal;
21use serde_json::{Map, Value, json};
22use std::collections::HashMap;
23use std::sync::atomic::{AtomicBool, Ordering};
24use std::sync::{Arc, Mutex, OnceLock};
25use std::time::Instant;
26
27const FOLLOW_UP_EXHAUSTED: &str = "budget exhausted before a follow-up request (a retry or tool round)";
28
29pub trait FrameTasks: Send + Sync {
32 fn caller_task(&self, lane: &str) -> Option<String>;
34 fn spawn(&self, lane: &str, input: Value, model: Option<&str>, timeout_ms: u64) -> Result<String, String>;
36 fn result(&self, lane: &str, task_id: &str, signal: &Signal) -> Value;
39 fn reserve(&self, request: &Value) -> Result<Option<Value>, String>;
42 fn settle(&self, reservation: &Value, status: &str, usage: Option<&Value>);
45 fn code_tool(&self, lane: &str) -> Result<Arc<dyn AgentTool>, String>;
47 fn close_lane(&self, lane: &str);
49 fn frame_context(&self, lane: &str, model: &ai::Model, meter: FrameMeter) {
53 let _ = (lane, model, meter);
54 }
55 fn charge_turn(&self, root: Option<&str>, reply: &AssistantMessage) {
58 let _ = (root, reply);
59 }
60 fn gate_request(&self, lane: &str, root: Option<&str>) -> FrameGate {
62 let _ = (lane, root);
63 FrameGate::Allow
64 }
65 fn gate_tool(&self, lane: &str, root: Option<&str>) -> Option<(String, bool)> {
68 let _ = (lane, root);
69 None
70 }
71}
72
73#[derive(Clone, Debug, PartialEq, Eq)]
75pub enum FrameGate {
76 Allow,
77 Final(String),
79 Block(String),
81}
82
83pub type FrameMeter = Arc<dyn Fn() -> Option<u64> + Send + Sync>;
85
86pub struct FrameExecution {
88 pub task_id: String,
89 pub input: Value,
90 pub lane: String,
91 pub usage_root: Option<String>,
92 pub deadline_at: Option<f64>,
93 pub timeout_ms: u64,
94 pub signal: Signal,
96}
97
98pub(crate) struct Admission {
100 pub depth: u64,
101 parent: Option<Arc<Live>>,
102 caller_task: Option<String>,
103 calls: Option<u64>,
104 tokens: Option<u64>,
105 default_tokens: bool,
106}
107
108pub(crate) struct Live {
110 id: String,
111 node: usize,
112 frame: Mutex<Frame>,
113 budget: Arc<Budget>,
114 options: Arc<Options>,
115 engine: Mutex<Engine>,
117 outcome: Mutex<Option<Outcome>>,
118 finalized: AtomicBool,
119}
120
121impl Live {
122 fn engine(&self) -> Engine {
123 lock(&self.engine).clone()
124 }
125
126 fn outcome(&self) -> Option<Outcome> {
127 lock(&self.outcome).clone()
128 }
129
130 fn set_outcome(&self, outcome: Outcome) {
131 *lock(&self.outcome) = Some(outcome);
132 }
133
134 fn spec(&self) -> Spec<'_> {
135 Spec {
136 contract: self.options.contract.as_ref(),
137 max_repairs: self.options.max_repairs,
138 depth: self.options.depth,
139 model: self.options.model.as_deref(),
140 }
141 }
142}
143
144pub struct FrameSession {
146 ledger: Mutex<Ledger>,
147 labels: Mutex<HashMap<String, String>>,
148 traces: Mutex<Traces>,
149 live: Mutex<Vec<Arc<Live>>>,
151 lanes: Mutex<HashMap<String, Arc<Live>>>,
153 tasks: OnceLock<Arc<dyn FrameTasks>>,
154}
155
156impl FrameSession {
157 pub fn new(store: Arc<dyn TraceStore>) -> Arc<Self> {
159 Arc::new(FrameSession {
160 ledger: Mutex::new(Ledger::default()),
161 labels: Mutex::new(HashMap::new()),
162 traces: Mutex::new(Traces::new(store)),
163 live: Mutex::new(Vec::new()),
164 lanes: Mutex::new(HashMap::new()),
165 tasks: OnceLock::new(),
166 })
167 }
168
169 pub fn set_tasks(&self, tasks: Arc<dyn FrameTasks>) {
171 let _ = self.tasks.set(tasks);
172 }
173
174 pub(crate) fn has_tasks(&self) -> bool {
175 self.tasks.get().is_some()
176 }
177
178 pub(crate) fn label(&self, digest: &str) -> Option<String> {
179 lock(&self.labels).get(digest).cloned()
180 }
181
182 pub(crate) fn set_label(&self, digest: &str, label: String) {
183 lock(&self.labels).insert(digest.to_string(), label);
184 }
185
186 pub(crate) fn admission(
190 &self,
191 lane: &str,
192 requested_depth: Option<u64>,
193 calls: Option<u64>,
194 tokens: Option<u64>,
195 default_tokens: Option<u64>,
196 ) -> Admission {
197 let parent = lock(&self.lanes).get(lane).cloned();
198 let ceiling = parent
199 .as_ref()
200 .map_or(MAX_DEPTH, |frame| frame.options.depth.saturating_sub(1));
201 let depth = requested_depth
202 .unwrap_or(if parent.is_some() { ceiling } else { 1 })
203 .min(ceiling);
204 let caller_task = match &parent {
205 Some(frame) => lock(&frame.frame).task_id.clone(),
206 None => self.tasks.get().and_then(|tasks| tasks.caller_task(lane)),
207 };
208 let use_default = tokens.is_none() && parent.is_none() && default_tokens.is_some();
209 Admission {
210 depth,
211 calls,
212 tokens: if use_default { default_tokens } else { tokens },
213 default_tokens: use_default,
214 parent,
215 caller_task,
216 }
217 }
218
219 pub(crate) fn budget(&self, admission: &Admission) -> Budget {
221 let parent_node = admission.parent.as_ref().map(|frame| frame.node);
222 let node = lock(&self.ledger).add(parent_node, admission.calls, admission.tokens);
223 Budget {
224 id: format!("budget-{}", jsrt::random::uuid_v4()),
225 node,
226 depth: admission.depth,
227 parent: admission.parent.as_ref().map(|frame| frame.id.clone()),
228 default_tokens: admission.default_tokens,
229 }
230 }
231
232 #[allow(clippy::too_many_arguments)]
235 pub(crate) fn admit(
236 &self,
237 budget: &Arc<Budget>,
238 options: &Arc<Options>,
239 engine: &Engine,
240 admission: &Admission,
241 task: &str,
242 views: Vec<View>,
243 kind: &'static str,
244 batch: usize,
245 ) -> Arc<Live> {
246 let mut ledger = lock(&self.ledger);
247 let mut frame = Frame::new(&mut ledger, budget, task, views);
248 frame.kind = kind;
249 frame.batch = batch;
250 frame.caller_task = admission.caller_task.clone();
251 frame.parent_frame = admission.parent.as_ref().map(|frame| frame.id.clone());
252 let mut outcome = None;
253 if options.depth < 1 {
254 outcome = Some(Outcome::Incomplete(
255 "depth_exhausted",
256 "this frame's budget allows no nested frames".into(),
257 ));
258 } else if ledger.hold(budget.node, Kind::Calls, 1) {
259 frame.call_held = true;
260 } else {
261 outcome = Some(Outcome::Incomplete(
262 "budget_exhausted",
263 "no call left in the budget for this frame".into(),
264 ));
265 }
266 drop(ledger);
267 let live = Arc::new(Live {
268 id: frame.id.clone(),
269 node: frame.node,
270 frame: Mutex::new(frame),
271 budget: budget.clone(),
272 options: options.clone(),
273 engine: Mutex::new(engine.clone()),
274 outcome: Mutex::new(outcome),
275 finalized: AtomicBool::new(false),
276 });
277 lock(&self.live).push(live.clone());
278 live
279 }
280
281 fn release_call(&self, live: &Live) {
283 let mut ledger = lock(&self.ledger);
284 let mut frame = lock(&live.frame);
285 if frame.call_held {
286 frame.call_held = false;
287 ledger.release(live.node, Kind::Calls, 1);
288 }
289 }
290
291 pub(crate) fn run_frame(&self, live: &Arc<Live>, lane: &str, cell: &Signal) {
294 if live.outcome().is_none() {
295 let outcome = self.start(live, lane, cell);
296 if let Some(outcome) = outcome
297 && live.outcome().is_none()
298 {
299 live.set_outcome(outcome);
300 }
301 }
302 self.release_call(live);
303 self.finalize(live);
304 }
305
306 fn start(&self, live: &Arc<Live>, lane: &str, cell: &Signal) -> Option<Outcome> {
307 if cell.is_aborted() {
308 return Some(Outcome::Error("cancelled: the calling cell was aborted".into()));
309 }
310 match live.engine().for_model(live.options.model.as_deref()) {
313 Ok(engine) => *lock(&live.engine) = engine,
314 Err(refused) => return Some(Outcome::Error(refused)),
315 }
316 let Some(tasks) = self.tasks.get() else {
317 return Some(self.converse(live, cell, None));
318 };
319 let task = lock(&live.frame).task.clone();
320 let input = json!({"frame": live.id, "task": bounded(&task, 200)});
321 let timeout_ms = live.options.timeout.as_millis() as u64;
322 let task_id = match tasks.spawn(lane, input, live.options.model.as_deref(), timeout_ms) {
323 Ok(id) => id,
324 Err(error) => return Some(Outcome::Error(format!("Admission refused: {error}"))),
325 };
326 lock(&live.frame).task_id = Some(task_id.clone());
327 let result = tasks.result(lane, &task_id, cell);
328 Some(if result["status"] == "succeeded" {
329 Outcome::Complete(result.get("value").cloned().unwrap_or(Value::Null))
330 } else {
331 let status = result["status"].as_str().unwrap_or("failed");
332 let error = result["error"].as_str().unwrap_or("frame did not finish");
333 Outcome::Error(format!("{status}: {error}"))
334 })
335 }
336
337 pub fn execute(self: &Arc<Self>, run: FrameExecution) -> Result<Value, String> {
340 let id = run.input["frame"].as_str().unwrap_or_default();
341 let live = lock(&self.live).iter().find(|live| live.id == id).cloned();
342 let Some(live) = live else {
343 return Err("rlm-frame tasks are started only by rlm.infer and rlm.map".into());
344 };
345 {
346 let mut frame = lock(&live.frame);
347 frame.task_id = Some(run.task_id.clone());
348 frame.lane = Some(run.lane.clone());
349 }
350 lock(&self.lanes).insert(run.lane.clone(), live.clone());
351 let outcome = if live.options.depth > 1 {
352 let outcome = self.converse_with_code(&live, &run);
353 if let Some(tasks) = self.tasks.get() {
354 tasks.close_lane(&run.lane);
355 }
356 outcome
357 } else {
358 self.converse(&live, &run.signal, Some(&run))
359 };
360 lock(&self.lanes).remove(&run.lane);
361 self.release_call(&live);
362 live.set_outcome(outcome.clone());
363 match outcome {
364 Outcome::Complete(value) => Ok(value),
365 _ if run.signal.is_aborted() => Err("cancelled".into()),
366 Outcome::Incomplete(reason, detail) => Err(format!("Incomplete ({reason}): {detail}")),
367 Outcome::Error(error) => Err(error),
368 }
369 }
370
371 fn reserve(&self, live: &Live, run: Option<&FrameExecution>, attempt: u64) -> Result<Option<Value>, String> {
373 let (Some(run), Some(tasks)) = (run, self.tasks.get()) else {
374 return Ok(None);
375 };
376 let mut request = json!({
377 "kind": "model",
378 "taskId": run.task_id,
379 "parentTaskId": run.task_id,
380 "requestKey": format!("{}:frame:{attempt}", run.task_id),
381 "budgetId": live.budget.id,
382 });
383 if let Some(root) = &run.usage_root {
384 request["rootId"] = json!(root);
385 }
386 match run.deadline_at {
387 Some(deadline) => request["deadlineAt"] = json!(deadline),
388 None => request["timeoutMs"] = json!(run.timeout_ms),
389 }
390 tasks.reserve(&request)
391 }
392
393 fn settle(&self, reservation: Option<Value>, status: &str, usage: Option<Value>) {
394 if let (Some(reservation), Some(tasks)) = (reservation, self.tasks.get()) {
395 tasks.settle(&reservation, status, usage.as_ref());
396 }
397 }
398
399 fn converse(&self, live: &Live, signal: &Signal, run: Option<&FrameExecution>) -> Outcome {
402 let options = &live.options;
403 let engine = &live.engine();
404 let deadline = Instant::now() + options.timeout;
405 let system = frame_system(options);
406 let mut conversation_chars = jsrt::utf16::len(&system) as u64;
407 let mut message = {
408 let mut frame = lock(&live.frame);
409 frame.model = Some(engine.own_model());
410 frame_prompt(&frame.task, &frame.views, options.contract.as_ref())
411 };
412 let mut history: Vec<Message> = Vec::new();
413 for attempt in 0u64.. {
414 if signal.is_aborted() || Instant::now() >= deadline {
415 return Outcome::Error("cancelled".into());
416 }
417 let estimate =
418 (conversation_chars + jsrt::utf16::len(&message) as u64).div_ceil(3) + REQUEST_OVERHEAD_TOKENS;
419 if let Some(error) = self.window_error(engine, estimate) {
420 return Outcome::Error(error);
421 }
422 let Some((held, tranche)) = self.claim(live, estimate) else {
423 return Outcome::Incomplete("budget_exhausted", exhausted_detail(attempt));
424 };
425 conversation_chars += jsrt::utf16::len(&message) as u64;
426 history.push(user_message(&message));
427 let reservation = match self.reserve(live, run, attempt) {
428 Ok(reservation) => reservation,
429 Err(error) => {
430 self.release(live, held);
431 return Outcome::Error(error);
432 }
433 };
434 let response = engine.request(&system, &history, tranche, options, deadline, signal);
435 let charged = match &response {
436 Ok(Some(reply)) => charged_tokens(&reply.usage),
437 _ => 0,
438 };
439 self.settle_request(
440 live,
441 held,
442 tranche,
443 response.as_ref().ok().and_then(Option::as_ref),
444 charged,
445 );
446 let (status, usage) = match &response {
447 Ok(Some(reply)) => ("succeeded", measured(std::slice::from_ref(reply))),
448 Ok(None) => ("cancelled", None),
449 Err(_) if signal.is_aborted() => ("cancelled", None),
450 Err(_) => ("failed", None),
451 };
452 self.settle(reservation, status, usage);
453 let reply = match response {
454 Ok(Some(reply)) => reply,
455 Ok(None) => return Outcome::Error("cancelled".into()),
456 Err(error) => return Outcome::Error(format!("frame request failed: {}", Value::String(error))),
457 };
458 add_measured(live, std::slice::from_ref(&reply));
459 if let (Some(run), Some(tasks)) = (run, self.tasks.get()) {
460 tasks.charge_turn(run.usage_root.as_deref(), &reply);
461 }
462 if signal.is_aborted() {
463 return Outcome::Error("cancelled".into());
464 }
465 if let Some(outcome) = failed_reply(&reply) {
466 return outcome;
467 }
468 let text = reply_text(&reply);
469 history.push(Message::Assistant(reply));
470 conversation_chars += jsrt::utf16::len(&text) as u64;
471 match self.judge(live, attempt, text) {
472 Ok(outcome) => return outcome,
473 Err(next) => message = next,
474 }
475 }
476 unreachable!("the attempt counter is unbounded")
477 }
478
479 fn converse_with_code(self: &Arc<Self>, live: &Arc<Live>, run: &FrameExecution) -> Outcome {
483 let Some(tasks) = self.tasks.get() else {
484 return Outcome::Error("frames that run code need a task host".into());
485 };
486 let options = &live.options;
487 let engine = &live.engine();
488 let tool = match tasks.code_tool(&run.lane) {
489 Ok(tool) => tool,
490 Err(error) => return Outcome::Error(error),
491 };
492 let deadline = Instant::now() + options.timeout;
493 let system = frame_system(options);
494 let mut message = {
495 let mut frame = lock(&live.frame);
496 frame.model = Some(engine.own_model());
497 frame_prompt(&frame.task, &frame.views, options.contract.as_ref())
498 };
499 let pending: Arc<Mutex<Option<(u64, u64)>>> = Arc::default();
500 let context_use: Arc<Mutex<Option<u64>>> = Arc::default();
501 let meter_use = context_use.clone();
502 tasks.frame_context(&run.lane, &engine.model, Arc::new(move || *lock(&meter_use)));
503 let stream = BudgetedStream {
504 session: self.clone(),
505 live: live.clone(),
506 pending: pending.clone(),
507 deadline,
508 signal: run.signal.clone(),
509 context_use,
510 tasks: tasks.clone(),
511 lane: run.lane.clone(),
512 root: run.usage_root.clone(),
513 };
514 let mut agent_options = AgentOptions::new(engine.model.clone());
515 agent_options.system_prompt = Some(system.clone());
516 agent_options.tools = vec![tool];
517 let (gate_tasks, gate_lane, gate_root) = (tasks.clone(), run.lane.clone(), run.usage_root.clone());
519 agent_options.before_tool_call = Some(Arc::new(move |_, _| {
520 let decision = gate_tasks.gate_tool(&gate_lane, gate_root.as_deref());
521 Box::pin(async move {
522 Ok(decision.map(|(reason, terminate)| agent::BeforeToolCallResult {
523 block: true,
524 reason: Some(reason),
525 terminate: Some(terminate),
526 }))
527 })
528 }));
529 agent_options.thinking_level = options
530 .thinking
531 .clone()
532 .filter(|_| engine.model.reasoning)
533 .unwrap_or_else(|| "off".into());
534 let mut agent = Agent::new(agent_options, Box::new(stream));
535 let runtime = match tokio::runtime::Builder::new_current_thread().enable_all().build() {
536 Ok(runtime) => runtime,
537 Err(error) => return Outcome::Error(error.to_string()),
538 };
539 let control = agent.control();
541 let listener = run.signal.on_abort(move |_| control.abort());
542 let mut conversation_chars = jsrt::utf16::len(&system) as u64;
543 let outcome = 'attempts: {
544 for attempt in 0u64.. {
545 if run.signal.is_aborted() || Instant::now() >= deadline {
546 break 'attempts Outcome::Error("cancelled".into());
547 }
548 let estimate =
549 (conversation_chars + jsrt::utf16::len(&message) as u64).div_ceil(3) + REQUEST_OVERHEAD_TOKENS;
550 if let Some(error) = self.window_error(engine, estimate) {
551 break 'attempts Outcome::Error(error);
552 }
553 let Some(hold) = self.claim(live, estimate) else {
554 break 'attempts Outcome::Incomplete("budget_exhausted", exhausted_detail(attempt));
555 };
556 *lock(&pending) = Some(hold);
557 conversation_chars += jsrt::utf16::len(&message) as u64;
558 let reservation = match self.reserve(live, Some(run), attempt) {
559 Ok(reservation) => reservation,
560 Err(error) => {
561 if let Some((held, _)) = lock(&pending).take() {
562 self.release(live, held);
563 }
564 break 'attempts Outcome::Error(error);
565 }
566 };
567 let before = agent.messages.len();
568 runtime.block_on(agent.prompt_text(&message, Vec::new()));
569 if let Some((held, _)) = lock(&pending).take() {
571 self.release(live, held);
572 }
573 let replies: Vec<AssistantMessage> = agent.messages[before..]
574 .iter()
575 .filter_map(|message| match message {
576 Message::Assistant(reply) => Some(reply.clone()),
577 _ => None,
578 })
579 .collect();
580 let status = if run.signal.is_aborted() {
581 "cancelled"
582 } else {
583 "succeeded"
584 };
585 self.settle(reservation, status, measured(&replies));
586 add_measured(live, &replies);
587 if run.signal.is_aborted() {
588 break 'attempts Outcome::Error("cancelled".into());
589 }
590 if let Some(exhausted) = lock(&live.frame).exhausted.clone() {
591 break 'attempts Outcome::Incomplete("budget_exhausted", exhausted);
592 }
593 let Some(reply) = replies.last() else {
594 break 'attempts Outcome::Error("frame run did not complete: failed".into());
595 };
596 if let Some(outcome) = failed_reply(reply) {
597 break 'attempts outcome;
598 }
599 let text = reply_text(reply);
600 conversation_chars += jsrt::utf16::len(&text) as u64;
601 match self.judge(live, attempt, text) {
602 Ok(outcome) => break 'attempts outcome,
603 Err(next) => message = next,
604 }
605 }
606 unreachable!("the attempt counter is unbounded")
607 };
608 if let Some(listener) = listener {
609 run.signal.forget(listener);
610 }
611 outcome
612 }
613
614 fn window_error(&self, engine: &Engine, estimate: u64) -> Option<String> {
616 let window = engine.model.context_window;
617 (window > 0 && estimate + MIN_TRANCHE > window).then(|| {
618 format!(
619 "context too large for {}/{}: about {estimate} tokens against a {window}-token window; pass smaller slices",
620 engine.model.provider, engine.model.id
621 )
622 })
623 }
624
625 fn claim(&self, live: &Live, estimate: u64) -> Option<(u64, u64)> {
627 let mut ledger = lock(&self.ledger);
628 let mut frame = lock(&live.frame);
629 let held = frame.call_held;
630 frame.call_held = false;
631 let model_max = live.engine().model.max_tokens;
632 ledger.claim(live.node, estimate, model_max, held)
633 }
634
635 fn release(&self, live: &Live, held: u64) {
637 let mut ledger = lock(&self.ledger);
638 ledger.release(live.node, Kind::Calls, 1);
639 ledger.release(live.node, Kind::Tokens, held);
640 }
641
642 fn settle_request(&self, live: &Live, held: u64, tranche: u64, reply: Option<&AssistantMessage>, charged: u64) {
645 let mut ledger = lock(&self.ledger);
646 ledger.release(live.node, Kind::Calls, 1);
647 ledger.release(live.node, Kind::Tokens, held);
648 if reply.is_some() {
649 ledger.charge(live.node, Kind::Calls, 1);
650 ledger.charge(live.node, Kind::Tokens, charged);
651 }
652 drop(ledger);
653 if let Some(reply) = reply {
654 let mut frame = lock(&live.frame);
655 if frame.requests.len() < trace::TRACE_REQUEST_LIMIT {
656 frame.requests.push(json!({
657 "at": jsrt::date::now_ms() as u64, "tranche": tranche, "held": held, "charged": charged,
658 "status": serde_json::to_value(reply.stop_reason).unwrap_or(Value::Null),
659 }));
660 }
661 }
662 }
663
664 fn judge(&self, live: &Live, attempt: u64, text: String) -> Result<Outcome, String> {
666 let options = &live.options;
667 let output = bounded(&text, OUTPUT_PREVIEW_CHARS);
668 let mut frame = lock(&live.frame);
669 frame.outputs.push(output.clone());
670 if frame.outputs.len() > LAST_OUTPUTS {
671 frame.outputs.remove(0);
672 }
673 let record = |error: Option<&str>| {
674 let mut record = json!({"attempt": attempt, "at": jsrt::date::now_ms() as u64, "output": output});
675 if let Some(error) = error {
676 record["error"] = error.into();
677 }
678 record
679 };
680 let error = if jsrt::string::trim(&text).is_empty() {
681 "the reply is empty".to_string()
682 } else {
683 let parsed = match &options.contract {
684 None => Ok(Value::String(text)),
685 Some(contract) => parse_reply(&text, contract),
686 };
687 match parsed {
688 Ok(value) => {
689 frame.attempts.push(record(None));
690 return Ok(Outcome::Complete(value));
691 }
692 Err(error) => error,
693 }
694 };
695 frame.attempts.push(record(Some(&error)));
696 if attempt >= options.max_repairs {
697 return Ok(Outcome::Incomplete(
698 "contract_unmet",
699 format!("no valid answer after {} attempt(s): {error}", attempt + 1),
700 ));
701 }
702 let fallback = json!({"type": "string", "minLength": 1});
703 Err(repair_prompt(&error, options.contract.as_ref().unwrap_or(&fallback)))
704 }
705
706 fn finalize(&self, live: &Arc<Live>) {
708 if live.finalized.swap(true, Ordering::SeqCst) {
709 return;
710 }
711 lock(&live.frame).ended_at = Some(jsrt::date::now_ms() as u64);
712 let outcome = live.outcome();
713 let (document, summary) = {
714 let mut ledger = lock(&self.ledger);
715 let frame = lock(&live.frame);
716 let spec = live.spec();
717 (
718 trace::trace(&mut ledger, &live.budget, &frame, &spec, outcome.as_ref()),
719 trace::summary(&ledger, &live.budget, &frame, outcome.as_ref()),
720 )
721 };
722 lock(&self.traces).write(&live.id, &document, summary);
723 lock(&self.live).retain(|other| !Arc::ptr_eq(other, live));
724 for view in &mut lock(&live.frame).views {
726 view.text = String::new();
727 }
728 }
729
730 pub(crate) fn observation(&self, live: &Live) -> Value {
732 let outcome = live
733 .outcome()
734 .unwrap_or_else(|| Outcome::Error("frame did not run".into()));
735 let mut ledger = lock(&self.ledger);
736 let frame = lock(&live.frame);
737 let mut out = Map::new();
738 let complete = matches!(outcome, Outcome::Complete(_));
739 match outcome {
740 Outcome::Complete(value) => {
741 out.insert("status".into(), "complete".into());
742 out.insert("value".into(), value);
743 }
744 Outcome::Incomplete(reason, detail) => {
745 out.insert("status".into(), "incomplete".into());
746 out.insert("reason".into(), reason.into());
747 let detail = if reason == "budget_exhausted" && live.budget.default_tokens {
748 format!(
749 "{detail} (rlm.map's default budget is {} tokens; pass budget=Budget(tokens=...) to allow more)",
750 ledger.nodes[live.budget.node].tokens.limit.unwrap_or_default()
751 )
752 } else {
753 detail
754 };
755 out.insert("detail".into(), detail.into());
756 }
757 Outcome::Error(error) => {
758 out.insert("status".into(), "error".into());
759 out.insert("error".into(), error.into());
760 }
761 }
762 out.insert("trace_id".into(), frame.id.clone().into());
763 if let Some(task) = &frame.task_id {
764 out.insert("task_id".into(), task.clone().into());
765 }
766 out.insert("spent".into(), frame.spent(&ledger));
767 out.insert("remaining".into(), frame.remaining(&mut ledger, live.options.depth));
768 if !complete {
769 out.insert("last_outputs".into(), frame.outputs.clone().into());
770 }
771 Value::Object(out)
772 }
773
774 pub(crate) fn budget_reply(&self, budget: &Budget) -> Value {
776 let mut ledger = lock(&self.ledger);
777 let mut reply = budget.snapshot(&ledger);
778 reply["remaining"] = json!({
779 "calls": ledger.available(budget.node, Kind::Calls),
780 "tokens": ledger.available(budget.node, Kind::Tokens),
781 "depth": budget.depth,
782 });
783 reply
784 }
785
786 pub(crate) fn measured(&self, live: &Live) -> (u64, u64, f64) {
787 lock(&live.frame).measured
788 }
789
790 pub(crate) fn list(&self, payload: &Map<String, Value>) -> Result<Value, String> {
792 let live: Vec<Arc<Live>> = lock(&self.live).clone();
793 let view = |live: &Live, full: bool| {
794 let outcome = live.outcome();
795 let mut ledger = lock(&self.ledger);
796 let frame = lock(&live.frame);
797 if full {
798 trace::trace(&mut ledger, &live.budget, &frame, &live.spec(), outcome.as_ref())
799 } else {
800 trace::summary(&ledger, &live.budget, &frame, outcome.as_ref())
801 }
802 };
803 let summaries: Vec<Value> = live.iter().map(|frame| view(frame, false)).collect();
804 let by_id = |id: &str| live.iter().find(|frame| frame.id == id).map(|frame| view(frame, true));
805 lock(&self.traces).list(payload, &by_id, &summaries)
806 }
807}
808
809fn exhausted_detail(attempt: u64) -> String {
810 if attempt == 0 {
811 "the budget cannot cover the first request".to_string()
812 } else {
813 format!("the budget ran out after {attempt} attempt(s) without a valid answer")
814 }
815}
816
817fn user_message(text: &str) -> Message {
818 Message::User(ai::UserMessage {
819 content: ai::UserMessageContent::Text(text.to_string()),
820 timestamp: 0,
821 })
822}
823
824fn reply_text(reply: &AssistantMessage) -> String {
825 reply
826 .content
827 .iter()
828 .filter_map(|part| match part {
829 AssistantContent::Text(text) => Some(text.text.as_str()),
830 _ => None,
831 })
832 .collect::<Vec<_>>()
833 .join("\n")
834}
835
836fn failed_reply(reply: &AssistantMessage) -> Option<Outcome> {
838 let limited = reply
839 .diagnostics
840 .iter()
841 .flatten()
842 .any(|d| d.kind == "provider_usage_limit");
843 if limited {
844 let detail = reply
845 .error_message
846 .clone()
847 .unwrap_or("the provider's usage limit is reached".into());
848 return Some(Outcome::Incomplete("usage_limit", detail));
849 }
850 if matches!(reply.stop_reason, StopReason::Error | StopReason::Aborted) {
851 let status = if reply.stop_reason == StopReason::Aborted {
852 "aborted"
853 } else {
854 "failed"
855 };
856 let error = reply.error_message.clone().unwrap_or_default();
857 return Some(Outcome::Error(format!(
858 "frame run did not complete: {status} ({})",
859 json!({"message": error})
860 )));
861 }
862 None
863}
864
865fn measured(replies: &[AssistantMessage]) -> Option<Value> {
868 if replies.is_empty() || replies.iter().any(|reply| reply.stop_reason == StopReason::Error) {
869 return None;
870 }
871 let (mut input, mut output, mut total, mut cost) = (0u64, 0u64, 0u64, 0f64);
872 for reply in replies {
873 input += reply.usage.input;
874 output += reply.usage.output;
875 total += reply.usage.total_tokens;
876 cost += reply.usage.cost.total;
877 }
878 Some(json!({"inputTokens": input, "outputTokens": output, "totalTokens": total, "cost": cost}))
879}
880
881fn add_measured(live: &Live, replies: &[AssistantMessage]) {
882 let mut frame = lock(&live.frame);
883 for reply in replies {
884 frame.measured.0 += reply.usage.input;
885 frame.measured.1 += reply.usage.output;
886 frame.measured.2 += reply.usage.cost.total;
887 }
888}
889
890fn unsent(engine: &Engine, stop_reason: StopReason, error: String) -> AssistantMessage {
892 AssistantMessage {
893 content: Vec::new(),
894 api: engine.model.api.clone(),
895 provider: engine.model.provider.clone(),
896 model: engine.model.id.clone(),
897 response_model: None,
898 response_id: None,
899 provider_thinking_level: None,
900 diagnostics: None,
901 usage: ai::Usage::default(),
902 stop_reason,
903 deferred: None,
904 error_message: Some(error),
905 raw_stop_reason: None,
906 end_turn: None,
907 provider_metadata: None,
908 timestamp: jsrt::date::now_ms(),
909 }
910}
911
912struct BudgetedStream {
916 session: Arc<FrameSession>,
917 live: Arc<Live>,
918 pending: Arc<Mutex<Option<(u64, u64)>>>,
919 deadline: Instant,
920 signal: Signal,
921 context_use: Arc<Mutex<Option<u64>>>,
923 tasks: Arc<dyn FrameTasks>,
925 lane: String,
926 root: Option<String>,
927}
928
929impl StreamFn for BudgetedStream {
930 fn stream<'a>(
931 &'a self,
932 _model: &'a ai::Model,
933 context: &'a ai::TranscriptContext,
934 options: &'a ai::SimpleStreamOptions,
935 sink: &'a mut ai::EventSink<'_>,
936 ) -> ai::http::BoxFuture<'a, AssistantMessage> {
937 Box::pin(async move {
938 let session = &self.session;
939 let live = &self.live;
940 let engine = &live.engine();
941 let mut options = options.clone();
943 let gated;
944 let context = match self.tasks.gate_request(&self.lane, self.root.as_deref()) {
945 FrameGate::Allow => context,
946 FrameGate::Block(reason) => {
947 if let Some((held, _)) = lock(&self.pending).take() {
948 session.release(live, held);
949 }
950 return unsent(engine, StopReason::Error, reason);
951 }
952 FrameGate::Final(note) => {
953 options.tool_choice = Some("none".into());
954 let mut messages = context.messages.clone();
955 messages.push(user_message(¬e));
956 gated = ai::TranscriptContext { messages };
957 &gated
958 }
959 };
960 let hold = lock(&self.pending).take().or_else(|| {
961 let estimate = serde_json::to_string(context)
962 .map_or(0, |text| text.len() as u64)
963 .div_ceil(4)
964 + REQUEST_OVERHEAD_TOKENS;
965 session.claim(live, estimate)
966 });
967 let Some((held, tranche)) = hold else {
968 lock(&live.frame).exhausted = Some(FOLLOW_UP_EXHAUSTED.into());
969 return unsent(
970 engine,
971 StopReason::Error,
972 format!("Inference frame {}: {FOLLOW_UP_EXHAUSTED}", live.id),
973 );
974 };
975 engine.prepare(&mut options, tranche, live.options.contract.as_ref());
976 let response = engine
977 .stream(context, &options, sink, self.deadline, &self.signal)
978 .await;
979 let charged = match &response {
980 Ok(Some(reply)) => charged_tokens(&reply.usage),
981 _ => 0,
982 };
983 session.settle_request(
984 live,
985 held,
986 tranche,
987 response.as_ref().ok().and_then(Option::as_ref),
988 charged,
989 );
990 if let Ok(Some(reply)) = &response {
991 self.tasks.charge_turn(self.root.as_deref(), reply);
992 let mut messages = context.messages.clone();
993 messages.push(Message::Assistant(reply.clone()));
994 *lock(&self.context_use) = Some(ai::transcript::estimate_context_tokens(&messages).tokens);
995 }
996 match response {
997 Ok(Some(reply)) => reply,
998 Ok(None) => unsent(engine, StopReason::Aborted, "cancelled".into()),
999 Err(error) => unsent(engine, StopReason::Error, error),
1000 }
1001 })
1002 }
1003}