Skip to main content

inference/
session.rs

1//! A session's frames (`InferenceRuntime` of inference.ts): admission into the shared budget ledger, the frames
2//! still running, their traces, and how a frame runs.
3//!
4//! With a task host ([`FrameTasks`], the session's typed-agent host) a frame is admitted as an `rlm-frame@1` task
5//! under its caller: the calling thread waits for the task's terminal result (cancelling it when the calling cell is
6//! aborted) while the task host runs [`FrameSession::execute`] on the task's own thread, which reserves and settles
7//! each provider request in the usage ledger. A frame with `budget.depth > 1` runs its conversation as an agent on
8//! the task's lane with the `rlm` cell as its only tool; requests from that lane's kernel are nested frames, which
9//! draw on the frame's pool with one level less depth. Without a task host (module tests) frames run on the
10//! calling thread and cannot run code.
11
12use super::{
13    Engine, Kind, LAST_OUTPUTS, Ledger, MAX_DEPTH, MIN_TRANCHE, OUTPUT_PREVIEW_CHARS, Options, Outcome,
14    REQUEST_OVERHEAD_TOKENS, View, bounded, charged_tokens, frame_prompt, frame_system, lock, parse_reply,
15    repair_prompt, trace,
16};
17use crate::trace::{Budget, Frame, Spec, TraceStore, Traces};
18use agent::{Agent, AgentOptions, AgentTool, StreamFn};
19use ai::{AssistantContent, AssistantMessage, Message, StopReason};
20use kernel::Signal;
21use serde_json::{Map, Value, json};
22use std::collections::HashMap;
23use std::sync::atomic::{AtomicBool, Ordering};
24use std::sync::{Arc, Mutex, OnceLock};
25use std::time::Instant;
26
27const FOLLOW_UP_EXHAUSTED: &str = "budget exhausted before a follow-up request (a retry or tool round)";
28
29/// The task host frames are admitted through (`NativeHostApi`): `rlm-frame@1` tasks in the session's task journal,
30/// the usage ledger they charge, and the lanes that frames with depth run their `rlm` cells on.
31pub trait FrameTasks: Send + Sync {
32    /// The task `lane` acts for (`callerTaskId`); none for the root lane.
33    fn caller_task(&self, lane: &str) -> Option<String>;
34    /// Admit an `rlm-frame@1` task with `input` (`{frame, task}`) for `lane`; returns its task id.
35    fn spawn(&self, lane: &str, input: Value, model: Option<&str>, timeout_ms: u64) -> Result<String, String>;
36    /// Wait for the task's terminal result (`{status, value?, error?}`); once `signal` aborts the task is cancelled
37    /// ("the calling cell was aborted") and its cancelled result returned.
38    fn result(&self, lane: &str, task_id: &str, signal: &Signal) -> Value;
39    /// Reserve one model request in the usage ledger (`{kind: "model", rootId?, taskId, parentTaskId, requestKey,
40    /// budgetId, timeoutMs | deadlineAt}`); `None` without a ledger.
41    fn reserve(&self, request: &Value) -> Result<Option<Value>, String>;
42    /// Settle a reservation with the request's status and provider-reported usage (`{inputTokens, outputTokens,
43    /// totalTokens, cost}`).
44    fn settle(&self, reservation: &Value, status: &str, usage: Option<&Value>);
45    /// The `rlm` tool on `lane` (its own kernel) for a frame that runs code.
46    fn code_tool(&self, lane: &str) -> Result<Arc<dyn AgentTool>, String>;
47    /// Close `lane` once its frame ended.
48    fn close_lane(&self, lane: &str);
49    /// The frame running code on `lane` talks to `model`; `meter` reads its conversation's context use (the
50    /// newest answer's usage plus estimates after it), for the context reminder of the lane's cells. Ends with
51    /// [`FrameTasks::close_lane`].
52    fn frame_context(&self, lane: &str, model: &ai::Model, meter: FrameMeter) {
53        let _ = (lane, model, meter);
54    }
55    /// Count one model response of a frame toward its root's tree limits (`recordTurn(usageRootForLane(lane,
56    /// runId))` of every lane's `after_response`).
57    fn charge_turn(&self, root: Option<&str>, reply: &AssistantMessage) {
58        let _ = (root, reply);
59    }
60    /// The tree-limit gate of a request of the frame running code on `lane` (`before_request`).
61    fn gate_request(&self, lane: &str, root: Option<&str>) -> FrameGate {
62        let _ = (lane, root);
63        FrameGate::Allow
64    }
65    /// The tree-limit gate of a cell of the frame running code on `lane` (`before_tool`): `(reason, terminate)`
66    /// refuses it.
67    fn gate_tool(&self, lane: &str, root: Option<&str>) -> Option<(String, bool)> {
68        let _ = (lane, root);
69        None
70    }
71}
72
73/// What the tree-limit gate decided for a frame's request.
74#[derive(Clone, Debug, PartialEq, Eq)]
75pub enum FrameGate {
76    Allow,
77    /// The run's final request: tool choice `none`, with the note appended to its context.
78    Final(String),
79    /// Refused unsent with the reason.
80    Block(String),
81}
82
83/// A frame conversation's context use, once it has one.
84pub type FrameMeter = Arc<dyn Fn() -> Option<u64> + Send + Sync>;
85
86/// One `rlm-frame@1` task run, as the task host hands it over (`NativeFrameRun`).
87pub struct FrameExecution {
88    pub task_id: String,
89    pub input: Value,
90    pub lane: String,
91    pub usage_root: Option<String>,
92    pub deadline_at: Option<f64>,
93    pub timeout_ms: u64,
94    /// Aborted when the task is cancelled or times out.
95    pub signal: Signal,
96}
97
98/// Where a new call's budget sits: its depth, the frame it was made from, and its pool limits.
99pub(crate) struct Admission {
100    pub depth: u64,
101    parent: Option<Arc<Live>>,
102    caller_task: Option<String>,
103    calls: Option<u64>,
104    tokens: Option<u64>,
105    default_tokens: bool,
106}
107
108/// A frame that has not finished: its state, the call it belongs to, and its outcome once known.
109pub(crate) struct Live {
110    id: String,
111    node: usize,
112    frame: Mutex<Frame>,
113    budget: Arc<Budget>,
114    options: Arc<Options>,
115    /// The model the frame calls: the session's until admission resolves the model the frame names.
116    engine: Mutex<Engine>,
117    outcome: Mutex<Option<Outcome>>,
118    finalized: AtomicBool,
119}
120
121impl Live {
122    fn engine(&self) -> Engine {
123        lock(&self.engine).clone()
124    }
125
126    fn outcome(&self) -> Option<Outcome> {
127        lock(&self.outcome).clone()
128    }
129
130    fn set_outcome(&self, outcome: Outcome) {
131        *lock(&self.outcome) = Some(outcome);
132    }
133
134    fn spec(&self) -> Spec<'_> {
135        Spec {
136            contract: self.options.contract.as_ref(),
137            max_repairs: self.options.max_repairs,
138            depth: self.options.depth,
139            model: self.options.model.as_deref(),
140        }
141    }
142}
143
144/// The frames of one session; clones of the `Arc` share them.
145pub struct FrameSession {
146    ledger: Mutex<Ledger>,
147    labels: Mutex<HashMap<String, String>>,
148    traces: Mutex<Traces>,
149    /// Frames admitted and not yet finished, in admission order.
150    live: Mutex<Vec<Arc<Live>>>,
151    /// The frame each running frame task's lane acts for.
152    lanes: Mutex<HashMap<String, Arc<Live>>>,
153    tasks: OnceLock<Arc<dyn FrameTasks>>,
154}
155
156impl FrameSession {
157    /// A session whose traces live in `store`; frames run on their callers' threads until [`Self::set_tasks`].
158    pub fn new(store: Arc<dyn TraceStore>) -> Arc<Self> {
159        Arc::new(FrameSession {
160            ledger: Mutex::new(Ledger::default()),
161            labels: Mutex::new(HashMap::new()),
162            traces: Mutex::new(Traces::new(store)),
163            live: Mutex::new(Vec::new()),
164            lanes: Mutex::new(HashMap::new()),
165            tasks: OnceLock::new(),
166        })
167    }
168
169    /// Admit every later frame through `tasks` (set once).
170    pub fn set_tasks(&self, tasks: Arc<dyn FrameTasks>) {
171        let _ = self.tasks.set(tasks);
172    }
173
174    pub(crate) fn has_tasks(&self) -> bool {
175        self.tasks.get().is_some()
176    }
177
178    pub(crate) fn label(&self, digest: &str) -> Option<String> {
179        lock(&self.labels).get(digest).cloned()
180    }
181
182    pub(crate) fn set_label(&self, digest: &str, label: String) {
183        lock(&self.labels).insert(digest.to_string(), label);
184    }
185
186    /// `options`' budget placement: a request from a running frame's lane is nested under that frame, with at
187    /// most one level less depth (default: all of it); a root request defaults to depth 1. A top-level map without
188    /// a token limit takes `default_tokens`.
189    pub(crate) fn admission(
190        &self,
191        lane: &str,
192        requested_depth: Option<u64>,
193        calls: Option<u64>,
194        tokens: Option<u64>,
195        default_tokens: Option<u64>,
196    ) -> Admission {
197        let parent = lock(&self.lanes).get(lane).cloned();
198        let ceiling = parent
199            .as_ref()
200            .map_or(MAX_DEPTH, |frame| frame.options.depth.saturating_sub(1));
201        let depth = requested_depth
202            .unwrap_or(if parent.is_some() { ceiling } else { 1 })
203            .min(ceiling);
204        let caller_task = match &parent {
205            Some(frame) => lock(&frame.frame).task_id.clone(),
206            None => self.tasks.get().and_then(|tasks| tasks.caller_task(lane)),
207        };
208        let use_default = tokens.is_none() && parent.is_none() && default_tokens.is_some();
209        Admission {
210            depth,
211            calls,
212            tokens: if use_default { default_tokens } else { tokens },
213            default_tokens: use_default,
214            parent,
215            caller_task,
216        }
217    }
218
219    /// The call's budget node, under its parent frame's node when nested.
220    pub(crate) fn budget(&self, admission: &Admission) -> Budget {
221        let parent_node = admission.parent.as_ref().map(|frame| frame.node);
222        let node = lock(&self.ledger).add(parent_node, admission.calls, admission.tokens);
223        Budget {
224            id: format!("budget-{}", jsrt::random::uuid_v4()),
225            node,
226            depth: admission.depth,
227            parent: admission.parent.as_ref().map(|frame| frame.id.clone()),
228            default_tokens: admission.default_tokens,
229        }
230    }
231
232    /// `newFrame`: a frame under `budget`. Its first request's call is claimed now, in admission order, so a shared
233    /// pool decides deterministically which frames run; without depth or a call it is already incomplete.
234    #[allow(clippy::too_many_arguments)]
235    pub(crate) fn admit(
236        &self,
237        budget: &Arc<Budget>,
238        options: &Arc<Options>,
239        engine: &Engine,
240        admission: &Admission,
241        task: &str,
242        views: Vec<View>,
243        kind: &'static str,
244        batch: usize,
245    ) -> Arc<Live> {
246        let mut ledger = lock(&self.ledger);
247        let mut frame = Frame::new(&mut ledger, budget, task, views);
248        frame.kind = kind;
249        frame.batch = batch;
250        frame.caller_task = admission.caller_task.clone();
251        frame.parent_frame = admission.parent.as_ref().map(|frame| frame.id.clone());
252        let mut outcome = None;
253        if options.depth < 1 {
254            outcome = Some(Outcome::Incomplete(
255                "depth_exhausted",
256                "this frame's budget allows no nested frames".into(),
257            ));
258        } else if ledger.hold(budget.node, Kind::Calls, 1) {
259            frame.call_held = true;
260        } else {
261            outcome = Some(Outcome::Incomplete(
262                "budget_exhausted",
263                "no call left in the budget for this frame".into(),
264            ));
265        }
266        drop(ledger);
267        let live = Arc::new(Live {
268            id: frame.id.clone(),
269            node: frame.node,
270            frame: Mutex::new(frame),
271            budget: budget.clone(),
272            options: options.clone(),
273            engine: Mutex::new(engine.clone()),
274            outcome: Mutex::new(outcome),
275            finalized: AtomicBool::new(false),
276        });
277        lock(&self.live).push(live.clone());
278        live
279    }
280
281    /// Release the call claimed at admission if no request used it.
282    fn release_call(&self, live: &Live) {
283        let mut ledger = lock(&self.ledger);
284        let mut frame = lock(&live.frame);
285        if frame.call_held {
286            frame.call_held = false;
287            ledger.release(live.node, Kind::Calls, 1);
288        }
289    }
290
291    /// `runFrame`: admit the frame as an `rlm-frame@1` task under the caller, wait for its result, and cancel it with
292    /// the calling cell (without a task host: run it here). Always ends with the frame finalized.
293    pub(crate) fn run_frame(&self, live: &Arc<Live>, lane: &str, cell: &Signal) {
294        if live.outcome().is_none() {
295            let outcome = self.start(live, lane, cell);
296            if let Some(outcome) = outcome
297                && live.outcome().is_none()
298            {
299                live.set_outcome(outcome);
300            }
301        }
302        self.release_call(live);
303        self.finalize(live);
304    }
305
306    fn start(&self, live: &Arc<Live>, lane: &str, cell: &Signal) -> Option<Outcome> {
307        if cell.is_aborted() {
308            return Some(Outcome::Error("cancelled: the calling cell was aborted".into()));
309        }
310        // A frame naming another model calls it through the same provider dispatch (the session's model
311        // resolution); one this host cannot resolve is refused.
312        match live.engine().for_model(live.options.model.as_deref()) {
313            Ok(engine) => *lock(&live.engine) = engine,
314            Err(refused) => return Some(Outcome::Error(refused)),
315        }
316        let Some(tasks) = self.tasks.get() else {
317            return Some(self.converse(live, cell, None));
318        };
319        let task = lock(&live.frame).task.clone();
320        let input = json!({"frame": live.id, "task": bounded(&task, 200)});
321        let timeout_ms = live.options.timeout.as_millis() as u64;
322        let task_id = match tasks.spawn(lane, input, live.options.model.as_deref(), timeout_ms) {
323            Ok(id) => id,
324            Err(error) => return Some(Outcome::Error(format!("Admission refused: {error}"))),
325        };
326        lock(&live.frame).task_id = Some(task_id.clone());
327        let result = tasks.result(lane, &task_id, cell);
328        Some(if result["status"] == "succeeded" {
329            Outcome::Complete(result.get("value").cloned().unwrap_or(Value::Null))
330        } else {
331            let status = result["status"].as_str().unwrap_or("failed");
332            let error = result["error"].as_str().unwrap_or("frame did not finish");
333            Outcome::Error(format!("{status}: {error}"))
334        })
335    }
336
337    /// The task host's frame runner (`execute`): run the frame task `run` on this thread. The frame's value
338    /// succeeds the task; an `Incomplete` or an error fails it, and an aborted run is cancelled.
339    pub fn execute(self: &Arc<Self>, run: FrameExecution) -> Result<Value, String> {
340        let id = run.input["frame"].as_str().unwrap_or_default();
341        let live = lock(&self.live).iter().find(|live| live.id == id).cloned();
342        let Some(live) = live else {
343            return Err("rlm-frame tasks are started only by rlm.infer and rlm.map".into());
344        };
345        {
346            let mut frame = lock(&live.frame);
347            frame.task_id = Some(run.task_id.clone());
348            frame.lane = Some(run.lane.clone());
349        }
350        lock(&self.lanes).insert(run.lane.clone(), live.clone());
351        let outcome = if live.options.depth > 1 {
352            let outcome = self.converse_with_code(&live, &run);
353            if let Some(tasks) = self.tasks.get() {
354                tasks.close_lane(&run.lane);
355            }
356            outcome
357        } else {
358            self.converse(&live, &run.signal, Some(&run))
359        };
360        lock(&self.lanes).remove(&run.lane);
361        self.release_call(&live);
362        live.set_outcome(outcome.clone());
363        match outcome {
364            Outcome::Complete(value) => Ok(value),
365            _ if run.signal.is_aborted() => Err("cancelled".into()),
366            Outcome::Incomplete(reason, detail) => Err(format!("Incomplete ({reason}): {detail}")),
367            Outcome::Error(error) => Err(error),
368        }
369    }
370
371    /// Reserve attempt `attempt` of a task-run frame in the usage ledger.
372    fn reserve(&self, live: &Live, run: Option<&FrameExecution>, attempt: u64) -> Result<Option<Value>, String> {
373        let (Some(run), Some(tasks)) = (run, self.tasks.get()) else {
374            return Ok(None);
375        };
376        let mut request = json!({
377            "kind": "model",
378            "taskId": run.task_id,
379            "parentTaskId": run.task_id,
380            "requestKey": format!("{}:frame:{attempt}", run.task_id),
381            "budgetId": live.budget.id,
382        });
383        if let Some(root) = &run.usage_root {
384            request["rootId"] = json!(root);
385        }
386        match run.deadline_at {
387            Some(deadline) => request["deadlineAt"] = json!(deadline),
388            None => request["timeoutMs"] = json!(run.timeout_ms),
389        }
390        tasks.reserve(&request)
391    }
392
393    fn settle(&self, reservation: Option<Value>, status: &str, usage: Option<Value>) {
394        if let (Some(reservation), Some(tasks)) = (reservation, self.tasks.get()) {
395            tasks.settle(&reservation, status, usage.as_ref());
396        }
397    }
398
399    /// The frame's code-free conversation: the seeded prompt, then re-asks within budget until the contract holds.
400    /// A request the provider does not answer before the deadline (or before the frame is cancelled) is refunded.
401    fn converse(&self, live: &Live, signal: &Signal, run: Option<&FrameExecution>) -> Outcome {
402        let options = &live.options;
403        let engine = &live.engine();
404        let deadline = Instant::now() + options.timeout;
405        let system = frame_system(options);
406        let mut conversation_chars = jsrt::utf16::len(&system) as u64;
407        let mut message = {
408            let mut frame = lock(&live.frame);
409            frame.model = Some(engine.own_model());
410            frame_prompt(&frame.task, &frame.views, options.contract.as_ref())
411        };
412        let mut history: Vec<Message> = Vec::new();
413        for attempt in 0u64.. {
414            if signal.is_aborted() || Instant::now() >= deadline {
415                return Outcome::Error("cancelled".into());
416            }
417            let estimate =
418                (conversation_chars + jsrt::utf16::len(&message) as u64).div_ceil(3) + REQUEST_OVERHEAD_TOKENS;
419            if let Some(error) = self.window_error(engine, estimate) {
420                return Outcome::Error(error);
421            }
422            let Some((held, tranche)) = self.claim(live, estimate) else {
423                return Outcome::Incomplete("budget_exhausted", exhausted_detail(attempt));
424            };
425            conversation_chars += jsrt::utf16::len(&message) as u64;
426            history.push(user_message(&message));
427            let reservation = match self.reserve(live, run, attempt) {
428                Ok(reservation) => reservation,
429                Err(error) => {
430                    self.release(live, held);
431                    return Outcome::Error(error);
432                }
433            };
434            let response = engine.request(&system, &history, tranche, options, deadline, signal);
435            let charged = match &response {
436                Ok(Some(reply)) => charged_tokens(&reply.usage),
437                _ => 0,
438            };
439            self.settle_request(
440                live,
441                held,
442                tranche,
443                response.as_ref().ok().and_then(Option::as_ref),
444                charged,
445            );
446            let (status, usage) = match &response {
447                Ok(Some(reply)) => ("succeeded", measured(std::slice::from_ref(reply))),
448                Ok(None) => ("cancelled", None),
449                Err(_) if signal.is_aborted() => ("cancelled", None),
450                Err(_) => ("failed", None),
451            };
452            self.settle(reservation, status, usage);
453            let reply = match response {
454                Ok(Some(reply)) => reply,
455                Ok(None) => return Outcome::Error("cancelled".into()),
456                Err(error) => return Outcome::Error(format!("frame request failed: {}", Value::String(error))),
457            };
458            add_measured(live, std::slice::from_ref(&reply));
459            if let (Some(run), Some(tasks)) = (run, self.tasks.get()) {
460                tasks.charge_turn(run.usage_root.as_deref(), &reply);
461            }
462            if signal.is_aborted() {
463                return Outcome::Error("cancelled".into());
464            }
465            if let Some(outcome) = failed_reply(&reply) {
466                return outcome;
467            }
468            let text = reply_text(&reply);
469            history.push(Message::Assistant(reply));
470            conversation_chars += jsrt::utf16::len(&text) as u64;
471            match self.judge(live, attempt, text) {
472                Ok(outcome) => return outcome,
473                Err(next) => message = next,
474            }
475        }
476        unreachable!("the attempt counter is unbounded")
477    }
478
479    /// A frame with depth: its conversation is an agent on the task's lane with the `rlm` cell as its only tool.
480    /// Each attempt is one prompt; its first request uses the hold claimed for it, every further request (a tool
481    /// round) claims its own or is refused unsent, and each response is charged as it arrives.
482    fn converse_with_code(self: &Arc<Self>, live: &Arc<Live>, run: &FrameExecution) -> Outcome {
483        let Some(tasks) = self.tasks.get() else {
484            return Outcome::Error("frames that run code need a task host".into());
485        };
486        let options = &live.options;
487        let engine = &live.engine();
488        let tool = match tasks.code_tool(&run.lane) {
489            Ok(tool) => tool,
490            Err(error) => return Outcome::Error(error),
491        };
492        let deadline = Instant::now() + options.timeout;
493        let system = frame_system(options);
494        let mut message = {
495            let mut frame = lock(&live.frame);
496            frame.model = Some(engine.own_model());
497            frame_prompt(&frame.task, &frame.views, options.contract.as_ref())
498        };
499        let pending: Arc<Mutex<Option<(u64, u64)>>> = Arc::default();
500        let context_use: Arc<Mutex<Option<u64>>> = Arc::default();
501        let meter_use = context_use.clone();
502        tasks.frame_context(&run.lane, &engine.model, Arc::new(move || *lock(&meter_use)));
503        let stream = BudgetedStream {
504            session: self.clone(),
505            live: live.clone(),
506            pending: pending.clone(),
507            deadline,
508            signal: run.signal.clone(),
509            context_use,
510            tasks: tasks.clone(),
511            lane: run.lane.clone(),
512            root: run.usage_root.clone(),
513        };
514        let mut agent_options = AgentOptions::new(engine.model.clone());
515        agent_options.system_prompt = Some(system.clone());
516        agent_options.tools = vec![tool];
517        // A spent tree's cells are refused (`before_tool` of the frame's lane).
518        let (gate_tasks, gate_lane, gate_root) = (tasks.clone(), run.lane.clone(), run.usage_root.clone());
519        agent_options.before_tool_call = Some(Arc::new(move |_, _| {
520            let decision = gate_tasks.gate_tool(&gate_lane, gate_root.as_deref());
521            Box::pin(async move {
522                Ok(decision.map(|(reason, terminate)| agent::BeforeToolCallResult {
523                    block: true,
524                    reason: Some(reason),
525                    terminate: Some(terminate),
526                }))
527            })
528        }));
529        agent_options.thinking_level = options
530            .thinking
531            .clone()
532            .filter(|_| engine.model.reasoning)
533            .unwrap_or_else(|| "off".into());
534        let mut agent = Agent::new(agent_options, Box::new(stream));
535        let runtime = match tokio::runtime::Builder::new_current_thread().enable_all().build() {
536            Ok(runtime) => runtime,
537            Err(error) => return Outcome::Error(error.to_string()),
538        };
539        // The frame's cancellation aborts the agent's run (an already cancelled frame stops at its first attempt).
540        let control = agent.control();
541        let listener = run.signal.on_abort(move |_| control.abort());
542        let mut conversation_chars = jsrt::utf16::len(&system) as u64;
543        let outcome = 'attempts: {
544            for attempt in 0u64.. {
545                if run.signal.is_aborted() || Instant::now() >= deadline {
546                    break 'attempts Outcome::Error("cancelled".into());
547                }
548                let estimate =
549                    (conversation_chars + jsrt::utf16::len(&message) as u64).div_ceil(3) + REQUEST_OVERHEAD_TOKENS;
550                if let Some(error) = self.window_error(engine, estimate) {
551                    break 'attempts Outcome::Error(error);
552                }
553                let Some(hold) = self.claim(live, estimate) else {
554                    break 'attempts Outcome::Incomplete("budget_exhausted", exhausted_detail(attempt));
555                };
556                *lock(&pending) = Some(hold);
557                conversation_chars += jsrt::utf16::len(&message) as u64;
558                let reservation = match self.reserve(live, Some(run), attempt) {
559                    Ok(reservation) => reservation,
560                    Err(error) => {
561                        if let Some((held, _)) = lock(&pending).take() {
562                            self.release(live, held);
563                        }
564                        break 'attempts Outcome::Error(error);
565                    }
566                };
567                let before = agent.messages.len();
568                runtime.block_on(agent.prompt_text(&message, Vec::new()));
569                // A hold the prompt never used is released (`settlePrompt`).
570                if let Some((held, _)) = lock(&pending).take() {
571                    self.release(live, held);
572                }
573                let replies: Vec<AssistantMessage> = agent.messages[before..]
574                    .iter()
575                    .filter_map(|message| match message {
576                        Message::Assistant(reply) => Some(reply.clone()),
577                        _ => None,
578                    })
579                    .collect();
580                let status = if run.signal.is_aborted() {
581                    "cancelled"
582                } else {
583                    "succeeded"
584                };
585                self.settle(reservation, status, measured(&replies));
586                add_measured(live, &replies);
587                if run.signal.is_aborted() {
588                    break 'attempts Outcome::Error("cancelled".into());
589                }
590                if let Some(exhausted) = lock(&live.frame).exhausted.clone() {
591                    break 'attempts Outcome::Incomplete("budget_exhausted", exhausted);
592                }
593                let Some(reply) = replies.last() else {
594                    break 'attempts Outcome::Error("frame run did not complete: failed".into());
595                };
596                if let Some(outcome) = failed_reply(reply) {
597                    break 'attempts outcome;
598                }
599                let text = reply_text(reply);
600                conversation_chars += jsrt::utf16::len(&text) as u64;
601                match self.judge(live, attempt, text) {
602                    Ok(outcome) => break 'attempts outcome,
603                    Err(next) => message = next,
604                }
605            }
606            unreachable!("the attempt counter is unbounded")
607        };
608        if let Some(listener) = listener {
609            run.signal.forget(listener);
610        }
611        outcome
612    }
613
614    /// The request's window check: an estimate the model's context window cannot hold is an error.
615    fn window_error(&self, engine: &Engine, estimate: u64) -> Option<String> {
616        let window = engine.model.context_window;
617        (window > 0 && estimate + MIN_TRANCHE > window).then(|| {
618            format!(
619                "context too large for {}/{}: about {estimate} tokens against a {window}-token window; pass smaller slices",
620                engine.model.provider, engine.model.id
621            )
622        })
623    }
624
625    /// Claim the next request's call (the admission call first) and tokens.
626    fn claim(&self, live: &Live, estimate: u64) -> Option<(u64, u64)> {
627        let mut ledger = lock(&self.ledger);
628        let mut frame = lock(&live.frame);
629        let held = frame.call_held;
630        frame.call_held = false;
631        let model_max = live.engine().model.max_tokens;
632        ledger.claim(live.node, estimate, model_max, held)
633    }
634
635    /// Release a hold no request used.
636    fn release(&self, live: &Live, held: u64) {
637        let mut ledger = lock(&self.ledger);
638        ledger.release(live.node, Kind::Calls, 1);
639        ledger.release(live.node, Kind::Tokens, held);
640    }
641
642    /// Settle a sent request: release its hold and charge what the provider reported (a failed or timed-out request
643    /// is refunded). One lock, so a concurrent frame never sees the hold released before the charge lands.
644    fn settle_request(&self, live: &Live, held: u64, tranche: u64, reply: Option<&AssistantMessage>, charged: u64) {
645        let mut ledger = lock(&self.ledger);
646        ledger.release(live.node, Kind::Calls, 1);
647        ledger.release(live.node, Kind::Tokens, held);
648        if reply.is_some() {
649            ledger.charge(live.node, Kind::Calls, 1);
650            ledger.charge(live.node, Kind::Tokens, charged);
651        }
652        drop(ledger);
653        if let Some(reply) = reply {
654            let mut frame = lock(&live.frame);
655            if frame.requests.len() < trace::TRACE_REQUEST_LIMIT {
656                frame.requests.push(json!({
657                    "at": jsrt::date::now_ms() as u64, "tranche": tranche, "held": held, "charged": charged,
658                    "status": serde_json::to_value(reply.stop_reason).unwrap_or(Value::Null),
659                }));
660            }
661        }
662    }
663
664    /// Record an answer and judge it against the contract: the outcome, or the re-ask message.
665    fn judge(&self, live: &Live, attempt: u64, text: String) -> Result<Outcome, String> {
666        let options = &live.options;
667        let output = bounded(&text, OUTPUT_PREVIEW_CHARS);
668        let mut frame = lock(&live.frame);
669        frame.outputs.push(output.clone());
670        if frame.outputs.len() > LAST_OUTPUTS {
671            frame.outputs.remove(0);
672        }
673        let record = |error: Option<&str>| {
674            let mut record = json!({"attempt": attempt, "at": jsrt::date::now_ms() as u64, "output": output});
675            if let Some(error) = error {
676                record["error"] = error.into();
677            }
678            record
679        };
680        let error = if jsrt::string::trim(&text).is_empty() {
681            "the reply is empty".to_string()
682        } else {
683            let parsed = match &options.contract {
684                None => Ok(Value::String(text)),
685                Some(contract) => parse_reply(&text, contract),
686            };
687            match parsed {
688                Ok(value) => {
689                    frame.attempts.push(record(None));
690                    return Ok(Outcome::Complete(value));
691                }
692                Err(error) => error,
693            }
694        };
695        frame.attempts.push(record(Some(&error)));
696        if attempt >= options.max_repairs {
697            return Ok(Outcome::Incomplete(
698                "contract_unmet",
699                format!("no valid answer after {} attempt(s): {error}", attempt + 1),
700            ));
701        }
702        let fallback = json!({"type": "string", "minLength": 1});
703        Err(repair_prompt(&error, options.contract.as_ref().unwrap_or(&fallback)))
704    }
705
706    /// `finalize`: the frame ends; its trace and summary are written and it leaves the running frames.
707    fn finalize(&self, live: &Arc<Live>) {
708        if live.finalized.swap(true, Ordering::SeqCst) {
709            return;
710        }
711        lock(&live.frame).ended_at = Some(jsrt::date::now_ms() as u64);
712        let outcome = live.outcome();
713        let (document, summary) = {
714            let mut ledger = lock(&self.ledger);
715            let frame = lock(&live.frame);
716            let spec = live.spec();
717            (
718                trace::trace(&mut ledger, &live.budget, &frame, &spec, outcome.as_ref()),
719                trace::summary(&ledger, &live.budget, &frame, outcome.as_ref()),
720            )
721        };
722        lock(&self.traces).write(&live.id, &document, summary);
723        lock(&self.live).retain(|other| !Arc::ptr_eq(other, live));
724        // Keep the metadata for the observation; drop the materialized text.
725        for view in &mut lock(&live.frame).views {
726            view.text = String::new();
727        }
728    }
729
730    /// The frame's observation: its outcome, trace id, task id, what it spent and what remains.
731    pub(crate) fn observation(&self, live: &Live) -> Value {
732        let outcome = live
733            .outcome()
734            .unwrap_or_else(|| Outcome::Error("frame did not run".into()));
735        let mut ledger = lock(&self.ledger);
736        let frame = lock(&live.frame);
737        let mut out = Map::new();
738        let complete = matches!(outcome, Outcome::Complete(_));
739        match outcome {
740            Outcome::Complete(value) => {
741                out.insert("status".into(), "complete".into());
742                out.insert("value".into(), value);
743            }
744            Outcome::Incomplete(reason, detail) => {
745                out.insert("status".into(), "incomplete".into());
746                out.insert("reason".into(), reason.into());
747                let detail = if reason == "budget_exhausted" && live.budget.default_tokens {
748                    format!(
749                        "{detail} (rlm.map's default budget is {} tokens; pass budget=Budget(tokens=...) to allow more)",
750                        ledger.nodes[live.budget.node].tokens.limit.unwrap_or_default()
751                    )
752                } else {
753                    detail
754                };
755                out.insert("detail".into(), detail.into());
756            }
757            Outcome::Error(error) => {
758                out.insert("status".into(), "error".into());
759                out.insert("error".into(), error.into());
760            }
761        }
762        out.insert("trace_id".into(), frame.id.clone().into());
763        if let Some(task) = &frame.task_id {
764            out.insert("task_id".into(), task.clone().into());
765        }
766        out.insert("spent".into(), frame.spent(&ledger));
767        out.insert("remaining".into(), frame.remaining(&mut ledger, live.options.depth));
768        if !complete {
769            out.insert("last_outputs".into(), frame.outputs.clone().into());
770        }
771        Value::Object(out)
772    }
773
774    /// The budget reply of an `rlm.map`: the shared node's snapshot and what remains of it.
775    pub(crate) fn budget_reply(&self, budget: &Budget) -> Value {
776        let mut ledger = lock(&self.ledger);
777        let mut reply = budget.snapshot(&ledger);
778        reply["remaining"] = json!({
779            "calls": ledger.available(budget.node, Kind::Calls),
780            "tokens": ledger.available(budget.node, Kind::Tokens),
781            "depth": budget.depth,
782        });
783        reply
784    }
785
786    pub(crate) fn measured(&self, live: &Live) -> (u64, u64, f64) {
787        lock(&live.frame).measured
788    }
789
790    /// `rlm.frames`: a trace by id or the newest summaries, running frames included.
791    pub(crate) fn list(&self, payload: &Map<String, Value>) -> Result<Value, String> {
792        let live: Vec<Arc<Live>> = lock(&self.live).clone();
793        let view = |live: &Live, full: bool| {
794            let outcome = live.outcome();
795            let mut ledger = lock(&self.ledger);
796            let frame = lock(&live.frame);
797            if full {
798                trace::trace(&mut ledger, &live.budget, &frame, &live.spec(), outcome.as_ref())
799            } else {
800                trace::summary(&ledger, &live.budget, &frame, outcome.as_ref())
801            }
802        };
803        let summaries: Vec<Value> = live.iter().map(|frame| view(frame, false)).collect();
804        let by_id = |id: &str| live.iter().find(|frame| frame.id == id).map(|frame| view(frame, true));
805        lock(&self.traces).list(payload, &by_id, &summaries)
806    }
807}
808
809fn exhausted_detail(attempt: u64) -> String {
810    if attempt == 0 {
811        "the budget cannot cover the first request".to_string()
812    } else {
813        format!("the budget ran out after {attempt} attempt(s) without a valid answer")
814    }
815}
816
817fn user_message(text: &str) -> Message {
818    Message::User(ai::UserMessage {
819        content: ai::UserMessageContent::Text(text.to_string()),
820        timestamp: 0,
821    })
822}
823
824fn reply_text(reply: &AssistantMessage) -> String {
825    reply
826        .content
827        .iter()
828        .filter_map(|part| match part {
829            AssistantContent::Text(text) => Some(text.text.as_str()),
830            _ => None,
831        })
832        .collect::<Vec<_>>()
833        .join("\n")
834}
835
836/// A reply that ends the frame: the provider's usage limit (Incomplete), or a run that failed or was aborted.
837fn failed_reply(reply: &AssistantMessage) -> Option<Outcome> {
838    let limited = reply
839        .diagnostics
840        .iter()
841        .flatten()
842        .any(|d| d.kind == "provider_usage_limit");
843    if limited {
844        let detail = reply
845            .error_message
846            .clone()
847            .unwrap_or("the provider's usage limit is reached".into());
848        return Some(Outcome::Incomplete("usage_limit", detail));
849    }
850    if matches!(reply.stop_reason, StopReason::Error | StopReason::Aborted) {
851        let status = if reply.stop_reason == StopReason::Aborted {
852            "aborted"
853        } else {
854            "failed"
855        };
856        let error = reply.error_message.clone().unwrap_or_default();
857        return Some(Outcome::Error(format!(
858            "frame run did not complete: {status} ({})",
859            json!({"message": error})
860        )));
861    }
862    None
863}
864
865/// `measurement`: the provider-reported usage of a prompt's replies (none when a reply carries none, e.g. a
866/// request that failed before any response).
867fn measured(replies: &[AssistantMessage]) -> Option<Value> {
868    if replies.is_empty() || replies.iter().any(|reply| reply.stop_reason == StopReason::Error) {
869        return None;
870    }
871    let (mut input, mut output, mut total, mut cost) = (0u64, 0u64, 0u64, 0f64);
872    for reply in replies {
873        input += reply.usage.input;
874        output += reply.usage.output;
875        total += reply.usage.total_tokens;
876        cost += reply.usage.cost.total;
877    }
878    Some(json!({"inputTokens": input, "outputTokens": output, "totalTokens": total, "cost": cost}))
879}
880
881fn add_measured(live: &Live, replies: &[AssistantMessage]) {
882    let mut frame = lock(&live.frame);
883    for reply in replies {
884        frame.measured.0 += reply.usage.input;
885        frame.measured.1 += reply.usage.output;
886        frame.measured.2 += reply.usage.cost.total;
887    }
888}
889
890/// An assistant message the run records without sending anything (a refused or cancelled request).
891fn unsent(engine: &Engine, stop_reason: StopReason, error: String) -> AssistantMessage {
892    AssistantMessage {
893        content: Vec::new(),
894        api: engine.model.api.clone(),
895        provider: engine.model.provider.clone(),
896        model: engine.model.id.clone(),
897        response_model: None,
898        response_id: None,
899        provider_thinking_level: None,
900        diagnostics: None,
901        usage: ai::Usage::default(),
902        stop_reason,
903        deferred: None,
904        error_message: Some(error),
905        raw_stop_reason: None,
906        end_turn: None,
907        provider_metadata: None,
908        timestamp: jsrt::date::now_ms(),
909    }
910}
911
912/// A code frame's model requests (`before_request`, `before_payload`, `after_response` of inference.ts): each one
913/// uses the prompt's pre-claimed hold or claims its own (refused unsent when the budget cannot cover it), is capped
914/// at its tranche, and is charged what the provider reports.
915struct BudgetedStream {
916    session: Arc<FrameSession>,
917    live: Arc<Live>,
918    pending: Arc<Mutex<Option<(u64, u64)>>>,
919    deadline: Instant,
920    signal: Signal,
921    /// The conversation's context use after the newest answer (`estimateContextTokens` over the request and it).
922    context_use: Arc<Mutex<Option<u64>>>,
923    /// The tree-limit gate and turn charge of the frame's lane, under its usage root.
924    tasks: Arc<dyn FrameTasks>,
925    lane: String,
926    root: Option<String>,
927}
928
929impl StreamFn for BudgetedStream {
930    fn stream<'a>(
931        &'a self,
932        _model: &'a ai::Model,
933        context: &'a ai::TranscriptContext,
934        options: &'a ai::SimpleStreamOptions,
935        sink: &'a mut ai::EventSink<'_>,
936    ) -> ai::http::BoxFuture<'a, AssistantMessage> {
937        Box::pin(async move {
938            let session = &self.session;
939            let live = &self.live;
940            let engine = &live.engine();
941            // A spent tree: one final request without tools, then nothing more.
942            let mut options = options.clone();
943            let gated;
944            let context = match self.tasks.gate_request(&self.lane, self.root.as_deref()) {
945                FrameGate::Allow => context,
946                FrameGate::Block(reason) => {
947                    if let Some((held, _)) = lock(&self.pending).take() {
948                        session.release(live, held);
949                    }
950                    return unsent(engine, StopReason::Error, reason);
951                }
952                FrameGate::Final(note) => {
953                    options.tool_choice = Some("none".into());
954                    let mut messages = context.messages.clone();
955                    messages.push(user_message(&note));
956                    gated = ai::TranscriptContext { messages };
957                    &gated
958                }
959            };
960            let hold = lock(&self.pending).take().or_else(|| {
961                let estimate = serde_json::to_string(context)
962                    .map_or(0, |text| text.len() as u64)
963                    .div_ceil(4)
964                    + REQUEST_OVERHEAD_TOKENS;
965                session.claim(live, estimate)
966            });
967            let Some((held, tranche)) = hold else {
968                lock(&live.frame).exhausted = Some(FOLLOW_UP_EXHAUSTED.into());
969                return unsent(
970                    engine,
971                    StopReason::Error,
972                    format!("Inference frame {}: {FOLLOW_UP_EXHAUSTED}", live.id),
973                );
974            };
975            engine.prepare(&mut options, tranche, live.options.contract.as_ref());
976            let response = engine
977                .stream(context, &options, sink, self.deadline, &self.signal)
978                .await;
979            let charged = match &response {
980                Ok(Some(reply)) => charged_tokens(&reply.usage),
981                _ => 0,
982            };
983            session.settle_request(
984                live,
985                held,
986                tranche,
987                response.as_ref().ok().and_then(Option::as_ref),
988                charged,
989            );
990            if let Ok(Some(reply)) = &response {
991                self.tasks.charge_turn(self.root.as_deref(), reply);
992                let mut messages = context.messages.clone();
993                messages.push(Message::Assistant(reply.clone()));
994                *lock(&self.context_use) = Some(ai::transcript::estimate_context_tokens(&messages).tokens);
995            }
996            match response {
997                Ok(Some(reply)) => reply,
998                Ok(None) => unsent(engine, StopReason::Aborted, "cancelled".into()),
999                Err(error) => unsent(engine, StopReason::Error, error),
1000            }
1001        })
1002    }
1003}