Skip to main content

inference/
trace.rs

1//! Frame state, traces and `rlm.frames` (inference.ts `trace`, `summary`, `writeTrace`, `list`): every finished frame
2//! leaves a trace document (task, views, contract, budget, attempts, requests) and a summary in a bounded index,
3//! both kept in a [`TraceStore`] (the session values `ultron.rlm.frames/<id>` and `ultron.rlm.frames/index`), so
4//! `rlm.frames` reads them in any later cell, and after the session reopens.
5
6use super::{Kind, Ledger, OUTPUT_PREVIEW_CHARS, Outcome, View, bounded, fields, optional_integer};
7use serde_json::{Map, Value, json};
8use std::collections::HashMap;
9use std::sync::Mutex;
10
11const TRACE_INDEX_LIMIT: usize = 200;
12pub(crate) const TRACE_REQUEST_LIMIT: usize = 50;
13
14/// Where frame traces live (`FrameTraceStore`): one document per key, replaced whole.
15pub trait TraceStore: Send + Sync {
16    fn read(&self, key: &str) -> Option<Value>;
17    fn write(&self, key: &str, document: &Value) -> Result<(), String>;
18}
19
20/// Traces in memory (`createMemoryFrameStore`): a session without durable storage.
21#[derive(Default)]
22pub struct MemoryTraceStore {
23    pub documents: Mutex<HashMap<String, Value>>,
24}
25
26impl TraceStore for MemoryTraceStore {
27    fn read(&self, key: &str) -> Option<Value> {
28        self.documents
29            .lock()
30            .unwrap_or_else(|poisoned| poisoned.into_inner())
31            .get(key)
32            .cloned()
33    }
34
35    fn write(&self, key: &str, document: &Value) -> Result<(), String> {
36        self.documents
37            .lock()
38            .unwrap_or_else(|poisoned| poisoned.into_inner())
39            .insert(key.to_string(), document.clone());
40        Ok(())
41    }
42}
43
44/// The budget node one `rlm.infer` or `rlm.map` call shares among its frames.
45pub(crate) struct Budget {
46    pub id: String,
47    pub node: usize,
48    pub depth: u64,
49    /// The frame whose budget this one draws on (a nested call), by id.
50    pub parent: Option<String>,
51    /// The token limit is `rlm.map`'s default, not the caller's: exhaustion then says how to raise it.
52    pub default_tokens: bool,
53}
54
55impl Budget {
56    pub fn snapshot(&self, ledger: &Ledger) -> Value {
57        let node = &ledger.nodes[self.node];
58        json!({
59            "id": self.id, "parent": self.parent, "depth": self.depth,
60            "limits": {"calls": node.calls.limit, "tokens": node.tokens.limit},
61            "spent": {"calls": node.calls.spent, "tokens": node.tokens.spent},
62        })
63    }
64
65    /// `budgetSummary`: the shared pool of a frame summary.
66    fn summary(&self, ledger: &Ledger) -> Value {
67        let node = &ledger.nodes[self.node];
68        json!({"id": self.id, "calls": node.calls.limit, "tokens": node.tokens.limit, "depth": self.depth})
69    }
70}
71
72/// One frame: its request, what it spent and sent, and what came back.
73pub(crate) struct Frame {
74    pub id: String,
75    pub node: usize,
76    pub task: String,
77    pub views: Vec<View>,
78    /// The model the frame ran on; the requested one (`spec.model`) when it never ran.
79    pub model: Option<String>,
80    pub outputs: Vec<String>,
81    pub attempts: Vec<Value>,
82    pub requests: Vec<Value>,
83    /// Provider-reported input and output tokens and cost (summed into `rlm.map`'s reply).
84    pub measured: (u64, u64, f64),
85    pub started_at: u64,
86    pub ended_at: Option<u64>,
87    pub kind: &'static str,
88    pub batch: usize,
89    /// The `rlm-frame@1` task it runs as, and that task's lane.
90    pub task_id: Option<String>,
91    pub lane: Option<String>,
92    /// The task that made the request (`callerTaskId`), and the frame it was made from (a nested frame).
93    pub caller_task: Option<String>,
94    pub parent_frame: Option<String>,
95    /// The call claimed at admission, not yet used by the first request.
96    pub call_held: bool,
97    /// Why a follow-up request (a tool round) was refused.
98    pub exhausted: Option<String>,
99}
100
101impl Frame {
102    /// A frame whose node is a child of `budget`'s, at `depth`.
103    pub fn new(ledger: &mut Ledger, budget: &Budget, task: &str, views: Vec<View>) -> Self {
104        Self {
105            id: format!("frame-{}", jsrt::random::uuid_v4()),
106            node: ledger.add(Some(budget.node), None, None),
107            task: task.to_string(),
108            views,
109            model: None,
110            outputs: Vec::new(),
111            attempts: Vec::new(),
112            requests: Vec::new(),
113            measured: (0, 0, 0.0),
114            started_at: jsrt::date::now_ms() as u64,
115            ended_at: None,
116            kind: "infer",
117            batch: 1,
118            task_id: None,
119            lane: None,
120            caller_task: None,
121            parent_frame: None,
122            call_held: false,
123            exhausted: None,
124        }
125    }
126
127    pub fn spent(&self, ledger: &Ledger) -> Value {
128        let node = &ledger.nodes[self.node];
129        json!({"calls": node.calls.spent, "tokens": node.tokens.spent})
130    }
131
132    pub fn remaining(&self, ledger: &mut Ledger, depth: u64) -> Value {
133        json!({
134            "calls": ledger.available(self.node, Kind::Calls),
135            "tokens": ledger.available(self.node, Kind::Tokens),
136            "depth": depth,
137        })
138    }
139}
140
141/// What a frame's trace records besides the frame itself.
142pub(crate) struct Spec<'a> {
143    pub contract: Option<&'a Value>,
144    pub max_repairs: u64,
145    pub depth: u64,
146    pub model: Option<&'a str>,
147}
148
149fn status(outcome: Option<&Outcome>) -> &'static str {
150    match outcome {
151        None => "running",
152        Some(Outcome::Complete(_)) => "complete",
153        Some(Outcome::Incomplete(..)) => "incomplete",
154        Some(Outcome::Error(_)) => "error",
155    }
156}
157
158fn nullable(value: &Option<String>) -> Value {
159    value.clone().map_or(Value::Null, Value::from)
160}
161
162/// `trace(frame)`: the full document `rlm.frames(id)` returns (`status: "running"` while it runs). View metadata
163/// only, never the materialized text.
164pub(crate) fn trace(
165    ledger: &mut Ledger,
166    budget: &Budget,
167    frame: &Frame,
168    spec: &Spec,
169    outcome: Option<&Outcome>,
170) -> Value {
171    let mut out = Map::new();
172    out.insert("version".into(), 1.into());
173    out.insert("id".into(), frame.id.clone().into());
174    out.insert("status".into(), status(outcome).into());
175    match outcome {
176        Some(Outcome::Incomplete(reason, detail)) => {
177            out.insert("reason".into(), (*reason).into());
178            out.insert("detail".into(), detail.clone().into());
179        }
180        Some(Outcome::Error(error)) => {
181            out.insert("error".into(), bounded(error, OUTPUT_PREVIEW_CHARS).into());
182        }
183        Some(Outcome::Complete(value)) => {
184            out.insert("value".into(), bounded(&value.to_string(), OUTPUT_PREVIEW_CHARS).into());
185        }
186        None => {}
187    }
188    out.insert("task".into(), bounded(&frame.task, OUTPUT_PREVIEW_CHARS).into());
189    let views: Vec<Value> = frame
190        .views
191        .iter()
192        .map(|view| {
193            let mut item = Map::new();
194            item.insert("label".into(), view.label.clone().into());
195            item.insert("chars".into(), view.chars.into());
196            item.insert("byReference".into(), view.by_reference.into());
197            if let Some(digest) = &view.digest {
198                item.insert("digest".into(), digest.clone().into());
199            }
200            if let Some(start) = &view.start {
201                item.insert("start".into(), start.clone());
202            }
203            if let Some(end) = &view.end {
204                item.insert("end".into(), end.clone());
205            }
206            Value::Object(item)
207        })
208        .collect();
209    out.insert("views".into(), views.into());
210    out.insert("contract".into(), spec.contract.cloned().unwrap_or(Value::Null));
211    out.insert("maxRepairs".into(), spec.max_repairs.into());
212    out.insert("depth".into(), spec.depth.into());
213    out.insert("budget".into(), budget.snapshot(ledger));
214    out.insert("spent".into(), frame.spent(ledger));
215    out.insert("remaining".into(), frame.remaining(ledger, spec.depth));
216    out.insert("taskId".into(), nullable(&frame.task_id));
217    out.insert("lane".into(), nullable(&frame.lane));
218    out.insert("callerTaskId".into(), nullable(&frame.caller_task));
219    out.insert("parentFrame".into(), nullable(&frame.parent_frame));
220    let model = frame.model.as_deref().or(spec.model);
221    out.insert("model".into(), model.map_or(Value::Null, Value::from));
222    out.insert("startedAt".into(), frame.started_at.into());
223    out.insert("endedAt".into(), frame.ended_at.map_or(Value::Null, Value::from));
224    out.insert("attempts".into(), frame.attempts.clone().into());
225    out.insert("requests".into(), frame.requests.clone().into());
226    Value::Object(out)
227}
228
229/// `summary(frame)`: one row of `rlm.frames()`.
230pub(crate) fn summary(ledger: &Ledger, budget: &Budget, frame: &Frame, outcome: Option<&Outcome>) -> Value {
231    let mut out = Map::new();
232    out.insert("id".into(), frame.id.clone().into());
233    out.insert("status".into(), status(outcome).into());
234    if let Some(Outcome::Incomplete(reason, _)) = outcome {
235        out.insert("reason".into(), (*reason).into());
236    }
237    // `task.replace(/\s+/g, " ")`.
238    let mut collapsed = String::new();
239    for c in frame.task.chars() {
240        if jsrt::string::is_space(c) {
241            if !collapsed.ends_with(' ') {
242                collapsed.push(' ');
243            }
244        } else {
245            collapsed.push(c);
246        }
247    }
248    out.insert("task".into(), bounded(&collapsed, 120).into());
249    out.insert("views".into(), frame.views.len().into());
250    out.insert("spent".into(), frame.spent(ledger));
251    out.insert("taskId".into(), nullable(&frame.task_id));
252    out.insert("parentFrame".into(), nullable(&frame.parent_frame));
253    out.insert("startedAt".into(), frame.started_at.into());
254    out.insert("endedAt".into(), frame.ended_at.map_or(Value::Null, Value::from));
255    out.insert("kind".into(), frame.kind.into());
256    out.insert("batch".into(), frame.batch.into());
257    out.insert("callerTaskId".into(), nullable(&frame.caller_task));
258    out.insert("lane".into(), nullable(&frame.lane));
259    out.insert("budget".into(), budget.summary(ledger));
260    Value::Object(out)
261}
262
263/// The session's frame traces: the store and the index of the newest 200 summaries (read from the store once).
264pub(crate) struct Traces {
265    store: std::sync::Arc<dyn TraceStore>,
266    index: Option<Vec<Value>>,
267}
268
269impl Traces {
270    pub fn new(store: std::sync::Arc<dyn TraceStore>) -> Self {
271        Self { store, index: None }
272    }
273
274    fn index(&mut self) -> &mut Vec<Value> {
275        let store = &self.store;
276        self.index.get_or_insert_with(|| match store.read("index") {
277            Some(Value::Array(items)) => items,
278            _ => Vec::new(),
279        })
280    }
281
282    /// `writeTrace`: the trace document, then the index with the summary (a write failure loses only the trace).
283    pub fn write(&mut self, id: &str, trace: &Value, summary: Value) {
284        let _ = self.store.write(id, trace);
285        let index = self.index();
286        match index.iter().position(|item| item["id"] == id) {
287            Some(at) => index[at] = summary,
288            None => index.push(summary),
289        }
290        if index.len() > TRACE_INDEX_LIMIT {
291            index.drain(..index.len() - TRACE_INDEX_LIMIT);
292        }
293        let index = Value::Array(index.clone());
294        let _ = self.store.write("index", &index);
295    }
296
297    /// `rlm.frames`: one frame's trace by id (a running frame's from `live`), else the newest `limit` (default 20)
298    /// summaries, newest first, running frames included.
299    pub fn list(
300        &mut self,
301        payload: &Map<String, Value>,
302        live_trace: &dyn Fn(&str) -> Option<Value>,
303        live_summaries: &[Value],
304    ) -> Result<Value, String> {
305        fields(payload, &["id", "limit"])?;
306        if let Some(id) = payload.get("id") {
307            let id = id
308                .as_str()
309                .filter(|id| id.starts_with("frame-"))
310                .ok_or("id must be a frame id")?;
311            if let Some(trace) = live_trace(id) {
312                return Ok(trace);
313            }
314            return self.store.read(id).ok_or_else(|| format!("Unknown frame {id}"));
315        }
316        let limit = optional_integer(payload.get("limit"), "limit", 1, TRACE_INDEX_LIMIT as u64)?.unwrap_or(20);
317        let mut merged: Vec<Value> = self
318            .index()
319            .iter()
320            .map(|item| {
321                live_summaries
322                    .iter()
323                    .find(|live| live["id"] == item["id"])
324                    .unwrap_or(item)
325                    .clone()
326            })
327            .collect();
328        for live in live_summaries {
329            if !merged.iter().any(|item| item["id"] == live["id"]) {
330                merged.push(live.clone());
331            }
332        }
333        let frames: Vec<Value> = merged.into_iter().rev().take(limit as usize).collect();
334        Ok(json!({ "frames": frames }))
335    }
336}
337
338/// `chars` of a view as the trace reports it.
339pub(crate) fn view_chars(text: &str) -> u64 {
340    jsrt::utf16::len(text) as u64
341}