1use super::{Inference, fields, optional_integer};
8use kernel::{HostHandler, Signal};
9use serde_json::{Map, Value, json};
10use std::collections::HashMap;
11use std::sync::Arc;
12use std::sync::atomic::{AtomicUsize, Ordering};
13
14const MAX_MAP_FRAMES: usize = 10_000;
15const MAX_CONCURRENCY: u64 = 16;
16const MAP_TOKENS_ENV: &str = "ULTRON_RLM_MAP_TOKENS";
17
18pub fn frame_env(var: &dyn Fn(&str) -> Option<String>) -> HashMap<String, String> {
20 [
21 config::env::RLM_FRAME_MODEL_ENV,
22 config::env::REVIEW_MODEL_ENV,
23 MAP_TOKENS_ENV,
24 ]
25 .into_iter()
26 .filter_map(|key| var(key).map(|value| (key.to_string(), value)))
27 .collect()
28}
29
30pub const INFERENCE_PROMPT: &str = "Large inputs stay out of your context as handles; examples: `help(rlm.load)`, `help(rlm.map)`.
32- `h = await rlm.load(path_or_text)`: `h.search(regex)`, `h.count(regex)`, `h.lines(a, b)`, `h.chunks(n)`; views print their text.
33- `await rlm.infer(task, context=[views], contract=T, budget=Budget(calls, tokens, depth))`: one private sub-model call (no transcript, no tools), contract-validated and re-asked; running out returns a falsy `Incomplete`.
34- `await rlm.map(task, items, contract=T)`: one frame per item, in order, under one shared budget (`budget=Budget(tokens=...)` caps it).
35- Every frame is a model request: filter with code first, give each frame only the passage it must judge, and compute exact numbers in Python.";
36
37#[derive(Clone, Debug, Default)]
39pub struct ModelSettings {
40 pub frame_model: Option<String>,
41 pub frame_thinking: Option<String>,
42 pub child_model: Option<String>,
43 pub review_model: Option<String>,
44 pub map_tokens: Option<u64>,
46}
47
48pub struct Frames {
50 inference: Inference,
51 env: HashMap<String, String>,
52 settings: Box<dyn Fn() -> ModelSettings + Send + Sync>,
53}
54
55impl Frames {
56 pub fn new(
57 inference: Inference,
58 env: HashMap<String, String>,
59 settings: impl Fn() -> ModelSettings + Send + Sync + 'static,
60 ) -> Self {
61 Self {
62 inference,
63 env,
64 settings: Box::new(settings),
65 }
66 }
67
68 fn env_model(&self, name: &str) -> Option<String> {
69 config::env::env_model(&|key| self.env.get(key).cloned(), name)
70 }
71
72 fn frame_model(&self, settings: &ModelSettings) -> Value {
74 match (self.env_model(config::env::RLM_FRAME_MODEL_ENV), &settings.frame_model) {
75 (Some(model), _) => json!({"model": model, "source": "env", "env": config::env::RLM_FRAME_MODEL_ENV}),
76 (None, Some(model)) => json!({"model": model, "source": "setting"}),
77 (None, None) => json!({"source": "session"}),
78 }
79 }
80
81 fn models(&self) -> Value {
83 let settings = (self.settings)();
84 let frame = self.frame_model(&settings);
85 let review = match (self.env_model(config::env::REVIEW_MODEL_ENV), &settings.review_model) {
86 (Some(model), _) => json!({"model": model, "source": "env", "env": config::env::REVIEW_MODEL_ENV}),
87 (None, Some(model)) => json!({"model": model, "source": "setting"}),
88 (None, None) => match frame.get("model") {
89 Some(model) => json!({"model": model, "source": "frame"}),
90 None => frame.clone(),
91 },
92 };
93 let child = match &settings.child_model {
94 Some(model) => json!({"model": model, "source": "setting"}),
95 None => json!({"source": "session"}),
96 };
97 json!({"frame": frame, "review": review, "child": child, "frameThinking": settings.frame_thinking})
98 }
99
100 fn default_map_tokens(&self) -> Option<u64> {
103 let value = self
104 .env
105 .get(MAP_TOKENS_ENV)
106 .map(|v| jsrt::string::trim(v).parse::<f64>());
107 match value {
108 Some(Ok(n)) if n > 0.0 && jsrt::number::is_safe_integer(n) => Some(n as u64),
109 _ => (self.settings)().map_tokens,
110 }
111 }
112
113 fn defaults(&self, payload: &mut Map<String, Value>) {
115 let settings = (self.settings)();
116 let depth = payload
117 .get("budget")
118 .and_then(|b| b.get("depth"))
119 .filter(|d| !d.is_null());
120 let unset = |payload: &Map<String, Value>, key: &str| payload.get(key).is_none_or(Value::is_null);
121 if unset(payload, "model")
122 && depth.is_none_or(|d| d.as_f64() == Some(1.0))
123 && let Some(model) = self.frame_model(&settings).get("model")
124 {
125 payload.insert("model".into(), model.clone());
126 }
127 if unset(payload, "thinking")
128 && let Some(level) = settings.frame_thinking
129 {
130 payload.insert("thinking".into(), level.into());
131 }
132 }
133
134 fn map(&self, payload: &mut Map<String, Value>, signal: &Signal) -> Result<Value, String> {
135 fields(
136 payload,
137 &[
138 "frames",
139 "prefix",
140 "contract",
141 "budget",
142 "model",
143 "max_repairs",
144 "concurrency",
145 "timeout_ms",
146 "thinking",
147 ],
148 )?;
149 let items = match payload.get("frames") {
150 Some(Value::Array(items)) => items.clone(),
151 _ => return Err("frames must be a list".into()),
152 };
153 if items.len() > MAX_MAP_FRAMES {
154 return Err(format!("rlm.map takes at most {MAX_MAP_FRAMES} frames"));
155 }
156 let concurrency = optional_integer(payload.get("concurrency"), "concurrency", 1, MAX_CONCURRENCY)?.unwrap_or(8);
157 self.defaults(payload);
158 let inference = &self.inference;
159 let (mut options, admission) = inference.options(payload, self.default_map_tokens())?;
160 options.prefix = inference.views(payload.get("prefix"), 1)?;
162 let mut requests = Vec::new();
164 for item in &items {
165 let item = item.as_object().ok_or("frame must be an object")?;
166 let task = Inference::task_of(item)?;
167 requests.push((task, inference.views(item.get("context"), options.depth)?));
168 }
169 if options.depth > 1 && !inference.session.has_tasks() {
170 return Err(format!(
171 "budget depth {} needs frames that run code (the rlm tool), which this host does not run yet",
172 options.depth
173 ));
174 }
175 let session = &inference.session;
176 let options = Arc::new(options);
177 let budget = Arc::new(session.budget(&admission));
178 let frames: Vec<_> = requests
181 .into_iter()
182 .map(|(task, views)| {
183 session.admit(
184 &budget,
185 &options,
186 &inference.engine,
187 &admission,
188 task,
189 views,
190 "map",
191 items.len(),
192 )
193 })
194 .collect();
195 let next = AtomicUsize::new(0);
197 let worker = || {
198 while let Some(frame) = frames.get(next.fetch_add(1, Ordering::SeqCst)) {
199 session.run_frame(frame, &inference.lane, signal);
200 }
201 };
202 let workers = (concurrency as usize).min(frames.len());
203 std::thread::scope(|scope| {
204 for _ in 1..workers {
205 scope.spawn(worker);
206 }
207 worker();
208 });
209 let mut usage = (0, 0, 0.0);
210 let results: Vec<Value> = frames
211 .iter()
212 .map(|frame| {
213 let measured = session.measured(frame);
214 usage = (usage.0 + measured.0, usage.1 + measured.1, usage.2 + measured.2);
215 session.observation(frame)
216 })
217 .collect();
218 Ok(json!({
219 "results": results,
220 "budget": session.budget_reply(&budget),
221 "usage": jsrt::json::normalize_numbers(json!({"input_tokens": usage.0, "output_tokens": usage.1, "cost": usage.2})),
222 }))
223 }
224}
225
226impl HostHandler for Frames {
227 fn handle(&self, kind: &str, mut payload: Map<String, Value>, signal: &Signal) -> Result<Value, String> {
228 match kind {
229 "rlm.map" => self.map(&mut payload, signal),
230 "rlm.models" => Ok(self.models()),
231 "rlm.infer" => {
232 self.defaults(&mut payload);
233 self.inference.handle(kind, payload, signal)
234 }
235 _ => self.inference.handle(kind, payload, signal),
236 }
237 }
238}