runtime/run.rs
1// This Source Code Form is subject to the terms of the Mozilla Public
2// License, v. 2.0. If a copy of the MPL was not distributed with this
3// file, You can obtain one at https://mozilla.org/MPL/2.0/.
4// Copyright (c) 2026 2youg1 and the sprawling contributors
5
6//! The run driver: dispatch, turns, freeze — the single authority for the
7//! event sequence a run leaves behind. The real city and citysim call this
8//! same code, so the simulator's byte fixtures are evidence about
9//! production rather than about a second implementation of the same idea.
10//!
11//! Time arrives through the `now` hook and is never sampled here. The
12//! caller decides what a clock is: a counter in the simulator, the one
13//! sanctioned wall-clock sample in `bin::assembly`.
14//!
15//! `crates/runtime/spec/Run.lean` proves what the event sequence must
16//! hold (§8-15): `handoff_written` and `run_frozen` close every run, once.
17
18use kernel::{
19 Address, AxError, BuildingPolicy, Carrier, Completion, Ledger, Locator, Model, Payload, RunId,
20 TimeMs, ToolCall, ToolDef,
21};
22
23use kernel::ChatMessage;
24
25use crate::catalog::SkillPin;
26use crate::conversation::{Conversation, Opening};
27use crate::handoff::Handoff;
28use crate::prefix::FrozenPrefix;
29use crate::reminder::ContextGauge;
30use crate::turn::{CallShape, Interrupt};
31
32mod charter;
33mod checkpoint;
34mod harness;
35mod lifecycle;
36
37pub use charter::Charter;
38pub use harness::{Conclusion, Cut, HarnessRun};
39
40/// Everything constant about one run. Assembled by the caller, because
41/// what a prefix contains and which tools exist are decisions of the city
42/// that dispatches, not of the loop that runs.
43pub struct RunPlan {
44 pub run: RunId,
45 pub who: String,
46 pub addr: Address,
47 pub task: String,
48 pub goal: String,
49 /// Whether this session was handed a written task or a person.
50 /// Decided by the city when it laid the brief down, carried here so
51 /// the window and the prefix's run segment cannot disagree about
52 /// which situation the agent is in.
53 pub opening: Opening,
54 pub job: Locator,
55 /// The run that handed this work down, when one did. Written into
56 /// `run_started` so the tree is a fact anybody folding the ledger
57 /// can read, rather than an inference from two lines happening to
58 /// be next to each other.
59 pub parent: Option<RunId>,
60 /// The run this one replaces, when a resident succeeded itself.
61 /// Written into `run_started` beside `parent`, so a lineage is a
62 /// fact folded from the ledger rather than inferred from two runs
63 /// sharing an address.
64 pub predecessor: Option<RunId>,
65 /// Who dispatched this run, written into `run_started`: the line's
66 /// author is always the city's desk, so only the dispatch site
67 /// knows whether the person, the city or a resident sent it.
68 pub dispatched_by: kernel::event::Who,
69 /// The run policy this run was dispatched under, written into
70 /// `run_started` as it was chosen (`crates/kernel/Spec.lean` §8-77), and
71 /// the cell that holds it while the run goes: the one value here
72 /// that changes, and only at `SafePoint::BeforeWave` (§8-62). Whoever
73 /// builds the tools builds the cell first and hands each write gate
74 /// its reader. Named apart from `policy`, which is the building's,
75 /// because the two answer different questions: what the building
76 /// allows, and what this dispatch asked for inside that.
77 pub run_policy: crate::mode::PolicyCell,
78 /// The conversation this run starts from, when it is the first run of
79 /// a session that branched off another. Empty for every other run.
80 ///
81 /// It is the window's material and not the prefix's: a prefix segment
82 /// is a document a person can read and edit, while this is what the
83 /// model said and what the tools answered, and the ledger is where
84 /// those live (`runtime::fork::inherited_indexed` rebuilds them from it).
85 pub inherited: Vec<ChatMessage>,
86 pub shape: CallShape,
87 /// Where the context reminder's second rung sits for this run: the
88 /// configuration ladder's answer, or `None` when no layer stated one
89 /// and the policy default sounds. Frozen with the run like every
90 /// setting here, so "each rung sounds once per run" cannot depend on
91 /// when somebody edited a file.
92 pub second_threshold: Option<kernel::SecondThreshold>,
93 /// Where this run records the provider's count after each call. The
94 /// caller keeps a clone for whatever reports the reading, which is
95 /// `status`: the run writes it and nothing else does.
96 pub context: crate::ContextReading,
97 pub prefix: FrozenPrefix,
98 pub policy: BuildingPolicy,
99 pub tools: Vec<ToolDef>,
100 /// The skills this run's reading room admitted, and what each one
101 /// hashed to when the shelf was read. Written into `run_started`
102 /// beside `parent`, and for the same reason: once the process that
103 /// read the shelf is gone, the ledger is the only thing that can
104 /// still say which bytes this run was given.
105 pub skills: Vec<SkillPin>,
106 /// How many times this run may make a failed call again, as the
107 /// person set it on the endpoint it calls. Frozen with the run for
108 /// the reason every other setting here is: a number changed halfway
109 /// through would make the account of what this run did depend on
110 /// when somebody looked at a form.
111 pub retries: kernel::Retries,
112 /// The identity version this run's session froze, which
113 /// `run_started` records as it is (`crates/runtime/spec/Run.lean` §8-56).
114 pub naming: Option<kernel::B3Hash>,
115}
116
117/// Where the driver stops to ask whether anything arrived. The turn layer
118/// owns three cancellation-safe points; this enum is how the driver names
119/// them to a caller that knows nothing about turn internals.
120#[derive(Debug, Clone, Copy, PartialEq, Eq)]
121pub enum SafePoint {
122 BeforeAssemble {
123 turn: u32,
124 },
125 BeforeCall {
126 turn: u32,
127 },
128 BeforeWave {
129 turn: u32,
130 },
131 /// Before one call of the wave, asked once per call. A wave is as
132 /// many effects as the model asked for, so a halt that arrives
133 /// while the third of eight runs stops the fourth rather than the
134 /// next turn.
135 BeforeToolCall {
136 turn: u32,
137 call: u32,
138 },
139 /// After the wave, before the run acts on what the turn decided.
140 /// The one boundary a run that concluded still passes through, so a
141 /// cancel arriving that late stops the work the turn handed down
142 /// instead of arriving at a run that has already ended.
143 BeforeSpawn {
144 turn: u32,
145 },
146}
147
148/// What one turn did. Exhaustive on purpose: a new ending has to make
149/// every caller say what it does about it.
150#[derive(Debug)]
151pub enum Advance {
152 Turned,
153 Concluded(Completion),
154}
155
156/// The things the driver cannot decide for itself. Closures rather
157/// than traits: no second implementation exists yet, and this
158/// library introduces a trait at a seam that already has one.
159pub struct RunHooks<'a> {
160 /// The clock, called by the driver on its own thread and nowhere else:
161 /// once per dispatch line, once per turn, once when each model attempt
162 /// is sent and once when its reply is whole, once when each tool call
163 /// starts and once when it answers, and once per freeze that is not a
164 /// cancellation.
165 pub now: &'a mut dyn FnMut() -> Result<TimeMs, AxError>,
166 /// Microseconds on a clock that only moves forward, from an origin
167 /// of the caller's choosing, read beside `now` each time a model
168 /// attempt goes out, its first content arrives and its reply is
169 /// whole, and each time a tool call starts and answers. Their
170 /// differences are the durations `model_returned` and `tool_result`
171 /// record (kernel D20); the absolute value is never recorded.
172 pub monotonic_us: &'a mut dyn FnMut() -> u64,
173 /// Answers what arrived at a safe point.
174 pub interrupt: &'a mut dyn FnMut(SafePoint) -> Interrupt,
175 /// The pre-wave checkpoint. `None` runs without a net, which
176 /// the tool layer refuses for anything that can delete.
177 pub checkpoint: Option<&'a mut dyn FnMut(TimeMs) -> Result<Payload, AxError>>,
178 /// What a call may write, by its declared effect, asked before the
179 /// wave runs: a wave whose every call answers `Nothing` changes no
180 /// file, so it needs no checkpoint of its own (§8-45).
181 pub writes: &'a dyn Fn(&ToolCall) -> kernel::Writes,
182 /// Runs a wave's tool calls in three stages (see
183 /// [`crate::ConcurrentInvoke`]). `admit` receives the turn's stamp; a
184 /// call's own moments are read by the turn from `now`, never by the
185 /// tool face, which holds no clock of its own.
186 pub invoke: &'a mut dyn crate::ConcurrentInvoke,
187 /// Holds the run until the moment the watchdog set for the next call
188 /// to a provider that failed, and answers whether that call may go.
189 ///
190 /// `Halted` comes back as soon as a halt reaches the run, without
191 /// waiting the rest out: a backoff grows to a minute, and a brake
192 /// that engages a minute late is not one. A counted clock answers
193 /// `Allowed` at once, because nothing it replays waits in real time.
194 pub wait: &'a mut dyn FnMut(TimeMs) -> crate::NextCall,
195 /// Where text goes while the model is still saying it.
196 ///
197 /// `None` runs the model without asking for a stream, which is what
198 /// citysim and every offline replay do: an increment changes nothing
199 /// a run decides, so a driver with nowhere to put one asks for none
200 /// and the call is byte-identical to what it always was.
201 ///
202 /// It takes no result, because nothing here may fail a call. What
203 /// arrives is a thing to look at; the record of what was said is
204 /// written from `model_returned`, once, afterwards.
205 pub deltas: Option<&'a mut (dyn FnMut(&kernel::Increment) + 'a)>,
206}
207
208/// An active run: turns may still be taken.
209pub struct Active {
210 conversation: Conversation,
211 turns: u32,
212 last_turn_t: Option<TimeMs>,
213 /// How full the window is, read off each call's reported usage.
214 gauge: ContextGauge,
215 /// The cache shape of the request this run sent last, which is what
216 /// the next one is compared against. `None` before the first
217 /// request, and that absence is the `FirstRequest` a record states.
218 prior_shape: Option<crate::prefix::shape::PromptShape>,
219 /// The `prompt_assembled` payload this run wrote last, which a turn
220 /// does not write again (`crates/runtime/spec/Prefix.lean` §8-39, item 5).
221 prompt: crate::turn::PromptRecord,
222 /// Whether the next wave needs a checkpoint (§8-45).
223 checkpoint: checkpoint::CheckpointPolicy,
224 /// The lines the run's turns appended and no barrier has carried
225 /// yet, with the refs of those that one has (runtime D36).
226 lines: crate::turn::HeldLines,
227 /// The run's watchdog, here rather than on `drive`'s stack because a
228 /// turn's repair segment asks the same account round whether one more
229 /// send is allowed (`crates/runtime/spec/Watchdog.lean` §8-9).
230 watchdog: crate::Watchdog,
231}
232
233/// A frozen run. There is no method back to [`Active`]: waking an old run
234/// is not something this type can spell, and `resume` takes a [`Handoff`]
235/// rather than a run.
236pub struct Frozen {
237 completion: Completion,
238 turns: u32,
239 /// Kept past the freeze for one reader: the transcript written
240 /// beside the room, which is what this run's model actually saw.
241 conversation: Conversation,
242}
243
244pub struct Run<S> {
245 plan: RunPlan,
246 state: S,
247}
248
249impl Run<Frozen> {
250 pub fn completion(&self) -> &Completion {
251 &self.state.completion
252 }
253
254 /// What this run was dispatched with, for a successor that takes
255 /// the same work on.
256 pub fn plan(&self) -> &RunPlan {
257 &self.plan
258 }
259
260 /// What the model saw, scanned and ready to write beside the room.
261 ///
262 /// # Errors
263 /// Propagates a message that will not serialise.
264 pub fn transcript(&self) -> Result<crate::transcript::Transcript, AxError> {
265 crate::transcript::Transcript::of(self.plan.run, &self.state.conversation)
266 }
267
268 pub fn turns(&self) -> u32 {
269 self.state.turns
270 }
271
272 pub fn run_id(&self) -> RunId {
273 self.plan.run
274 }
275}
276
277/// Dispatch, turn until an ending, freeze. The loop is here rather than in
278/// each caller because the ending rules — an empty wave concludes, a
279/// cancellation is an ending too — are the part that must not drift
280/// between the city and the simulator.
281///
282/// **The loop counts no turns.** There is no ceiling to reach, because
283/// nobody can price a piece of work before it runs. What
284/// ends a run is what it did: a turn that concluded, a failure with a
285/// carrier event, or an interruption a safe point delivered — and what
286/// stops one from outside is `Halt`, which shuts the scope and kills the
287/// backlog members inside it.
288///
289/// # Errors
290/// Propagates ledger and provider failures.
291pub fn drive(
292 plan: RunPlan,
293 ledger: &mut dyn Ledger,
294 model: &mut dyn Model,
295 hooks: &mut RunHooks<'_>,
296 handoff: &Handoff,
297) -> Result<Run<Frozen>, AxError> {
298 let mut run = Run::dispatch(plan, ledger, hooks)?;
299 let ending = loop {
300 // Each model call goes out on the account its round opens with
301 // (kernel D54); a round stays open across the resends and switches
302 // of one call and closes when the provider answers.
303 if !run.state.watchdog.round_open() {
304 match run.state.watchdog.open_round(model.account_roster()) {
305 Ok(Some(account)) => model.select_account(&account),
306 Ok(None) => {}
307 Err(exhausted) => {
308 run.state.lines.barrier(ledger)?;
309 let t = (hooks.now)()?;
310 run.record_refusal(ledger, t, &exhausted)?;
311 break Completion::Cancelled;
312 }
313 }
314 }
315 match run.advance(ledger, model, hooks) {
316 Ok(Advance::Turned) => run.state.watchdog.on_provider_answered(),
317 Ok(Advance::Concluded(completion)) => break completion,
318 // A run always ends. A mid-turn failure whose code has a
319 // carrier event (provider down, budget, watchdog) is written
320 // into history under that carrier and the run freezes as
321 // cancelled, so a 401 from a provider never leaves a run
322 // "started" with an event stream that simply went quiet.
323 //
324 // A loadtime code names no carrier, and it leaves by the
325 // same door: the verdict is written first and the diagnosis
326 // travels afterwards, so the two facts do not compete.
327 // `E_WIRE_MISMATCH` means a provider spelled its dialect
328 // wrong while the ledger is in perfect health. Where the
329 // store really is the casualty, `freeze` fails on its own
330 // append and that failure is what travels, which is more
331 // honest than deciding in advance that nothing can be
332 // written.
333 Err(err) => {
334 // What the failed turn and the one before it held goes
335 // down first, so the carrier event and the watchdog's line
336 // follow them in the ledger as they followed them in time
337 // (runtime D36).
338 run.state.lines.barrier(ledger)?;
339 let Carrier::Event(kind) = err.code().carrier() else {
340 run.freeze(ledger, handoff, Completion::Cancelled, hooks)?;
341 return Err(err);
342 };
343 let t = (hooks.now)()?;
344 // A provider that failed for a reason worth retrying is
345 // asked again, and the attempt is recorded rather than
346 // repeated silently: the second `model_called` in the
347 // history is what a person reads the retry off.
348 // The watchdog decides whether there is a next call and
349 // when; the line records that moment before the wait,
350 // and the wait is where a halt reaches a run that has
351 // no turn in flight to stop.
352 // A switch goes at once on the account the round named,
353 // so the wait is never asked and a halt reaches the run
354 // at the next safe point instead.
355 let disposal = run.state.watchdog.on_provider_failure(&err, t);
356 match &disposal {
357 crate::Disposal::BackOff { until, .. } => {
358 let until = *until;
359 run.record_fired(ledger, t, &disposal)?;
360 match (hooks.wait)(until) {
361 crate::NextCall::Allowed => continue,
362 crate::NextCall::Halted => break Completion::Cancelled,
363 }
364 }
365 crate::Disposal::Switch { to, .. } => {
366 let to = to.clone();
367 run.record_fired(ledger, t, &disposal)?;
368 model.select_account(&to);
369 continue;
370 }
371 crate::Disposal::Freeze {
372 reason: crate::FreezeReason::AccountsExhausted,
373 } => {
374 let exhausted = run.state.watchdog.exhausted();
375 run.record_refusal(ledger, t, &exhausted)?;
376 }
377 crate::Disposal::Proceed
378 | crate::Disposal::CorrectiveSteer { .. }
379 | crate::Disposal::Freeze {
380 reason: crate::FreezeReason::Stall | crate::FreezeReason::ProviderRefused,
381 } => {
382 // The failure itself is the payload, through the
383 // one door: an encoding that fails travels as a
384 // refusal rather than as an empty object.
385 ledger.append(run.line(t, kind, Payload::of(&err)?))?;
386 }
387 }
388 break Completion::Cancelled;
389 }
390 }
391 };
392 run.freeze(ledger, handoff, ending, hooks)
393}