mlua_swarm/core/engine.rs
1//! `Engine` — the long-running stateful runtime plus the `with_state`
2//! helper (R1-R4 discipline).
3//!
4//! The engine owns the Domain side of the Data / Domain split:
5//! flow control (dispatch / verdict), state (`EngineState`), and the
6//! `submit_output` / `output_tail` surface that feeds it. Data-plane
7//! traffic (Big Response bodies) is delegated to the `output_store` module
8//! plus its paired `SpawnerLayer`s and passes through here without the
9//! engine core needing to grow.
10
11use crate::core::agent_context::{RUN_ID_KEY, STEP_CTX_KEY};
12use crate::core::config::EngineCfg;
13use crate::core::ctx::{Ctx, OperatorInfo, OperatorKind, SeniorBridge, SpawnHook};
14use crate::core::errors::EngineError;
15use crate::core::state::{
16 unwrap_skip_marker, wrap_skip_marker, CapTokenRecord, DispatchOutcome, EngineState, Event,
17 EventStream, OperatorSession, ResumeKey, ResumePending, SubmitOutcome, TaskSpec, TaskState,
18 TaskStatus,
19};
20use crate::store::replay::{hash_input_value, ReplayEntry};
21use crate::store::run::RunContext;
22use crate::types::{
23 default_role_verb_table, now_unix, CapToken, Role, RoleVerbGate, RunId, SessionId, StepId,
24 TokenSigner, Verb,
25};
26use crate::worker::adapter::SpawnerAdapter;
27use serde_json::Value;
28use std::collections::HashMap;
29use std::sync::Arc;
30use std::time::{Duration, Instant};
31use tokio::sync::{broadcast, Mutex};
32
33/// Process-wide long-running runtime. Cheap to `clone()` — an `Arc`
34/// lives inside.
35#[derive(Clone)]
36pub struct Engine {
37 inner: Arc<EngineInner>,
38}
39
40struct EngineInner {
41 state: Mutex<EngineState>,
42 cfg: EngineCfg,
43 signer: TokenSigner,
44 gate: RoleVerbGate,
45 event_tx: broadcast::Sender<Event>,
46 /// ID-keyed bridge registry (register-by-ID design). `SeniorBridge`
47 /// and `SpawnHook` are registered by ID; sessions bind to those IDs
48 /// only. Persistence stores just the ID, and on reattach the caller
49 /// re-registers under the same ID to restore presence.
50 senior_bridges: tokio::sync::RwLock<HashMap<String, Arc<dyn SeniorBridge>>>,
51 spawn_hooks: tokio::sync::RwLock<HashMap<String, Arc<dyn SpawnHook>>>,
52 /// ID registry for full-spawn Operator backends (backends that take the
53 /// entire spawn via `execute`). Sibling to `senior_bridges` /
54 /// `spawn_hooks`. `OperatorDelegateMiddleware` looks these up via
55 /// `ctx` and, when `kind = MainAi` / `Composite`, bypasses
56 /// `inner.spawn` and calls `operator.execute` instead.
57 operators: tokio::sync::RwLock<HashMap<String, Arc<dyn crate::operator::Operator>>>,
58 /// Base and hint layer factories for the `SpawnerStack`. At
59 /// `service::linker::link` time, `compiled.router` is wrapped with
60 /// the base factories plus the hint factories resolved from
61 /// `blueprint.spawner_hints.layers`. This is the engine-side
62 /// counterpart to the discipline "Flow / Blueprint doesn't spell out
63 /// middleware implementations — it declares the capabilities it needs
64 /// as hint keys".
65 layer_registry: crate::middleware::LayerRegistry,
66 /// Optional Data-plane `OutputStore` backend (subtask-4 / ST2 rework —
67 /// see `submit_output`'s doc). `None` (the default) preserves
68 /// pre-subtask-4 behavior exactly: `submit_output` /
69 /// `submit_worker_result_trusted` only touch the Domain-plane
70 /// `EngineState.output_store` HashMap, same as before this was added.
71 /// `Some` additionally dual-writes every `Final` event into this store
72 /// via [`crate::store::output::OutputStore::append`], making it
73 /// queryable (e.g. by `mlua-swarm-server`'s `GET /v1/tasks/:id/ctx`)
74 /// even for an in-flight run. A plain `std::sync::RwLock` (not
75 /// `tokio::sync::RwLock`) — set once at boot via [`Engine::set_output_store`]
76 /// from a synchronous call site (`mlua-swarm-server`'s router builder),
77 /// then only ever briefly read (clone the `Option<Arc<..>>`, never held
78 /// across an `.await`) from the async submit path.
79 data_store: std::sync::RwLock<Option<Arc<dyn crate::store::output::OutputStore>>>,
80 /// GH #50 (Subtask 2 — runtime plumbing): agent name → declared
81 /// [`mlua_swarm_schema::VerdictContract`], the Engine-side registry
82 /// [`Self::verdict_contract_for_task`] resolves against. Populated via
83 /// [`Self::register_verdict_contracts`] — same sync-`RwLock`,
84 /// set-outside-the-lock idiom as `data_store` above. Empty by default
85 /// (every pre-GH-#50 `Engine`), which is exactly the opt-in "no
86 /// contract declared" state `verdict_contract_for_task` treats as
87 /// `None`. Populated from a live `Compiler::compile`'s
88 /// `CompiledAgentTable.verdict_contracts` output by
89 /// `TaskLaunchService::launch`, immediately after `compiler.compile`
90 /// succeeds — see [`Self::register_verdict_contracts`]'s doc for the
91 /// overwrite semantics of that merge.
92 verdict_contracts: std::sync::RwLock<HashMap<String, mlua_swarm_schema::VerdictContract>>,
93}
94
95/// Renders a `TaskSpec.initial_directive` / `EngineState.prompts`
96/// `Value` down to the `String` shape that string-consuming boundaries
97/// require (issue #18). Strings pass through verbatim; anything else
98/// (Object / Array / Number / Bool / Null) is serde-stringified. This
99/// is the single canonical rendering — the coercion that used to sit
100/// inside `EngineDispatcher::dispatch` moved here and is invoked only
101/// at consumer boundaries: `WorkerPayload.prompt` (HTTP
102/// `/v1/worker/prompt`), `WorkerInvocation.prompt` (in-process
103/// spawners), the subprocess spawner's directive arg/stdin, and the
104/// WS Spawn frame text render (`operator_ws::session`). Everything
105/// upstream (Blueprint dispatch → engine state → `fetch_prompt` →
106/// `Operator::execute`) keeps the `Value` end-to-end.
107pub(crate) fn render_directive_to_string(v: &Value) -> String {
108 match v {
109 Value::String(s) => s.clone(),
110 other => other.to_string(),
111 }
112}
113
114/// Renders a [`crate::worker::output::ContentRef`] down to the `Value` shape
115/// the BP-chain / `DispatchOutcome` consume. `Inline` passes its `value`
116/// through verbatim; `FileRef` is stringified into the same
117/// `{"file_ref", "mime", "size_hint"}` shape `materialize_final_submission`
118/// uses for its own file-materialize projection — one canonical
119/// stringification, not two independently-maintained copies (GH #36 ST1:
120/// shared by both the `Final`-pull and the `Artifact`-parts fold in
121/// [`Engine::dispatch_attempt_with`]'s doc).
122fn content_ref_to_value(content: crate::worker::output::ContentRef) -> Value {
123 match content {
124 crate::worker::output::ContentRef::Inline { value } => value,
125 crate::worker::output::ContentRef::FileRef {
126 path,
127 mime,
128 size_hint,
129 } => serde_json::json!({
130 "file_ref": path.to_string_lossy(),
131 "mime": mime,
132 "size_hint": size_hint,
133 }),
134 }
135}
136
137/// GH #51 — reduces a [`content_ref_to_value`] result down to the `String`
138/// shape the completion-time verdict-contract check compares against a
139/// declared `VerdictContract.values` token set. A `Value::String` unwraps
140/// to its raw contents (no surrounding JSON quotes) — this mirrors the
141/// pre-GH-#51 `check_verdict_contract` (`mlua-swarm-server`'s
142/// `worker.rs`), which always compared the raw submitted body string
143/// directly, never a JSON-stringified copy. Any OTHER `Value` shape
144/// (`Number` / `Object` / `Array` / `Bool` / `Null` — i.e. a `channel:
145/// "body"` contract whose completing value is not a string at all, or a
146/// `FileRef` content whose `content_ref_to_value` projection is an
147/// object) falls back to `Value::to_string()`'s JSON-encoded form: it can
148/// never collide with a plain declared token like `"PASS"`, so it
149/// naturally fails membership — consistent with the "non-string values
150/// under a body contract are violations" rule (issue #51's Proposal).
151fn content_ref_to_comparable_string(content: crate::worker::output::ContentRef) -> String {
152 let value = content_ref_to_value(content);
153 match value {
154 Value::String(s) => s,
155 other => other.to_string(),
156 }
157}
158
159/// [`Engine::dispatch_attempt_with`]'s Final-pull assembly (GH #36 ST1:
160/// named multi-part worker output), factored out as a pure function of the
161/// output-event tail so it is unit-testable without a live `Engine` /
162/// spawner.
163///
164/// Finds the LAST `Final` event in `tail` (mirrors the pre-GH-#36 pull:
165/// "last Final wins" if more than one was ever appended) and folds every
166/// `Artifact` event in the SAME tail WHOSE NAME APPEARS IN `staged_names`
167/// into a `"parts"` object keyed by `Artifact.name` — walked in tail (=
168/// event-append) order, so a name staged more than once within the attempt
169/// is last-write-wins (`Map` insert semantics, not an accumulating list;
170/// `Engine::stage_worker_artifact_trusted`'s doc). `staged_names` is the
171/// WORKER's own opt-in allowlist (`EngineState.worker_artifact_names`'s
172/// doc) — an `Artifact` on the tail whose name is NOT in `staged_names`
173/// (e.g. `AfterRunAuditMiddleware`'s `"audit:<step_ref>"` sidecar finding)
174/// is left alone, exactly as before GH #36; this is what keeps an audited
175/// step's BP-chain value byte-identical when the worker itself never
176/// staged a part.
177///
178/// At least one matching part: the returned value is `{"out": <final
179/// value>, "parts": {<name>: <value>, ...}}`. Zero matching parts: the
180/// returned value is the plain final value, unchanged from the pre-GH-#36
181/// shape — this is the back-compat guarantee, not an incidental default.
182///
183/// `None` when `tail` carries no `Final` at all (the caller's pre-existing
184/// "no Final in output_tail" error path).
185fn fold_final_and_parts(
186 tail: &[crate::worker::output::OutputEvent],
187 staged_names: &[String],
188) -> Option<(Value, bool)> {
189 let (final_content, ok) = tail.iter().rev().find_map(|ev| match ev {
190 crate::worker::output::OutputEvent::Final { content, ok } => Some((content.clone(), *ok)),
191 _ => None,
192 })?;
193 let final_value = content_ref_to_value(final_content);
194
195 let mut parts = serde_json::Map::new();
196 for ev in tail {
197 if let crate::worker::output::OutputEvent::Artifact { name, content } = ev {
198 if staged_names.iter().any(|staged| staged == name) {
199 parts.insert(name.clone(), content_ref_to_value(content.clone()));
200 }
201 }
202 }
203
204 let value = if parts.is_empty() {
205 final_value
206 } else {
207 serde_json::json!({ "out": final_value, "parts": Value::Object(parts) })
208 };
209 Some((value, ok))
210}
211
212impl Engine {
213 /// Backwards-compatible constructor that starts the engine without a
214 /// layer registry, preserving the signature already used by ~88
215 /// existing call sites. Use this when automatic middleware wrapping
216 /// at bind time is not needed. Callers such as `mlua-swarm-server` go through
217 /// `new_with_layers(cfg, registry)` to enable the hint-resolution path.
218 pub fn new(cfg: EngineCfg) -> Self {
219 Self::new_with_layers(cfg, crate::middleware::LayerRegistry::new())
220 }
221
222 /// Construct an `Engine` with an explicit `LayerRegistry`, enabling
223 /// hint-resolution: `spawner_hints.layers` declared on a `Blueprint`
224 /// are resolved against this registry when the spawner stack is bound
225 /// at `service::linker::link` time.
226 pub fn new_with_layers(
227 cfg: EngineCfg,
228 layer_registry: crate::middleware::LayerRegistry,
229 ) -> Self {
230 let (event_tx, _) = broadcast::channel(256);
231 let signer = TokenSigner::new(&cfg.token_secret);
232 Self {
233 inner: Arc::new(EngineInner {
234 state: Mutex::new(EngineState::new()),
235 cfg,
236 signer,
237 gate: default_role_verb_table(),
238 event_tx,
239 senior_bridges: tokio::sync::RwLock::new(HashMap::new()),
240 spawn_hooks: tokio::sync::RwLock::new(HashMap::new()),
241 operators: tokio::sync::RwLock::new(HashMap::new()),
242 layer_registry,
243 data_store: std::sync::RwLock::new(None),
244 verdict_contracts: std::sync::RwLock::new(HashMap::new()),
245 }),
246 }
247 }
248
249 /// Rebuild this `Engine` with a different `RoleVerbGate`. The gate is
250 /// treated as fixed-at-build-time, so this constructs a fresh
251 /// `EngineInner` (fresh empty `EngineState`) rather than mutating in
252 /// place — mainly a testing convenience for swapping gate rules.
253 pub fn with_gate(self, gate: RoleVerbGate) -> Self {
254 // The gate is fixed at build time — the intent is to build a fresh
255 // instance rather than mutating in place. As a testing convenience we
256 // do allow swapping the inner Arc. Simpler form: just rebuild
257 // Arc<EngineInner>.
258 let inner = Arc::new(EngineInner {
259 state: Mutex::new(EngineState::new()),
260 cfg: self.inner.cfg.clone(),
261 signer: self.inner.signer.clone(),
262 gate,
263 event_tx: self.inner.event_tx.clone(),
264 senior_bridges: tokio::sync::RwLock::new(HashMap::new()),
265 spawn_hooks: tokio::sync::RwLock::new(HashMap::new()),
266 operators: tokio::sync::RwLock::new(HashMap::new()),
267 layer_registry: self.inner.layer_registry.clone(),
268 data_store: std::sync::RwLock::new(None),
269 verdict_contracts: std::sync::RwLock::new(HashMap::new()),
270 });
271 Self { inner }
272 }
273
274 // ═══════════════════════════════════════════════════════════════════════
275 // Accessors. Production code drives execution through compile +
276 // `service::linker::link` + `dispatch_attempt_with(spawner)` inside
277 // `TaskLaunchService`; `Engine` itself is a pure execution surface — it
278 // does not own a BlueprintStore / EnhanceAdapter / Compiler, nor a
279 // global spawner (the spawner is carried per-request, never stashed on
280 // the engine).
281 // ═══════════════════════════════════════════════════════════════════════
282
283 /// Access the `EngineCfg` this engine was built with.
284 pub fn cfg(&self) -> &EngineCfg {
285 &self.inner.cfg
286 }
287
288 /// Expose the internal `LayerRegistry` — used when deriving a
289 /// sub-engine that needs the same registry re-injected. The
290 /// per-request sub-engine in `mlua-swarm-server` reads the parent engine's
291 /// registry through this accessor and passes it to
292 /// `Engine::new_with_layers(cfg, parent.layer_registry().clone())`.
293 pub fn layer_registry(&self) -> &crate::middleware::LayerRegistry {
294 &self.inner.layer_registry
295 }
296
297 /// Access the `TokenSigner` used to mint/verify `CapToken`s.
298 pub fn signer(&self) -> &TokenSigner {
299 &self.inner.signer
300 }
301
302 /// Clone a handle to the process-wide `Event` broadcast sender. Prefer
303 /// `subscribe` for a ready-to-use receiver.
304 pub fn event_tx(&self) -> broadcast::Sender<Event> {
305 self.inner.event_tx.clone()
306 }
307
308 /// Subscribe to the engine's `Event` broadcast stream.
309 pub fn subscribe(&self) -> EventStream {
310 self.inner.event_tx.subscribe()
311 }
312
313 /// Wires the Data-plane [`crate::store::output::OutputStore`] backend
314 /// used by `submit_output` / `submit_worker_result_trusted`'s
315 /// submit-time projection sink (subtask-4 / ST2 rework — see
316 /// `submit_output`'s doc). Synchronous (a plain `std::sync::RwLock`
317 /// write) so a caller can wire it up at boot from a non-`async`
318 /// context (`mlua-swarm-server`'s router builder passes the same
319 /// `Arc` it hands to its `AppState.data_store`, so `POST
320 /// /v1/data/emit` and every worker's ordinary `/v1/worker/submit` land
321 /// in the one store). Calling this more than once replaces the
322 /// previous backend; not calling it at all (the default) preserves
323 /// pre-subtask-4 behavior exactly — `submit_output` only touches the
324 /// Domain-plane `EngineState.output_store` HashMap.
325 pub fn set_output_store(&self, store: Arc<dyn crate::store::output::OutputStore>) {
326 let mut guard = self
327 .inner
328 .data_store
329 .write()
330 .unwrap_or_else(|poisoned| poisoned.into_inner());
331 *guard = Some(store);
332 }
333
334 /// Clones the currently-wired Data-plane store handle, if any. Kept
335 /// private and side-effect-free (no lock held past this call) —
336 /// callers (`materialize_final_submission`) do their actual `.append`
337 /// work outside of any lock.
338 fn output_store_backend(&self) -> Option<Arc<dyn crate::store::output::OutputStore>> {
339 self.inner
340 .data_store
341 .read()
342 .unwrap_or_else(|poisoned| poisoned.into_inner())
343 .clone()
344 }
345
346 /// GH #50 (Subtask 2): merges `contracts` (agent name → declared
347 /// [`mlua_swarm_schema::VerdictContract`]) into the engine's runtime
348 /// verdict-contract registry, later resolved per-task by
349 /// [`Self::verdict_contract_for_task`]. Same sync-write idiom as
350 /// [`Self::set_output_store`] — a plain `std::sync::RwLock` write, so
351 /// this can be called from a non-`async` context. Production call
352 /// site: `TaskLaunchService::launch`, immediately after a successful
353 /// `Compiler::compile`, passing `compiled.router.verdict_contracts.clone()`.
354 ///
355 /// # Overwrite semantics (explicit — read before adding a second call site)
356 ///
357 /// The registry is a single flat `HashMap` **keyed by agent name only**
358 /// (`String`), with process-wide (not per-task, not per-Blueprint,
359 /// not per-launch) scope. Registration is additive via
360 /// `HashMap::extend`: an entry for an agent name NOT already present is
361 /// added; an entry for an agent name ALREADY present is REPLACED
362 /// (last write wins) by the incoming one. Concretely: launching a
363 /// second Blueprint that also declares a `verdict` contract for an
364 /// agent named `"gate"` OVERWRITES whatever contract a first, still
365 /// in-flight, launch registered for an agent of that same name — even
366 /// if the two Blueprints intend it as two semantically different
367 /// agents that merely share a name, and even while the first launch's
368 /// tasks are still running. This is a **known limitation** of the v1
369 /// design; a per-task (or per-`RunId` / per-Blueprint) scoped registry
370 /// is a possible follow-up if two concurrently in-flight Blueprints
371 /// declaring conflicting contracts under the same agent name turns out
372 /// to matter in practice. Calling this with an empty map (or not at
373 /// all — the default) is a no-op, preserving pre-GH-#50 behavior
374 /// exactly (opt-in).
375 pub fn register_verdict_contracts(
376 &self,
377 contracts: HashMap<String, mlua_swarm_schema::VerdictContract>,
378 ) {
379 let mut guard = self
380 .inner
381 .verdict_contracts
382 .write()
383 .unwrap_or_else(|poisoned| poisoned.into_inner());
384 guard.extend(contracts);
385 }
386
387 /// GH #50 (Subtask 2): the declared
388 /// [`mlua_swarm_schema::VerdictContract`] for the agent currently
389 /// running `task_id`, if any. Resolves `task_id` → `TaskState.spec.agent`
390 /// (via `EngineState.tasks`, the same lookup [`Self::task_attempt`]
391 /// performs) and looks that agent name up in the registry
392 /// [`Self::register_verdict_contracts`] populates.
393 ///
394 /// `None` in both of these cases — deliberately collapsed to the same
395 /// value, mirroring [`Self::agent_context_for`]'s `Result`-into-`Option`
396 /// pattern (`.ok().flatten()`; a lookup failure here is never itself an
397 /// error worth surfacing to a caller):
398 /// - `task_id` is unknown (no `TaskState` for it).
399 /// - `task_id` resolves to a known agent, but that agent declared no
400 /// `verdict` contract (the opt-in default).
401 ///
402 /// Callers (`mlua-swarm-server`'s `worker_submit` / `worker_artifact`)
403 /// treat every `None` identically: skip the submit-time verdict gate
404 /// entirely, preserving pre-GH-#50 behavior byte-for-byte.
405 pub async fn verdict_contract_for_task(
406 &self,
407 task_id: &StepId,
408 ) -> Option<mlua_swarm_schema::VerdictContract> {
409 let tid = task_id.clone();
410 let agent = self
411 .with_state("verdict_contract_for_task", move |s| {
412 s.tasks.get(&tid).map(|t| t.spec.agent.clone())
413 })
414 .await
415 .ok()
416 .flatten()?;
417 self.inner
418 .verdict_contracts
419 .read()
420 .unwrap_or_else(|poisoned| poisoned.into_inner())
421 .get(&agent)
422 .cloned()
423 }
424
425 /// GH #51 — the value of the LAST staged `"verdict"` `Artifact` for
426 /// `(task_id, attempt)`, if any. Mirrors [`fold_final_and_parts`]'s
427 /// reverse-scan-of-`output_tail` pattern (last-write-wins per name,
428 /// same as that fold and [`Self::stage_worker_artifact_trusted`]'s
429 /// doc), narrowed to the single literal artifact name
430 /// `channel: "part"` contracts address (Pattern B — see
431 /// `blueprint-authoring.md`'s "Returning verdicts to drive BP flow").
432 ///
433 /// Infallible accessor: `None` is the normal "nothing staged yet"
434 /// case, not an error — the caller
435 /// ([`Self::verdict_contract_completion_check`]) is what converts
436 /// `None` into `Err(EngineError::VerdictPartMissing)`.
437 pub(crate) async fn staged_verdict_value_for(
438 &self,
439 task_id: &StepId,
440 attempt: u32,
441 ) -> Option<String> {
442 let tail = self.output_tail(task_id, attempt).await;
443 tail.iter().rev().find_map(|ev| match ev {
444 crate::worker::output::OutputEvent::Artifact { name, content } if name == "verdict" => {
445 Some(content_ref_to_comparable_string(content.clone()))
446 }
447 _ => None,
448 })
449 }
450
451 /// GH #51 — the single completion-time verdict-contract choke point,
452 /// embedded inside BOTH [`Self::submit_worker_result_trusted`] and
453 /// [`Self::submit_output`] (the two engine-side writes every HTTP/WS
454 /// completion route ultimately passes through). Not duplicated per
455 /// route handler — a future 4th completion route is gated for free
456 /// as long as it funnels through one of those two functions.
457 ///
458 /// `ok=false` is exempt on every route (this single early-return IS
459 /// the exemption, reused identically by both embedding sites — see
460 /// issue #51's "ok=false completions are exempt" acceptance
461 /// criterion). An agent with no declared contract, or a contract for
462 /// the OTHER channel, is untouched (`Ok(())`) — same opt-in,
463 /// byte-for-byte-preserving posture as
464 /// [`Self::verdict_contract_for_task`]'s doc.
465 ///
466 /// - `channel: "body"` — `value` (the completing `Final`'s content,
467 /// already reduced to a comparable string by the caller via
468 /// [`content_ref_to_comparable_string`]) must be a member of
469 /// `contract.values`.
470 /// - `channel: "part"` — [`Self::staged_verdict_value_for`] must find
471 /// a staged `"verdict"` artifact for this attempt (presence,
472 /// defense in depth over the staging-time membership check) AND its
473 /// value must be a member of `contract.values`.
474 async fn verdict_contract_completion_check(
475 &self,
476 task_id: &StepId,
477 attempt: u32,
478 ok: bool,
479 value: &str,
480 ) -> Result<(), EngineError> {
481 if !ok {
482 return Ok(());
483 }
484 let Some(contract) = self.verdict_contract_for_task(task_id).await else {
485 return Ok(());
486 };
487 match contract.channel {
488 mlua_swarm_schema::VerdictChannel::Body => {
489 if contract.values.iter().any(|v| v == value) {
490 Ok(())
491 } else {
492 Err(EngineError::VerdictValueRejected {
493 value: value.to_string(),
494 allowed: contract.values.clone(),
495 })
496 }
497 }
498 mlua_swarm_schema::VerdictChannel::Part => {
499 match self.staged_verdict_value_for(task_id, attempt).await {
500 None => Err(EngineError::VerdictPartMissing {
501 allowed: contract.values.clone(),
502 }),
503 Some(staged) if contract.values.iter().any(|v| v == &staged) => Ok(()),
504 Some(staged) => Err(EngineError::VerdictValueRejected {
505 value: staged,
506 allowed: contract.values.clone(),
507 }),
508 }
509 }
510 }
511 }
512
513 // ═══════════════════════════════════════════════════════════════════════
514 // §7 with_state — single Mutex + R1-R4 (try_lock + bounded retry + max-hold panic)
515 // ═══════════════════════════════════════════════════════════════════════
516
517 /// The closure is a **sync** `FnOnce` — you cannot pass an async
518 /// closure, which enforces R3 at the type level. Exceeding `max_hold`
519 /// panics so that R4 violations surface immediately.
520 pub async fn with_state<F, R>(&self, op: &'static str, f: F) -> Result<R, EngineError>
521 where
522 F: FnOnce(&mut EngineState) -> R,
523 {
524 let cfg = &self.inner.cfg;
525
526 // R2: try_lock + bounded retry
527 let mut guard_opt = None;
528 for attempt in 0..=cfg.max_retry {
529 match self.inner.state.try_lock() {
530 Ok(g) => {
531 guard_opt = Some(g);
532 break;
533 }
534 Err(_) if cfg.try_only => return Err(EngineError::LockBusy(op)),
535 Err(_) => {
536 let backoff = cfg.backoff_ms_step * (attempt as u64 + 1);
537 tokio::time::sleep(Duration::from_millis(backoff)).await;
538 }
539 }
540 }
541 let mut guard = guard_opt.ok_or(EngineError::LockBusyAfterRetry(op))?;
542
543 // R4: max_hold guard
544 let start = Instant::now();
545 let result = f(&mut guard);
546 let elapsed_ms = start.elapsed().as_millis();
547 drop(guard);
548
549 if elapsed_ms > cfg.max_hold_ms {
550 panic!(
551 "Engine.with_state('{op}') held {elapsed_ms}ms > max {}ms — suspected R3 violation (long op inside lock)",
552 cfg.max_hold_ms
553 );
554 }
555 Ok(result)
556 }
557
558 // ═══════════════════════════════════════════════════════════════════════
559 // Token verify (= sig + expire + gate + uses_left)
560 // ═══════════════════════════════════════════════════════════════════════
561
562 /// Four steps: (1) signature verify, (2) expiry check, (3) role × verb
563 /// gate, (4) `uses_left` consume.
564 pub async fn verify_token(&self, token: &CapToken, verb: Verb) -> Result<(), EngineError> {
565 // (1) sig
566 if !self.inner.signer.verify_sig(token) {
567 return Err(EngineError::BadSignature);
568 }
569 // (2) expire
570 if token.is_expired(now_unix()) {
571 return Err(EngineError::TokenExpired);
572 }
573 // (3) role × verb gate
574 if !self.inner.gate.is_allowed(token.role, verb) {
575 return Err(EngineError::RoleViolation {
576 role: token.role,
577 verb,
578 });
579 }
580 // (4) server-side uses_left consume
581 let fp = token.fingerprint();
582 self.with_state("token.consume", move |s| {
583 let rec = s
584 .tokens
585 .get_mut(&fp)
586 .ok_or_else(|| EngineError::TokenNotFound(fp.clone()))?;
587 rec.consume()
588 .map_err(|_: crate::core::state::CapTokenConsumeError| {
589 EngineError::TokenUsesExhausted
590 })?;
591 Ok::<(), EngineError>(())
592 })
593 .await??;
594 Ok(())
595 }
596
597 /// `verify_token` plus the **task-ownership gate**.
598 ///
599 /// When a Worker-role token calls a state-touch verb (`fetch_prompt` /
600 /// `post_result` / `read_task_state` / `cancel_task` / `poll_task`),
601 /// the gate checks that `CapTokenRecord.task_id` matches the argument
602 /// `task_id`; a mismatch returns `EngineError::TokenTaskMismatch`.
603 /// Operator / Senior / Observer tokens are outside the ownership gate
604 /// and may touch any task.
605 ///
606 /// **Verbs exempt from the gate.** `start_task` and `dispatch_attempt`
607 /// stay outside so recursive swarming keeps working; depth is capped
608 /// by `max_spawn_depth`.
609 pub async fn verify_token_for_task(
610 &self,
611 token: &CapToken,
612 verb: Verb,
613 task_id: &StepId,
614 ) -> Result<(), EngineError> {
615 self.verify_token(token, verb).await?;
616 if token.role != Role::Worker {
617 return Ok(());
618 }
619 let fp = token.fingerprint();
620 let arg_tid = task_id.clone();
621 self.with_state("token.ownership_gate", move |s| {
622 let bound = s.tokens.get(&fp).and_then(|r| r.task_id.as_ref()).cloned();
623 match bound {
624 Some(t) if t == arg_tid => Ok(()),
625 Some(t) => Err(EngineError::TokenTaskMismatch {
626 bound: t.into_string(),
627 arg: arg_tid.into_string(),
628 }),
629 None => Err(EngineError::TokenNotFound(fp.clone())),
630 }
631 })
632 .await??;
633 Ok(())
634 }
635
636 /// Resolve the bound `task_id` from a Worker-role token. Used on the
637 /// simple `/v1/worker/submit` endpoint, where the worker POSTs with a
638 /// token but no `task_id`. Returns `Err` if the token role is not
639 /// Worker, or if no bound task is set.
640 pub async fn task_id_from_token(&self, token: &CapToken) -> Result<StepId, EngineError> {
641 if token.role != Role::Worker {
642 return Err(EngineError::RoleViolation {
643 role: token.role,
644 verb: Verb::PostResult,
645 });
646 }
647 let fp = token.fingerprint();
648 self.with_state("task_id_from_token", move |s| {
649 s.tokens
650 .get(&fp)
651 .and_then(|r| r.task_id.as_ref())
652 .cloned()
653 .ok_or_else(|| EngineError::TokenNotFound(fp.clone()))
654 })
655 .await?
656 }
657
658 /// Resolve a short worker handle (`wh-XXXXXXXX`) to the bound
659 /// `task_id`. Used on `/v1/worker/submit` when the Bearer is a short
660 /// handle string rather than a full `CapToken` JSON. A missing entry
661 /// returns `TokenNotFound`, i.e. "the handle is not in the store".
662 pub async fn task_id_from_handle(&self, handle: &str) -> Result<StepId, EngineError> {
663 let h = handle.to_string();
664 self.with_state("task_id_from_handle", move |s| {
665 let fp = s
666 .worker_handles
667 .get(&h)
668 .cloned()
669 .ok_or_else(|| EngineError::TokenNotFound(format!("handle={h}")))?;
670 s.tokens
671 .get(&fp)
672 .and_then(|r| r.task_id.as_ref())
673 .cloned()
674 .ok_or_else(|| EngineError::TokenNotFound(format!("fp={fp}")))
675 })
676 .await?
677 }
678
679 /// Submit a worker result via a short handle. Skips token verification
680 /// and updates `output_tail` `Final` + `task.last_result` directly in
681 /// a thin path. The caller is expected to have already resolved
682 /// `task_id` via `task_id_from_handle` — the handle's presence in
683 /// `worker_handles` means it was minted server-side and is therefore
684 /// trusted.
685 ///
686 /// # GH #76 Skip tier: `outcome: SubmitOutcome`
687 ///
688 /// The `outcome` parameter (replacing the pre-#76 `ok: bool`) is the
689 /// caller's tier declaration:
690 ///
691 /// | outcome | `Final.ok` | `Final.content` | verdict-contract check |
692 /// |----------|------------|--------------------------------|------------------------|
693 /// | `Pass` | `true` | `value` verbatim | fires |
694 /// | `Blocked`| `false` | `value` verbatim | exempt (`ok=false`) |
695 /// | `Skip` | `true` | `wrap_skip_marker(value)` | exempt (Skip opt-out) |
696 ///
697 /// The Skip tier is opt-out from the verdict-contract completion check
698 /// on the same rationale [`crate::core::state::SubmitOutcome::Skip`]'s
699 /// doc records: the agent explicitly declared "not applicable", so
700 /// the payload is not a real verdict value to gate.
701 pub async fn submit_worker_result_trusted(
702 &self,
703 task_id: &StepId,
704 attempt: u32,
705 value: Value,
706 outcome: SubmitOutcome,
707 ) -> Result<(), EngineError> {
708 // Resolve outcome into the wire-level (value, ok, run_contract)
709 // triple exactly once, then reuse it below. Keeping the mapping
710 // literal in one place makes the "Skip wraps + skips contract"
711 // invariant grep-visible.
712 let (wire_value, wire_ok, run_contract_check) = match outcome {
713 SubmitOutcome::Pass => (value, true, true),
714 SubmitOutcome::Blocked => (value, false, false),
715 SubmitOutcome::Skip => (wrap_skip_marker(value), true, false),
716 };
717
718 // GH #51 — completion-time verdict-contract enforcement, embedded
719 // choke point 1 of 2 (see `Self::verdict_contract_completion_check`'s
720 // doc). This path always submits a `Final` by construction (there
721 // is no other event kind on `/v1/worker/submit`), so the check
722 // always applies — unlike `submit_output` below, no `if let
723 // OutputEvent::Final { .. }` guard is needed here since there is
724 // no other `OutputEvent` variant this function could be asked to
725 // write. Runs BEFORE the `output_tail` write immediately below:
726 // on `Err`, this returns immediately and neither `with_state` call
727 // in this function executes.
728 //
729 // GH #76 Skip tier: gated on `run_contract_check` — Skip is opt-out
730 // (see the outcome mapping table above), Blocked stays exempt via
731 // `verdict_contract_completion_check`'s existing `ok=false`
732 // early return (redundant flag here for grep locality).
733 if run_contract_check {
734 let comparable_value =
735 content_ref_to_comparable_string(crate::worker::output::ContentRef::Inline {
736 value: wire_value.clone(),
737 });
738 self.verdict_contract_completion_check(task_id, attempt, wire_ok, &comparable_value)
739 .await?;
740 }
741 let task_id_for_apply = task_id.clone();
742 let value_for_event = wire_value.clone();
743 self.with_state("submit_worker_result_trusted.output", move |s| {
744 let ev = crate::worker::output::OutputEvent::Final {
745 content: crate::worker::output::ContentRef::Inline {
746 value: value_for_event,
747 },
748 ok: wire_ok,
749 };
750 s.output_store
751 .entry((task_id_for_apply.clone(), attempt))
752 .or_default()
753 .push(ev.clone());
754 s.push_event(crate::core::state::Event::WorkerOutput {
755 task_id: task_id_for_apply,
756 attempt,
757 event: ev,
758 });
759 })
760 .await?;
761 let task_id_for_result = task_id.clone();
762 let value_for_result = wire_value.clone();
763 self.with_state("submit_worker_result_trusted.last_result", move |s| {
764 if let Some(t) = s.tasks.get_mut(&task_id_for_result) {
765 t.last_result = Some(value_for_result);
766 t.updated_at = now_unix();
767 }
768 })
769 .await?;
770 // subtask-4 / ST2 rework: this path always submits a `Final` (there
771 // is no other event kind on `/v1/worker/submit`), so the
772 // submit-time projection sink always fires — see
773 // `materialize_final_submission`'s doc and `submit_output`'s
774 // Invariants (fail-open, never turns a would-have-succeeded submit
775 // into a failure).
776 let content = crate::worker::output::ContentRef::Inline { value: wire_value };
777 self.materialize_final_submission(task_id, attempt, &content, wire_ok)
778 .await?;
779 Ok(())
780 }
781
782 /// Stage a named `Artifact` from a worker via a short handle (GH #36
783 /// ST1: named multi-part worker output). Trusted analog of
784 /// [`Self::submit_worker_result_trusted`] for `OutputEvent::Artifact`:
785 /// skips token verification for the same reason (the caller already
786 /// resolved `task_id` via `task_id_from_handle`, so the handle's
787 /// presence in `worker_handles` is itself the trust boundary).
788 ///
789 /// Appends to the same per-`(task_id, attempt)` `output_store` tail
790 /// [`Self::dispatch_attempt_with`]'s Final-pull later folds into
791 /// `{"out": <final>, "parts": {<name>: <value>, ...}}` (see that
792 /// method's doc for the fold semantics — event order, last-write-wins
793 /// per name), AND records `name` in `EngineState.worker_artifact_names`
794 /// — the fold's allowlist of the WORKER's own staged parts, as opposed
795 /// to every `Artifact` that happens to land on the shared tail (e.g. an
796 /// audit sidecar finding; see that field's doc). Also dual-writes to
797 /// the Data-plane `OutputStore` the same way [`Self::submit_output`]'s
798 /// `Artifact` arm does, via [`Self::materialize_artifact_submission`]
799 /// (the artifact's own `name` is its Data-plane key, no
800 /// canonicalization — see that method's doc).
801 pub async fn stage_worker_artifact_trusted(
802 &self,
803 task_id: &StepId,
804 attempt: u32,
805 name: String,
806 value: Value,
807 ) -> Result<(), EngineError> {
808 let content = crate::worker::output::ContentRef::Inline { value };
809 let task_id_for_apply = task_id.clone();
810 let name_for_apply = name.clone();
811 let content_for_apply = content.clone();
812 self.with_state("stage_worker_artifact_trusted.output", move |s| {
813 let ev = crate::worker::output::OutputEvent::Artifact {
814 name: name_for_apply.clone(),
815 content: content_for_apply,
816 };
817 s.output_store
818 .entry((task_id_for_apply.clone(), attempt))
819 .or_default()
820 .push(ev.clone());
821 s.worker_artifact_names
822 .entry((task_id_for_apply.clone(), attempt))
823 .or_default()
824 .push(name_for_apply);
825 s.push_event(crate::core::state::Event::WorkerOutput {
826 task_id: task_id_for_apply,
827 attempt,
828 event: ev,
829 });
830 })
831 .await?;
832 self.materialize_artifact_submission(task_id, attempt, &name, &content)
833 .await?;
834 Ok(())
835 }
836
837 /// GH #36 ST1: the set of `Artifact` names staged for `(task_id,
838 /// attempt)` via [`Self::stage_worker_artifact_trusted`] — see
839 /// `EngineState.worker_artifact_names`'s doc. Used by
840 /// [`Self::dispatch_attempt_with`]'s Final-pull to distinguish a
841 /// worker's own named parts from any other `Artifact` producer on the
842 /// same tail.
843 async fn worker_artifact_names_for(&self, task_id: &StepId, attempt: u32) -> Vec<String> {
844 let key = (task_id.clone(), attempt);
845 self.with_state("worker_artifact_names_for", move |s| {
846 s.worker_artifact_names
847 .get(&key)
848 .cloned()
849 .unwrap_or_default()
850 })
851 .await
852 .unwrap_or_default()
853 }
854
855 /// Mint a short handle and register it in the `worker_handles` map.
856 /// Called immediately after the worker-token mint inside
857 /// `dispatch_attempt_with`, and issues a handle bound to the same
858 /// token fingerprint. Format is `wh-<8 hex chars>` (11 chars total),
859 /// designed to remove the base64 copy-paste failure mode.
860 async fn mint_worker_handle(&self, worker_fp: String) -> Result<String, EngineError> {
861 // The handle is a sole bearer secret on the `/v1/worker/submit`
862 // short-handle path (`submit_worker_result_trusted` skips token
863 // verification), so it must be unguessable — OS RNG, not the
864 // predictable uid counter. 8 hex chars (~4B entropy) keeps the
865 // documented `wh-<8 hex>` wire shape; collision between live
866 // handles is negligible at in-process handle counts.
867 let short = crate::types::secure_hex(4);
868 let handle = format!("wh-{short}");
869 let h = handle.clone();
870 self.with_state("mint_worker_handle", move |s| {
871 s.worker_handles.insert(h, worker_fp);
872 })
873 .await?;
874 Ok(handle)
875 }
876
877 // ═══════════════════════════════════════════════════════════════════════
878 // Session API
879 // ═══════════════════════════════════════════════════════════════════════
880
881 /// Attach a new session with default `OperatorInfo` (`Automate`, no
882 /// bridges/hooks). Shorthand for `attach_with(.., OperatorInfo::default())`.
883 pub async fn attach(
884 &self,
885 operator_id: impl Into<String>,
886 role: Role,
887 ttl: Duration,
888 ) -> Result<CapToken, EngineError> {
889 self.attach_with(
890 operator_id,
891 role,
892 ttl,
893 crate::core::ctx::OperatorInfo::default(),
894 )
895 .await
896 }
897
898 // ═══════════════════════════════════════════════════════════════════════
899 // BridgeRegistry API.
900 // ═══════════════════════════════════════════════════════════════════════
901
902 /// Register a `SeniorBridge` under a name. An existing entry with the
903 /// same name is overwritten. On the persisted-session reattach path,
904 /// the caller re-registers under the same ID beforehand and the
905 /// bridge becomes effective again.
906 pub async fn register_senior_bridge(
907 &self,
908 id: impl Into<String>,
909 bridge: Arc<dyn SeniorBridge>,
910 ) {
911 self.inner
912 .senior_bridges
913 .write()
914 .await
915 .insert(id.into(), bridge);
916 }
917
918 /// Register a `SpawnHook` under a name. An existing entry with the
919 /// same name is overwritten.
920 pub async fn register_spawn_hook(&self, id: impl Into<String>, hook: Arc<dyn SpawnHook>) {
921 self.inner.spawn_hooks.write().await.insert(id.into(), hook);
922 }
923
924 /// Register an `Operator` (a spawn-body backend) under a name. An
925 /// existing entry with the same name is overwritten.
926 /// `OperatorDelegateMiddleware` looks this up via `ctx` and, when
927 /// `kind = MainAi` / `Composite`, bypasses `inner.spawn` and calls
928 /// `operator.execute` instead.
929 pub async fn register_operator(
930 &self,
931 id: impl Into<String>,
932 operator: Arc<dyn crate::operator::Operator>,
933 ) {
934 self.inner
935 .operators
936 .write()
937 .await
938 .insert(id.into(), operator);
939 }
940
941 /// Unregister a `SeniorBridge` by name (e.g. on WebSocket disconnect
942 /// or explicit teardown). A missing ID is a no-op.
943 pub async fn unregister_senior_bridge(&self, id: &str) {
944 self.inner.senior_bridges.write().await.remove(id);
945 }
946
947 /// Unregister a `SpawnHook` by name. A missing ID is a no-op.
948 pub async fn unregister_spawn_hook(&self, id: &str) {
949 self.inner.spawn_hooks.write().await.remove(id);
950 }
951
952 /// Unregister an `Operator` backend by name. A missing ID is a no-op.
953 pub async fn unregister_operator(&self, id: &str) {
954 self.inner.operators.write().await.remove(id);
955 }
956
957 /// Snapshot the list of registered `SpawnHook` IDs (for test
958 /// observation and debugging).
959 pub async fn list_spawn_hook_ids(&self) -> Vec<String> {
960 self.inner
961 .spawn_hooks
962 .read()
963 .await
964 .keys()
965 .cloned()
966 .collect()
967 }
968
969 /// Snapshot the list of registered `SeniorBridge` IDs.
970 pub async fn list_senior_bridge_ids(&self) -> Vec<String> {
971 self.inner
972 .senior_bridges
973 .read()
974 .await
975 .keys()
976 .cloned()
977 .collect()
978 }
979
980 /// Snapshot the list of registered `Operator` IDs.
981 pub async fn list_operator_ids(&self) -> Vec<String> {
982 self.inner.operators.read().await.keys().cloned().collect()
983 }
984
985 /// Attach specifying IDs directly. The caller is expected to have
986 /// pre-registered them via `register_senior_bridge` /
987 /// `register_spawn_hook` / `register_operator`. This is the canonical
988 /// path when persistence is in play.
989 ///
990 /// `kind` is the "Runtime Global" tier of the `OperatorKind` cascade
991 /// (stored verbatim on `OperatorSession.operator_kind`): `Some(_)` is
992 /// an explicit request (including `Some(OperatorKind::Automate)`) that
993 /// outranks the BP-level tiers; `None` leaves it unspecified so the
994 /// BP-level tiers / final default decide. See
995 /// `crate::core::ctx::collapse_operator_kind`.
996 #[allow(clippy::too_many_arguments)]
997 pub async fn attach_with_ids(
998 &self,
999 operator_id: impl Into<String>,
1000 role: Role,
1001 ttl: Duration,
1002 kind: Option<OperatorKind>,
1003 bridge_id: Option<String>,
1004 hook_id: Option<String>,
1005 operator_backend_id: Option<String>,
1006 operator_kind_overrides: HashMap<String, OperatorKind>,
1007 bp_agent_kinds: HashMap<String, OperatorKind>,
1008 bp_global_kind: Option<OperatorKind>,
1009 ) -> Result<CapToken, EngineError> {
1010 let operator_id = operator_id.into();
1011 let token = self
1012 .inner
1013 .signer
1014 .session(operator_id.clone(), role, vec!["*".into()], ttl);
1015 let session_id = SessionId::new();
1016 let fp = token.fingerprint();
1017 let now = now_unix();
1018 let token_for_store = token.clone();
1019
1020 self.with_state("attach_with_ids", |s| {
1021 s.tokens
1022 .insert(fp.clone(), CapTokenRecord::from_token(token_for_store));
1023 s.sessions.insert(
1024 session_id.clone(),
1025 OperatorSession {
1026 id: session_id.clone(),
1027 operator_id: operator_id.clone(),
1028 role,
1029 attached_at: now,
1030 last_seen: now,
1031 attached: true,
1032 owned_task_ids: Vec::new(),
1033 token_fp: fp.clone(),
1034 operator_kind: kind,
1035 runtime_agent_kinds: operator_kind_overrides,
1036 bp_agent_kinds,
1037 bp_global_kind,
1038 bridge_id,
1039 hook_id,
1040 operator_backend_id,
1041 },
1042 );
1043 s.push_event(Event::SessionAttached {
1044 session_id: session_id.clone(),
1045 role,
1046 });
1047 })
1048 .await?;
1049
1050 let _ = self
1051 .inner
1052 .event_tx
1053 .send(Event::SessionAttached { session_id, role });
1054 Ok(token)
1055 }
1056
1057 /// Build an `OperatorInfo` by looking up the session's registered IDs
1058 /// on the `BridgeRegistry`, plus resolving the 4-tier `OperatorKind`
1059 /// cascade for `agent_name` via `crate::core::ctx::collapse_operator_kind`.
1060 /// Used when `dispatch_attempt` injects `Ctx`. An unresolved ID
1061 /// (nothing registered) is silently `None` — the bridge / hook simply
1062 /// does not fire and the default behaviour applies.
1063 async fn resolve_operator_info(
1064 &self,
1065 session: &OperatorSession,
1066 agent_name: &str,
1067 ) -> OperatorInfo {
1068 let senior_bridge = if let Some(id) = &session.bridge_id {
1069 self.inner.senior_bridges.read().await.get(id).cloned()
1070 } else {
1071 None
1072 };
1073 let spawn_hook = if let Some(id) = &session.hook_id {
1074 self.inner.spawn_hooks.read().await.get(id).cloned()
1075 } else {
1076 None
1077 };
1078 let operator = if let Some(id) = &session.operator_backend_id {
1079 self.inner.operators.read().await.get(id).cloned()
1080 } else {
1081 None
1082 };
1083 let runtime_agent = session.runtime_agent_kinds.get(agent_name).copied();
1084 // "Runtime Global" tier: `Some(_)` is always an explicit request
1085 // (see the field doc on `OperatorSession.operator_kind`).
1086 let runtime_global = session.operator_kind;
1087 let bp_agent = session.bp_agent_kinds.get(agent_name).copied();
1088 let bp_global = session.bp_global_kind;
1089 let kind = crate::core::ctx::collapse_operator_kind(
1090 runtime_agent,
1091 runtime_global,
1092 bp_agent,
1093 bp_global,
1094 );
1095 OperatorInfo {
1096 kind,
1097 id: session.operator_id.clone(),
1098 senior_bridge,
1099 spawn_hook,
1100 operator,
1101 }
1102 }
1103
1104 /// Convenience attach that takes an `OperatorInfo` (three
1105 /// `Arc<dyn ...>` fields plus `kind`) **inline**.
1106 ///
1107 /// # Pipeline
1108 ///
1109 /// Each `Arc<dyn ...>` is auto-registered on the engine's registry
1110 /// under a synthetic ID (`br-<hex>` / `hk-<hex>` / `ob-<hex>`), and
1111 /// the session stores that synthetic ID. Subsequent `dispatch_attempt`
1112 /// calls rebuild the `Arc`s from those IDs via
1113 /// `resolve_operator_info`, and the three middlewares fire as usual.
1114 ///
1115 /// # ⚠ Non-persisted sessions only
1116 ///
1117 /// Because this API takes inline `Arc`s, the reattach path after
1118 /// session persistence cannot rebuild them — the synthetic IDs are
1119 /// not present in a freshly started process's registry. If you need
1120 /// persistence, use [`Self::attach_with_ids`] with `register_*` calls
1121 /// beforehand to go through **named IDs** instead.
1122 ///
1123 /// Handy for tests and short-lived in-process sessions. Production
1124 /// WebSocket callbacks and the like should prefer `attach_with_ids`
1125 /// as the canonical path.
1126 pub async fn attach_with(
1127 &self,
1128 operator_id: impl Into<String>,
1129 role: Role,
1130 ttl: Duration,
1131 operator_info: crate::core::ctx::OperatorInfo,
1132 ) -> Result<CapToken, EngineError> {
1133 let operator_id = operator_id.into();
1134 // The caller always hands in a fully-formed `OperatorInfo`
1135 // (including its `kind`), so it is stored as an explicit "Runtime
1136 // Global" tier request (`Some(kind)`) — this path never persists
1137 // BP-level tiers (both stay empty below), so `Some(kind)` resolves
1138 // to the same `kind` at dispatch either way; see
1139 // `OperatorSession.operator_kind` doc.
1140 let kind = operator_info.kind;
1141 // BridgeRegistry auto-register: when the caller hands in an
1142 // `Arc<dyn>` directly, register it under a synthesised ID (the inline
1143 // path aware of persistence). Callers who want to pre-register with a
1144 // named ID should use `register_senior_bridge` / `register_spawn_hook`
1145 // + `attach_with_ids`.
1146 let bridge_id = if let Some(bridge) = operator_info.senior_bridge.clone() {
1147 let id = format!("br-{}", crate::types::uid_hex(8));
1148 self.inner
1149 .senior_bridges
1150 .write()
1151 .await
1152 .insert(id.clone(), bridge);
1153 Some(id)
1154 } else {
1155 None
1156 };
1157 let hook_id = if let Some(hook) = operator_info.spawn_hook.clone() {
1158 let id = format!("hk-{}", crate::types::uid_hex(8));
1159 self.inner
1160 .spawn_hooks
1161 .write()
1162 .await
1163 .insert(id.clone(), hook);
1164 Some(id)
1165 } else {
1166 None
1167 };
1168 let operator_backend_id = if let Some(operator) = operator_info.operator.clone() {
1169 // `ob-` = operator-backend registry id. Renamed from `op-` in the
1170 // issue #11 prefix reconciliation: `op-` used to collide with the
1171 // WS operator sid shape (now unified into `S-<hex>` anyway), and a
1172 // shared prefix across two unrelated registries made log filtering
1173 // by prefix silently ambiguous.
1174 let id = format!("ob-{}", crate::types::uid_hex(8));
1175 self.inner
1176 .operators
1177 .write()
1178 .await
1179 .insert(id.clone(), operator);
1180 Some(id)
1181 } else {
1182 None
1183 };
1184
1185 let token = self
1186 .inner
1187 .signer
1188 .session(operator_id.clone(), role, vec!["*".into()], ttl);
1189 let session_id = SessionId::new();
1190 let fp = token.fingerprint();
1191 let now = now_unix();
1192 let token_for_store = token.clone();
1193
1194 self.with_state("attach_with", |s| {
1195 s.tokens
1196 .insert(fp.clone(), CapTokenRecord::from_token(token_for_store));
1197 s.sessions.insert(
1198 session_id.clone(),
1199 OperatorSession {
1200 id: session_id.clone(),
1201 operator_id,
1202 role,
1203 attached_at: now,
1204 last_seen: now,
1205 attached: true,
1206 owned_task_ids: Vec::new(),
1207 token_fp: fp.clone(),
1208 operator_kind: Some(kind),
1209 runtime_agent_kinds: HashMap::new(),
1210 bp_agent_kinds: HashMap::new(),
1211 bp_global_kind: None,
1212 bridge_id,
1213 hook_id,
1214 operator_backend_id,
1215 },
1216 );
1217 s.push_event(Event::SessionAttached {
1218 session_id: session_id.clone(),
1219 role,
1220 });
1221 })
1222 .await?;
1223
1224 let _ = self
1225 .inner
1226 .event_tx
1227 .send(Event::SessionAttached { session_id, role });
1228 Ok(token)
1229 }
1230
1231 /// Mark the session bound to `token` as detached (`attached = false`).
1232 /// Tasks are left in place — a later `attach`/`attach_with_ids` call
1233 /// carrying the same registered bridge/hook IDs can pick them back up.
1234 pub async fn detach(&self, token: &CapToken) -> Result<(), EngineError> {
1235 self.verify_token(token, Verb::DetachSession).await?;
1236 let fp = token.fingerprint();
1237 self.with_state("detach", move |s| {
1238 let sid = s
1239 .sessions
1240 .iter()
1241 .find(|(_, sess)| sess.token_fp == fp)
1242 .map(|(id, _)| id.clone());
1243 if let Some(sid) = sid {
1244 if let Some(sess) = s.sessions.get_mut(&sid) {
1245 sess.attached = false;
1246 }
1247 s.push_event(Event::SessionDetached {
1248 session_id: sid.clone(),
1249 });
1250 let _ = sid;
1251 }
1252 })
1253 .await?;
1254 Ok(())
1255 }
1256
1257 /// Refresh the session's `last_seen` timestamp and mark it `attached`.
1258 /// Called periodically by an attached client to avoid being flipped to
1259 /// detached by `start_detach_loop`.
1260 pub async fn heartbeat(&self, token: &CapToken) -> Result<(), EngineError> {
1261 self.verify_token(token, Verb::Heartbeat).await?;
1262 let now = now_unix();
1263 let fp = token.fingerprint();
1264 self.with_state("heartbeat", move |s| {
1265 if let Some(sess) = s.sessions.values_mut().find(|sess| sess.token_fp == fp) {
1266 sess.last_seen = now;
1267 sess.attached = true;
1268 }
1269 })
1270 .await?;
1271 Ok(())
1272 }
1273
1274 // ═══════════════════════════════════════════════════════════════════════
1275 // Task lifecycle
1276 // ═══════════════════════════════════════════════════════════════════════
1277
1278 /// Create a new `TaskState` from `spec` and register its initial
1279 /// prompt. When the calling token is a Worker (i.e. this is a
1280 /// recursive spawn), the new task inherits `parent.spawn_depth + 1`
1281 /// and is rejected with `SpawnDepthExceeded` once `max_spawn_depth` is
1282 /// hit; an Operator-issued call starts at depth 0.
1283 pub async fn start_task(
1284 &self,
1285 token: &CapToken,
1286 spec: TaskSpec,
1287 ) -> Result<StepId, EngineError> {
1288 self.verify_token(token, Verb::StartTask).await?;
1289 let task_id = StepId::new();
1290 let initial_directive = spec.initial_directive.clone();
1291 let task_id_clone = task_id.clone();
1292 let fp = token.fingerprint();
1293 let max_depth = self.inner.cfg.max_spawn_depth;
1294 self.with_state("start_task", move |s| {
1295 // Recursive swarm depth gate (recursion guard):
1296 // Worker tokens carry CapTokenRecord.parent_task_id. Give the
1297 // child parent's spawn_depth + 1; if it exceeds `max`, raise an
1298 // error. Operator tokens (parent_task_id=None) start at depth 0.
1299 let parent_depth_opt = s
1300 .tokens
1301 .get(&fp)
1302 .and_then(|rec| rec.task_id.as_ref())
1303 .and_then(|tid| s.tasks.get(tid))
1304 .map(|t| t.spawn_depth);
1305 let depth = match parent_depth_opt {
1306 Some(d) => {
1307 if d + 1 >= max_depth {
1308 return Err(EngineError::SpawnDepthExceeded {
1309 current: d + 1,
1310 max: max_depth,
1311 });
1312 }
1313 d + 1
1314 }
1315 None => 0,
1316 };
1317
1318 let mut task = TaskState::new(task_id_clone.clone(), spec);
1319 task.spawn_depth = depth;
1320 s.tasks.insert(task_id_clone.clone(), task);
1321 s.prompts
1322 .insert((task_id_clone.clone(), 1), initial_directive);
1323 // Link to the owner session (only Operator tokens match; Worker tokens have no session).
1324 if let Some(sess) = s.sessions.values_mut().find(|sess| sess.token_fp == fp) {
1325 sess.owned_task_ids.push(task_id_clone.clone());
1326 }
1327 s.push_event(Event::TaskCreated {
1328 task_id: task_id_clone.clone(),
1329 });
1330 Ok::<(), EngineError>(())
1331 })
1332 .await??;
1333 let _ = self.inner.event_tx.send(Event::TaskCreated {
1334 task_id: task_id.clone(),
1335 });
1336 Ok(task_id)
1337 }
1338
1339 /// Fetch a snapshot of `TaskState` for `task_id`, subject to the
1340 /// task-ownership gate (see `verify_token_for_task`).
1341 pub async fn read_task_state(
1342 &self,
1343 token: &CapToken,
1344 task_id: &StepId,
1345 ) -> Result<TaskState, EngineError> {
1346 self.verify_token_for_task(token, Verb::ReadTaskState, task_id)
1347 .await?;
1348 let task_id = task_id.clone();
1349 self.with_state("read_task_state", move |s| {
1350 s.tasks
1351 .get(&task_id)
1352 .cloned()
1353 .ok_or_else(|| EngineError::TaskNotFound(task_id.to_string()))
1354 })
1355 .await?
1356 }
1357
1358 /// Mark `task_id` as `Cancelled` and wake any caller blocked in
1359 /// `poll_task` for it.
1360 pub async fn cancel_task(&self, token: &CapToken, task_id: &StepId) -> Result<(), EngineError> {
1361 self.verify_token_for_task(token, Verb::CancelTask, task_id)
1362 .await?;
1363 let tid = task_id.clone();
1364 self.with_state("cancel_task", move |s| {
1365 let task = s
1366 .tasks
1367 .get_mut(&tid)
1368 .ok_or_else(|| EngineError::TaskNotFound(tid.to_string()))?;
1369 task.status = TaskStatus::Cancelled;
1370 task.updated_at = now_unix();
1371 s.push_event(Event::TaskCancelled {
1372 task_id: tid.clone(),
1373 });
1374 Ok::<(), EngineError>(())
1375 })
1376 .await??;
1377 self.wake_task(task_id).await?;
1378 Ok(())
1379 }
1380
1381 /// Dispatch a single attempt through the given `spawner`.
1382 ///
1383 /// The lock is only held for snapshot capture; the actual spawn and
1384 /// completion await happen outside the lock (R3 discipline).
1385 ///
1386 /// Sits on the Domain side of the Data / Domain split. The dispatch
1387 /// path itself does not touch big response bodies — those flow through
1388 /// the Data plane (`output_store` module + sink / input_inject
1389 /// `SpawnerLayer`s) around this method.
1390 ///
1391 /// The caller does the compile plus `service::linker::link` and
1392 /// carries the same stack through each dispatch. Because the spawner
1393 /// is passed per-request rather than looked up from engine-global
1394 /// state, parallel requests against a single `Engine` instance
1395 /// (different Blueprints, different spawners) do not race.
1396 ///
1397 /// `run_id`, when `Some` (issue #13 run_id propagation —
1398 /// `EngineDispatcher` threads it in from its `RunContext`), is
1399 /// inserted into `Ctx.meta.runtime["run_id"]` (a plain JSON string)
1400 /// alongside `worker_handle`, so `Operator::execute` implementations
1401 /// (e.g. `WSOperatorSession`) can read it back and surface it to the
1402 /// worker (Spawn directive / prompt). `None` (every pre-existing
1403 /// caller / test) omits the key entirely — unchanged behavior.
1404 pub async fn dispatch_attempt_with(
1405 &self,
1406 token: &CapToken,
1407 task_id: &StepId,
1408 spawner: &Arc<dyn SpawnerAdapter>,
1409 run_id: Option<&RunId>,
1410 ) -> Result<DispatchOutcome, EngineError> {
1411 self.verify_token(token, Verb::DispatchAttempt).await?;
1412 let task_id = task_id.clone();
1413
1414 // 1) Under the lock: increment the attempt number, mark Running, snapshot the
1415 // prompt, and pull `operator_info` from the session so we can inject it into Ctx.
1416 let fp = token.fingerprint();
1417 let tid_for_prep = task_id.clone();
1418 let (attempt, agent, session_snapshot, step_ctx) = self
1419 .with_state("dispatch.prep", move |s| {
1420 let task = s
1421 .tasks
1422 .get_mut(&tid_for_prep)
1423 .ok_or_else(|| EngineError::TaskNotFound(tid_for_prep.to_string()))?;
1424 task.attempt += 1;
1425 task.status = TaskStatus::Running;
1426 task.updated_at = now_unix();
1427 // The spawner pulls the prompt via engine.fetch_prompt. In prep,
1428 // if the prompts table has no entry for this attempt yet,
1429 // fall back and insert `initial_directive` so the subsequent
1430 // fetch_prompt succeeds.
1431 let attempt = task.attempt;
1432 let initial = task.spec.initial_directive.clone();
1433 s.prompts
1434 .entry((tid_for_prep.clone(), attempt))
1435 .or_insert(initial);
1436 let task = s
1437 .tasks
1438 .get(&tid_for_prep)
1439 .ok_or_else(|| EngineError::TaskNotFound(tid_for_prep.to_string()))?;
1440 let agent = task.spec.agent.clone();
1441 // GH #21 Phase 2: re-read `TaskSpec.step_ctx` on EVERY
1442 // attempt (not cached once at start_task) so retries and
1443 // Run-rekicks all carry the Step tier through to Ctx —
1444 // see TaskSpec.step_ctx's doc.
1445 let step_ctx = task.spec.step_ctx.clone();
1446 // Session snapshot (looked up by token nonce). When no session
1447 // exists (worker token invoked directly / test injection), fall
1448 // back to None → default OperatorInfo.
1449 let sess_clone = s
1450 .sessions
1451 .values()
1452 .find(|sess| sess.token_fp == fp)
1453 .cloned();
1454 Ok::<_, EngineError>((attempt, agent, sess_clone, step_ctx))
1455 })
1456 .await??;
1457 // BridgeRegistry lookup + per-agent OperatorKind cascade.
1458 let operator_info = match session_snapshot {
1459 Some(sess) => self.resolve_operator_info(&sess, &agent).await,
1460 None => OperatorInfo::default(),
1461 };
1462
1463 // 2) Outside the lock: worker token mint + spawn.
1464 //
1465 // Session-style mint (max_uses=None). Within one attempt the worker is
1466 // expected to hit `verify_token + fetch_prompt + fetch_data + post_result`
1467 // multiple times in order, so `one_time` would exhaust the token on the
1468 // very first verb. Capability is guarded by (a) the role × verb gate and
1469 // (b) the short TTL (1800s).
1470 let worker_token = self.inner.signer.session(
1471 format!("worker-of-{task_id}"),
1472 Role::Worker,
1473 vec!["*".into()],
1474 Duration::from_secs(1800),
1475 );
1476 let worker_fp = worker_token.fingerprint();
1477 let task_id_for_worker = task_id.clone();
1478 let worker_token_for_store = worker_token.clone();
1479 self.with_state("dispatch.mint_worker", move |s| {
1480 s.tokens.insert(
1481 worker_fp,
1482 CapTokenRecord::from_worker_token(worker_token_for_store, task_id_for_worker),
1483 );
1484 })
1485 .await?;
1486
1487 // Mint a short handle (`wh-XXXXXXXX`) and register it in worker_handles.
1488 // Used by the simplified Bearer path for SubAgents (short-handle form
1489 // avoids base64 copy-paste incidents).
1490 let worker_handle = self.mint_worker_handle(worker_token.fingerprint()).await?;
1491
1492 let mut ctx = Ctx::new(task_id.clone(), attempt, agent.clone());
1493 ctx.operator = operator_info; // activates MainAIMiddleware / Senior bridge
1494 ctx.meta
1495 .runtime
1496 .insert("worker_handle".to_string(), Value::String(worker_handle));
1497 if let Some(rid) = run_id {
1498 ctx.meta
1499 .runtime
1500 .insert(RUN_ID_KEY.to_string(), Value::String(rid.to_string()));
1501 }
1502 // GH #21 Phase 2: the Step tier's resolved context bundle (from
1503 // `TaskSpec.step_ctx`, re-read every attempt above) — consumed by
1504 // `AgentContextMiddleware`, which unpacks its keys ahead of the
1505 // Agent / BP-global tiers.
1506 if let Some(step_ctx) = step_ctx {
1507 ctx.meta.runtime.insert(STEP_CTX_KEY.to_string(), step_ctx);
1508 }
1509
1510 let worker = spawner
1511 .spawn(self, &ctx, task_id.clone(), attempt, worker_token)
1512 .await
1513 .map_err(|e| EngineError::DispatchFailed(e.to_string()))?;
1514
1515 // 3) Outside the lock: await worker.join() (signal-only). WorkerError is
1516 // stringified. The value is fetched via output_tail (sink path).
1517 let signal_result: Result<(), String> = worker.join().await.map_err(|e| e.to_string());
1518
1519 // Pull the last Final from output_tail and use it as the value. GH
1520 // #36 ST1 (named multi-part worker output): also fold every
1521 // `Artifact` the WORKER ITSELF staged on the same tail (via
1522 // `stage_worker_artifact_trusted` / `POST /v1/worker/artifact`)
1523 // into a `"parts"` object keyed by name — event order,
1524 // last-write-wins per name (a name staged twice overwrites,
1525 // mirroring `HashMap`/`Map` insert semantics, not an accumulating
1526 // list). `worker_artifact_names_for` is the allowlist that scopes
1527 // this to the worker's own opt-in parts — an `Artifact` some OTHER
1528 // producer appended to this same tail (e.g.
1529 // `AfterRunAuditMiddleware`'s `"audit:<step_ref>"` sidecar finding)
1530 // is left untouched (see `fold_final_and_parts`'s doc). When at
1531 // least one part was staged, the BP-chain value becomes `{"out":
1532 // <final value>, "parts": {...}}`; zero parts staged (the
1533 // pre-GH-#36 case, and every non-opt-in step) leaves the value
1534 // exactly the plain `Final` value, byte-identical to before this
1535 // change.
1536 let value_ok: Result<(Value, bool), String> = match signal_result {
1537 Ok(()) => {
1538 let tail = self.output_tail(&task_id, attempt).await;
1539 let staged_names = self.worker_artifact_names_for(&task_id, attempt).await;
1540 fold_final_and_parts(&tail, &staged_names)
1541 .ok_or_else(|| "no Final in output_tail".to_string())
1542 }
1543 Err(msg) => Err(msg),
1544 };
1545
1546 // 4) Under the lock: apply (split the borrow scope so push_event and task mut can co-exist).
1547 let outcome = self
1548 .with_state("dispatch.apply", |s| {
1549 if !s.tasks.contains_key(&task_id) {
1550 return Err(EngineError::TaskNotFound(task_id.to_string()));
1551 }
1552 match value_ok {
1553 Ok((value, ok)) => {
1554 // GH #76 Skip tier: a Final with ok=true carrying the
1555 // skip-marker sentinel is a Skip tier completion,
1556 // not an ordinary Pass. TaskStatus stays `Pass`
1557 // (the worker itself completed successfully);
1558 // the Skip signal rides on DispatchOutcome so
1559 // EngineDispatcher::dispatch can route it to
1560 // the flow-continuation-without-binding-write
1561 // sentinel path.
1562 let skip_inner = if ok { unwrap_skip_marker(&value) } else { None };
1563 let pass = ok;
1564 {
1565 let task = s.tasks.get_mut(&task_id).unwrap();
1566 task.last_result = Some(value.clone());
1567 task.updated_at = now_unix();
1568 task.status = if pass {
1569 TaskStatus::Pass
1570 } else {
1571 TaskStatus::Blocked
1572 };
1573 }
1574 s.push_event(Event::TaskAttemptCompleted {
1575 task_id: task_id.clone(),
1576 attempt,
1577 result: value.clone(),
1578 });
1579 if let Some(inner) = skip_inner {
1580 s.push_event(Event::TaskPass {
1581 task_id: task_id.clone(),
1582 result: value.clone(),
1583 });
1584 Ok::<_, EngineError>(DispatchOutcome::Skip(inner))
1585 } else if pass {
1586 s.push_event(Event::TaskPass {
1587 task_id: task_id.clone(),
1588 result: value.clone(),
1589 });
1590 Ok::<_, EngineError>(DispatchOutcome::Pass(value))
1591 } else {
1592 s.push_event(Event::TaskBlocked {
1593 task_id: task_id.clone(),
1594 result: value.clone(),
1595 });
1596 Ok(DispatchOutcome::Blocked(value))
1597 }
1598 }
1599 Err(msg) => {
1600 let task = s.tasks.get_mut(&task_id).unwrap();
1601 task.status = TaskStatus::Blocked;
1602 task.updated_at = now_unix();
1603 Err(EngineError::DispatchFailed(msg))
1604 }
1605 }
1606 })
1607 .await??;
1608
1609 // event broadcast (outside the lock — push_event feeds the in-memory tail; broadcast is a separate path).
1610 let _ = self.inner.event_tx.send(Event::TaskAttemptCompleted {
1611 task_id: task_id.clone(),
1612 attempt,
1613 result: match &outcome {
1614 DispatchOutcome::Pass(v)
1615 | DispatchOutcome::Blocked(v)
1616 | DispatchOutcome::Skip(v) => v.clone(),
1617 _ => Value::Null,
1618 },
1619 });
1620
1621 // Wake any callers waiting in poll_task.
1622 self.wake_task(&task_id).await?;
1623
1624 Ok(outcome)
1625 }
1626
1627 /// Dispatch a single attempt, opt-in to the replay-log Core primitive
1628 /// ([`crate::store::replay`]) via `run_ctx`.
1629 ///
1630 /// This is the [`Self::dispatch_attempt_with`] sibling used by callers
1631 /// that carry a `RunContext` with `replay_store` / `replay_cursor`
1632 /// populated. Behavior versus the plain `dispatch_attempt_with`:
1633 ///
1634 /// - **`run_ctx.replay_cursor` is `Some` AND the cursor has a matching
1635 /// `(step_ref, input_hash, occurrence)` row** — the stored value is
1636 /// returned verbatim as `DispatchOutcome::Pass(v)`; the `Adapter`
1637 /// (spawner + worker) is never touched. The task's `attempt` is
1638 /// still bumped and `TaskStatus` set to `Pass`, so downstream state
1639 /// (`task.last_result`, `TaskAttemptCompleted` / `TaskPass` events,
1640 /// `wake_task`) fires the same way an ordinary Pass would.
1641 /// - **Miss (or `replay_cursor: None`)** — the ordinary spawn path
1642 /// runs. When `run_ctx.replay_store` is `Some` AND the outcome is
1643 /// `Pass`, one `ReplayEntry` is appended carrying the whole `Ctx`
1644 /// snapshot (with `operator` dropped by `#[serde(skip)]`) plus the
1645 /// `step_output` value. `Blocked` / `Err` outcomes are never
1646 /// logged — a partial-failure row would poison the replay path
1647 /// after a subsequent successful retry.
1648 ///
1649 /// `run_ctx: None` collapses to the same behavior as
1650 /// `dispatch_attempt_with(token, task_id, spawner, None)` — no run
1651 /// tracing, no replay.
1652 pub async fn dispatch_attempt_with_run_ctx(
1653 &self,
1654 token: &CapToken,
1655 task_id: &StepId,
1656 spawner: &Arc<dyn SpawnerAdapter>,
1657 run_ctx: Option<&RunContext>,
1658 ) -> Result<DispatchOutcome, EngineError> {
1659 self.verify_token(token, Verb::DispatchAttempt).await?;
1660 let task_id = task_id.clone();
1661
1662 // 1) Under the lock: prep (bump attempt, snapshot agent/directive).
1663 let fp = token.fingerprint();
1664 let tid_for_prep = task_id.clone();
1665 let (attempt, agent, session_snapshot, step_ctx, initial_directive) = self
1666 .with_state("dispatch_run_ctx.prep", move |s| {
1667 let task = s
1668 .tasks
1669 .get_mut(&tid_for_prep)
1670 .ok_or_else(|| EngineError::TaskNotFound(tid_for_prep.to_string()))?;
1671 task.attempt += 1;
1672 task.status = TaskStatus::Running;
1673 task.updated_at = now_unix();
1674 let attempt = task.attempt;
1675 let initial = task.spec.initial_directive.clone();
1676 s.prompts
1677 .entry((tid_for_prep.clone(), attempt))
1678 .or_insert(initial.clone());
1679 let task = s
1680 .tasks
1681 .get(&tid_for_prep)
1682 .ok_or_else(|| EngineError::TaskNotFound(tid_for_prep.to_string()))?;
1683 let agent = task.spec.agent.clone();
1684 let step_ctx = task.spec.step_ctx.clone();
1685 let sess_clone = s
1686 .sessions
1687 .values()
1688 .find(|sess| sess.token_fp == fp)
1689 .cloned();
1690 Ok::<_, EngineError>((attempt, agent, sess_clone, step_ctx, initial))
1691 })
1692 .await??;
1693
1694 let operator_info = match session_snapshot {
1695 Some(sess) => self.resolve_operator_info(&sess, &agent).await,
1696 None => OperatorInfo::default(),
1697 };
1698
1699 // 2) Compute the replay key from step_ref (= agent) + hashed input.
1700 // Occurrence comes from the cursor's per-key counter (bumped
1701 // once per dispatch, so a loop that re-visits the same step
1702 // with the same input gets 0, 1, 2, … distinct rows).
1703 let step_ref = agent.clone();
1704 let input_hash = match run_ctx.and_then(|rc| rc.binding_digests.get(&step_ref)) {
1705 Some(binding_digest) => hash_input_value(&serde_json::json!({
1706 "input": initial_directive,
1707 "binding_digest": binding_digest,
1708 })),
1709 None => hash_input_value(&initial_directive),
1710 };
1711 let (replay_hit_value, occurrence) = if let Some(rc) = run_ctx {
1712 if let Some(cursor) = &rc.replay_cursor {
1713 let mut guard = cursor.lock().expect("replay cursor mutex poisoned");
1714 let occ = guard.next_occurrence(&step_ref, &input_hash);
1715 let hit = guard.find(&step_ref, &input_hash, occ);
1716 (hit, occ)
1717 } else {
1718 (None, 0)
1719 }
1720 } else {
1721 (None, 0)
1722 };
1723
1724 // 3) Build the Ctx that (a) either the spawner will see on a miss,
1725 // or (b) we log alongside the replay row.
1726 let mut ctx = Ctx::new(task_id.clone(), attempt, agent.clone());
1727 ctx.operator = operator_info;
1728 if let Some(rc) = run_ctx {
1729 ctx.meta
1730 .runtime
1731 .insert(RUN_ID_KEY.to_string(), Value::String(rc.run_id.to_string()));
1732 }
1733 if let Some(step_ctx) = step_ctx {
1734 ctx.meta.runtime.insert(STEP_CTX_KEY.to_string(), step_ctx);
1735 }
1736
1737 // 4) Replay-hit shortcut: skip the spawn+join, return stored value.
1738 let was_replay_hit = replay_hit_value.is_some();
1739 let value_ok: Result<(Value, bool), String> = if let Some(stored) = replay_hit_value {
1740 tracing::info!(
1741 task_id = %task_id,
1742 step_ref = %step_ref,
1743 occurrence = occurrence,
1744 "replayed from log; worker dispatch skipped"
1745 );
1746 Ok((stored, true))
1747 } else {
1748 // 5) Ordinary spawn path — mint a worker token+handle, run the
1749 // spawner, join, and pull the last Final from output_tail.
1750 let worker_token = self.inner.signer.session(
1751 format!("worker-of-{task_id}"),
1752 Role::Worker,
1753 vec!["*".into()],
1754 Duration::from_secs(1800),
1755 );
1756 let worker_fp = worker_token.fingerprint();
1757 let task_id_for_worker = task_id.clone();
1758 let worker_token_for_store = worker_token.clone();
1759 self.with_state("dispatch_run_ctx.mint_worker", move |s| {
1760 s.tokens.insert(
1761 worker_fp,
1762 CapTokenRecord::from_worker_token(worker_token_for_store, task_id_for_worker),
1763 );
1764 })
1765 .await?;
1766 let worker_handle = self.mint_worker_handle(worker_token.fingerprint()).await?;
1767 ctx.meta
1768 .runtime
1769 .insert("worker_handle".to_string(), Value::String(worker_handle));
1770
1771 let worker = spawner
1772 .spawn(self, &ctx, task_id.clone(), attempt, worker_token)
1773 .await
1774 .map_err(|e| EngineError::DispatchFailed(e.to_string()))?;
1775 let signal_result: Result<(), String> = worker.join().await.map_err(|e| e.to_string());
1776 match signal_result {
1777 Ok(()) => {
1778 let tail = self.output_tail(&task_id, attempt).await;
1779 let staged_names = self.worker_artifact_names_for(&task_id, attempt).await;
1780 fold_final_and_parts(&tail, &staged_names)
1781 .ok_or_else(|| "no Final in output_tail".to_string())
1782 }
1783 Err(msg) => Err(msg),
1784 }
1785 };
1786
1787 // 6) Apply — mirrors `dispatch_attempt_with`'s apply arm exactly
1788 // (task.last_result / status update + TaskAttemptCompleted /
1789 // TaskPass / TaskBlocked events).
1790 let outcome = self
1791 .with_state("dispatch_run_ctx.apply", |s| {
1792 if !s.tasks.contains_key(&task_id) {
1793 return Err(EngineError::TaskNotFound(task_id.to_string()));
1794 }
1795 match value_ok {
1796 Ok((value, ok)) => {
1797 // GH #76 Skip tier: Skip tier detection — same shape
1798 // as the sibling `dispatch_attempt_with` apply
1799 // arm above. See that arm's comment for the
1800 // TaskStatus / Event / DispatchOutcome contract.
1801 let skip_inner = if ok { unwrap_skip_marker(&value) } else { None };
1802 let pass = ok;
1803 {
1804 let task = s.tasks.get_mut(&task_id).unwrap();
1805 task.last_result = Some(value.clone());
1806 task.updated_at = now_unix();
1807 task.status = if pass {
1808 TaskStatus::Pass
1809 } else {
1810 TaskStatus::Blocked
1811 };
1812 }
1813 s.push_event(Event::TaskAttemptCompleted {
1814 task_id: task_id.clone(),
1815 attempt,
1816 result: value.clone(),
1817 });
1818 if let Some(inner) = skip_inner {
1819 s.push_event(Event::TaskPass {
1820 task_id: task_id.clone(),
1821 result: value.clone(),
1822 });
1823 Ok::<_, EngineError>(DispatchOutcome::Skip(inner))
1824 } else if pass {
1825 s.push_event(Event::TaskPass {
1826 task_id: task_id.clone(),
1827 result: value.clone(),
1828 });
1829 Ok::<_, EngineError>(DispatchOutcome::Pass(value))
1830 } else {
1831 s.push_event(Event::TaskBlocked {
1832 task_id: task_id.clone(),
1833 result: value.clone(),
1834 });
1835 Ok(DispatchOutcome::Blocked(value))
1836 }
1837 }
1838 Err(msg) => {
1839 let task = s.tasks.get_mut(&task_id).unwrap();
1840 task.status = TaskStatus::Blocked;
1841 task.updated_at = now_unix();
1842 Err(EngineError::DispatchFailed(msg))
1843 }
1844 }
1845 })
1846 .await??;
1847
1848 // 7) On MISS + Pass + replay_store present, append a replay row.
1849 // Replay-HIT rows are already logged from the original run and
1850 // must never be double-logged (Core primitive contract). A
1851 // secondary-persistence failure here (`tracing::warn!` +
1852 // swallow) matches the `run_ctx.run_store.append_step_entry`
1853 // convention in `EngineDispatcher::dispatch`: it must not mask
1854 // the primary dispatch outcome the caller already has in hand.
1855 if !was_replay_hit {
1856 if let (Some(rc), DispatchOutcome::Pass(v)) = (run_ctx, &outcome) {
1857 if let Some(store) = &rc.replay_store {
1858 match ReplayEntry::from_completion(
1859 rc.run_id.clone(),
1860 step_ref.clone(),
1861 input_hash.clone(),
1862 occurrence,
1863 &ctx,
1864 v,
1865 ) {
1866 Ok(entry) => {
1867 if let Err(e) = store.append(entry).await {
1868 tracing::warn!(
1869 run_id = %rc.run_id,
1870 step_ref = %step_ref,
1871 occurrence = occurrence,
1872 error = %e,
1873 "dispatch_attempt_with_run_ctx: replay_store.append failed"
1874 );
1875 }
1876 }
1877 Err(e) => {
1878 tracing::warn!(
1879 run_id = %rc.run_id,
1880 step_ref = %step_ref,
1881 occurrence = occurrence,
1882 error = %e,
1883 "dispatch_attempt_with_run_ctx: ReplayEntry encode failed"
1884 );
1885 }
1886 }
1887 }
1888 }
1889 }
1890
1891 let _ = self.inner.event_tx.send(Event::TaskAttemptCompleted {
1892 task_id: task_id.clone(),
1893 attempt,
1894 result: match &outcome {
1895 DispatchOutcome::Pass(v)
1896 | DispatchOutcome::Blocked(v)
1897 | DispatchOutcome::Skip(v) => v.clone(),
1898 _ => Value::Null,
1899 },
1900 });
1901
1902 self.wake_task(&task_id).await?;
1903
1904 Ok(outcome)
1905 }
1906
1907 // ═══════════════════════════════════════════════════════════════════════
1908 // Worker-side API (= prompt / data fetch + result post)
1909 // ═══════════════════════════════════════════════════════════════════════
1910
1911 /// Fetch the directive/prompt `Value` for `task_id`'s current attempt.
1912 /// Falls back to `initial_directive` when no prompt has been recorded
1913 /// yet for that attempt. Returns the `Value` end-to-end (issue #18);
1914 /// the render down to `String` happens only at the two consumer
1915 /// boundaries — the Worker HTTP path (`fetch_worker_payload*` →
1916 /// `WorkerPayload.prompt: String`) and the WS Spawn frame text
1917 /// render (`operator_ws::session`).
1918 pub async fn fetch_prompt(
1919 &self,
1920 token: &CapToken,
1921 task_id: &StepId,
1922 ) -> Result<Value, EngineError> {
1923 self.verify_token_for_task(token, Verb::FetchPrompt, task_id)
1924 .await?;
1925 let task_id = task_id.clone();
1926 self.with_state("fetch_prompt", move |s| {
1927 let task = s
1928 .tasks
1929 .get(&task_id)
1930 .ok_or_else(|| EngineError::TaskNotFound(task_id.to_string()))?;
1931 s.prompts
1932 .get(&(task_id.clone(), task.attempt.max(1)))
1933 .cloned()
1934 .ok_or_else(|| {
1935 EngineError::ResourceNotFound(format!(
1936 "prompt({}, attempt={})",
1937 task_id, task.attempt
1938 ))
1939 })
1940 })
1941 .await?
1942 }
1943
1944 /// Combined fetch for `HTTP /v1/worker/prompt`: returns `prompt` +
1945 /// (optional) `system` + `agent` + `attempt` in a single round trip.
1946 /// The verb gate reuses `FetchPrompt` — same semantics as "the worker
1947 /// pulls its task input".
1948 ///
1949 /// `system` is the value written by `OperatorSpawner::spawn` through
1950 /// `bake_worker_system_prompt` when it ran; otherwise `None` (no
1951 /// profile present, or the bake never happened).
1952 pub async fn fetch_worker_payload(
1953 &self,
1954 token: &CapToken,
1955 task_id: &StepId,
1956 ) -> Result<crate::types::WorkerPayload, EngineError> {
1957 self.verify_token_for_task(token, Verb::FetchPrompt, task_id)
1958 .await?;
1959 let task_id_clone = task_id.clone();
1960 let mut payload = self
1961 .with_state("fetch_worker_payload", move |s| {
1962 let task = s
1963 .tasks
1964 .get(&task_id_clone)
1965 .ok_or_else(|| EngineError::TaskNotFound(task_id_clone.to_string()))?;
1966 let attempt = task.attempt.max(1);
1967 let prompt = s
1968 .prompts
1969 .get(&(task_id_clone.clone(), attempt))
1970 .cloned()
1971 .ok_or_else(|| {
1972 EngineError::ResourceNotFound(format!(
1973 "prompt({}, attempt={})",
1974 task_id_clone, attempt
1975 ))
1976 })?;
1977 let system = s
1978 .systems
1979 .get(&(task_id_clone.clone(), attempt))
1980 .cloned()
1981 .unwrap_or(None);
1982 let agent = task.spec.agent.clone();
1983 let context = s
1984 .agent_ctx
1985 .get(&(task_id_clone.clone(), attempt))
1986 .map(|e| e.view.clone());
1987 Ok::<_, EngineError>(crate::types::WorkerPayload {
1988 task_id: task_id_clone.clone(),
1989 attempt,
1990 agent,
1991 prompt: render_directive_to_string(&prompt),
1992 system,
1993 context,
1994 system_ref: None,
1995 })
1996 })
1997 .await??;
1998 self.apply_system_ref_threshold(&mut payload).await?;
1999 Ok(payload)
2000 }
2001
2002 /// Fetch a worker payload via a short handle. Skips token verification
2003 /// and returns `prompt` + `system` + `agent` + `attempt` in a thin
2004 /// path. The caller is expected to have already resolved `task_id`
2005 /// via `task_id_from_handle` — the handle's presence in
2006 /// `worker_handles` means it was minted server-side and is therefore
2007 /// trusted.
2008 pub async fn fetch_worker_payload_trusted(
2009 &self,
2010 task_id: &StepId,
2011 ) -> Result<crate::types::WorkerPayload, EngineError> {
2012 let task_id_clone = task_id.clone();
2013 let mut payload = self
2014 .with_state("fetch_worker_payload_trusted", move |s| {
2015 let task = s
2016 .tasks
2017 .get(&task_id_clone)
2018 .ok_or_else(|| EngineError::TaskNotFound(task_id_clone.to_string()))?;
2019 let attempt = task.attempt.max(1);
2020 let prompt = s
2021 .prompts
2022 .get(&(task_id_clone.clone(), attempt))
2023 .cloned()
2024 .ok_or_else(|| {
2025 EngineError::ResourceNotFound(format!(
2026 "prompt({}, attempt={})",
2027 task_id_clone, attempt
2028 ))
2029 })?;
2030 let system = s
2031 .systems
2032 .get(&(task_id_clone.clone(), attempt))
2033 .cloned()
2034 .unwrap_or(None);
2035 let agent = task.spec.agent.clone();
2036 let context = s
2037 .agent_ctx
2038 .get(&(task_id_clone.clone(), attempt))
2039 .map(|e| e.view.clone());
2040 Ok::<_, EngineError>(crate::types::WorkerPayload {
2041 task_id: task_id_clone.clone(),
2042 attempt,
2043 agent,
2044 prompt: render_directive_to_string(&prompt),
2045 system,
2046 context,
2047 system_ref: None,
2048 })
2049 })
2050 .await??;
2051 self.apply_system_ref_threshold(&mut payload).await?;
2052 Ok(payload)
2053 }
2054
2055 /// GH #31: shared threshold-branch tail for
2056 /// [`Self::fetch_worker_payload`] / [`Self::fetch_worker_payload_trusted`].
2057 /// Both build a raw `WorkerPayload` inside `with_state` with `system`
2058 /// populated as before and `system_ref: None`; this runs *outside* any
2059 /// lock (R3 — `SystemRefMode::File`'s `tokio::fs` write is a genuine
2060 /// `.await`, which `with_state`'s sync-closure contract forbids inside
2061 /// the lock) and rewrites `payload.system` / `payload.system_ref` in
2062 /// place per `SystemRefConfig.threshold_bytes`: over-threshold clears
2063 /// `system` and populates `system_ref`; at-or-under-threshold leaves
2064 /// `system` as-is and `system_ref` stays `None`. A no-op when
2065 /// `payload.system` is already `None` (no `system_prompt` was baked).
2066 async fn apply_system_ref_threshold(
2067 &self,
2068 payload: &mut crate::types::WorkerPayload,
2069 ) -> Result<(), EngineError> {
2070 let Some(rendered) = payload.system.take() else {
2071 return Ok(());
2072 };
2073 let cfg = self.cfg().system_ref.clone();
2074 if rendered.len() <= cfg.threshold_bytes {
2075 payload.system = Some(rendered);
2076 return Ok(());
2077 }
2078 use sha2::Digest;
2079 let size_bytes = rendered.len() as u64;
2080 let sha256 = hex::encode(sha2::Sha256::digest(rendered.as_bytes()));
2081 let task_id = &payload.task_id;
2082 let attempt = payload.attempt;
2083 let system_ref = match cfg.mode {
2084 crate::types::SystemRefMode::Http => crate::types::SystemRef {
2085 // The engine has no knowledge of scheme/host here — see
2086 // `SystemRefMode::Http`'s doc for who fills that in.
2087 uri: format!("/v1/worker/prompt/system?task_id={task_id}&attempt={attempt}"),
2088 sha256,
2089 size_bytes,
2090 mode: crate::types::SystemRefMode::Http,
2091 },
2092 crate::types::SystemRefMode::File => {
2093 tokio::fs::create_dir_all(&cfg.store_dir).await?;
2094 let path = cfg.store_dir.join(format!("{task_id}-{attempt}.md"));
2095 tokio::fs::write(&path, rendered.as_bytes()).await?;
2096 crate::types::SystemRef {
2097 uri: format!("file://{}", path.display()),
2098 sha256,
2099 size_bytes,
2100 mode: crate::types::SystemRefMode::File,
2101 }
2102 }
2103 };
2104 payload.system = None;
2105 payload.system_ref = Some(system_ref);
2106 Ok(())
2107 }
2108
2109 /// GH #83: unconditionally materialize the baked system prompt for
2110 /// `(task_id, attempt)` to a file and return its path — the value
2111 /// source of the `{system_file}` placeholder in a `SubprocessDef`
2112 /// template. Unlike [`Self::apply_system_ref_threshold`] (whose
2113 /// `SystemRefMode::File` write only fires over
2114 /// `SystemRefConfig.threshold_bytes`, a behavior this helper does NOT
2115 /// touch), a template that names `{system_file}` needs a real path
2116 /// regardless of size, so the write here is unconditional. Reuses the
2117 /// same store dir and `{task_id}-{attempt}.md` naming as the File
2118 /// mode, so both paths converge on one on-disk identity per attempt.
2119 ///
2120 /// `Ok(None)` = no system prompt was baked for this attempt (the
2121 /// caller decides whether that is fail-loud — the Subprocess spawn
2122 /// path treats a `{system_file}` reference without a baked system as
2123 /// a `SpawnError`).
2124 pub async fn materialize_system_file(
2125 &self,
2126 task_id: &StepId,
2127 attempt: u32,
2128 ) -> Result<Option<std::path::PathBuf>, EngineError> {
2129 let key = (task_id.clone(), attempt);
2130 let rendered = self
2131 .with_state("materialize_system_file", move |s| {
2132 s.systems.get(&key).cloned().unwrap_or(None)
2133 })
2134 .await?;
2135 let Some(rendered) = rendered else {
2136 return Ok(None);
2137 };
2138 let cfg = self.cfg().system_ref.clone();
2139 tokio::fs::create_dir_all(&cfg.store_dir).await?;
2140 let path = cfg.store_dir.join(format!("{task_id}-{attempt}.md"));
2141 tokio::fs::write(&path, rendered.as_bytes()).await?;
2142 Ok(Some(path))
2143 }
2144
2145 /// Returns the effective [`mlua_swarm_schema::ContextPolicy`]
2146 /// `AgentContextMiddleware` resolved and snapshotted for `(task_id,
2147 /// attempt)` at spawn time (the same policy already applied to that
2148 /// key's `EngineState.agent_ctx` entry's `.view`, GH #23 fold).
2149 /// Pass-all (`ContextPolicy::default()`) when no entry exists — either
2150 /// a pre-ST5 spawn, or a spawner stack that never layered
2151 /// `AgentContextMiddleware` (fail-open, mirroring [`Self::output_tail`]'s
2152 /// "no entry = empty default" convention).
2153 ///
2154 /// `crates/mlua-swarm-server/src/worker.rs`'s `GET /v1/worker/prompt`
2155 /// handler reads this back to filter `WorkerPayload.context.steps` via
2156 /// `ContextPolicy::allows_step`, without re-deriving the policy from
2157 /// the Blueprint at fetch time (`projection-adapter` ST5).
2158 pub async fn context_policy_for(
2159 &self,
2160 task_id: &StepId,
2161 attempt: u32,
2162 ) -> mlua_swarm_schema::ContextPolicy {
2163 let key = (task_id.clone(), attempt);
2164 self.with_state("context_policy_for", move |s| {
2165 s.agent_ctx
2166 .get(&key)
2167 .map(|e| e.policy.clone())
2168 .unwrap_or_default()
2169 })
2170 .await
2171 .unwrap_or_default()
2172 }
2173
2174 /// GH #23: returns the Blueprint-wide
2175 /// [`crate::core::step_naming::StepNaming`] table snapshotted for
2176 /// `task_id` (the same `Arc` `crate::blueprint::EngineDispatcher::dispatch`
2177 /// stashed into `EngineState.step_namings` at dispatch time —
2178 /// `Self::start_task`'s `StepId`, not the `TaskId` work item). `None`
2179 /// when no entry exists — either the dispatcher was never given a
2180 /// `StepNaming` (`EngineDispatcher::with_step_naming` not called) or
2181 /// the lock could not be acquired; callers are expected to fall back
2182 /// to the pre-GH-#23 runtime union rule in that case (subtask-2/3
2183 /// consumers).
2184 pub async fn step_naming_for(
2185 &self,
2186 task_id: &StepId,
2187 ) -> Option<Arc<crate::core::step_naming::StepNaming>> {
2188 let key = task_id.clone();
2189 self.with_state("step_naming_for", move |s| {
2190 s.step_namings.get(&key).cloned()
2191 })
2192 .await
2193 .ok()
2194 .flatten()
2195 }
2196
2197 /// GH #27 (follow-up to #23): returns the Blueprint-wide
2198 /// [`crate::core::projection_placement::ProjectionPlacement`] resolver
2199 /// snapshotted for `task_id` (the same `Arc`
2200 /// `crate::blueprint::EngineDispatcher::dispatch` stashed into
2201 /// `EngineState.projection_placements` at dispatch time — mirroring
2202 /// [`Self::step_naming_for`]'s contract exactly). `None` when no entry
2203 /// exists — either the dispatcher was never given a
2204 /// `ProjectionPlacement` (`EngineDispatcher::with_projection_placement`
2205 /// not called) or the lock could not be acquired; callers are expected
2206 /// to fall back to `ProjectionPlacement::default()` (byte-compat with
2207 /// the pre-#27 hardcoded layout) in that case.
2208 pub async fn projection_placement_for(
2209 &self,
2210 task_id: &StepId,
2211 ) -> Option<Arc<crate::core::projection_placement::ProjectionPlacement>> {
2212 let key = task_id.clone();
2213 self.with_state("projection_placement_for", move |s| {
2214 s.projection_placements.get(&key).cloned()
2215 })
2216 .await
2217 .ok()
2218 .flatten()
2219 }
2220
2221 /// Returns the [`crate::core::agent_context::AgentContextView`]
2222 /// snapshotted for `(task_id, attempt)`, if `AgentContextMiddleware`
2223 /// stashed one — the same lookup [`Self::fetch_worker_payload`] /
2224 /// [`Self::fetch_worker_payload_trusted`] perform inline, exposed
2225 /// standalone for callers that only need the view (not a full
2226 /// `WorkerPayload`) — e.g. the HTTP debug-plane `GET
2227 /// /v1/tasks/:id/runs/:run/steps*` handlers resolving a
2228 /// materialized-file root for a step *other than* the one currently
2229 /// fetching its own prompt (`projection-adapter` ST5).
2230 pub async fn agent_context_for(
2231 &self,
2232 task_id: &StepId,
2233 attempt: u32,
2234 ) -> Option<crate::core::agent_context::AgentContextView> {
2235 let key = (task_id.clone(), attempt);
2236 self.with_state("agent_context_for", move |s| {
2237 s.agent_ctx.get(&key).map(|e| e.view.clone())
2238 })
2239 .await
2240 .ok()
2241 .flatten()
2242 }
2243
2244 /// Read the current attempt number for a task (server-side lookup, no
2245 /// token verification). Used on `HTTP /v1/worker/result` when the
2246 /// worker omits `attempt` and the server has to fill it in.
2247 pub async fn task_attempt(&self, task_id: &StepId) -> Result<u32, EngineError> {
2248 let task_id = task_id.clone();
2249 self.with_state("task_attempt", move |s| {
2250 s.tasks
2251 .get(&task_id)
2252 .map(|t| t.attempt)
2253 .ok_or_else(|| EngineError::TaskNotFound(task_id.to_string()))
2254 })
2255 .await?
2256 }
2257
2258 /// Server-side admin API that lets `OperatorSpawner::spawn` bake the
2259 /// rendered `system_prompt` into engine state. There is no verb gate
2260 /// — the only expected caller is inside the spawner. SubAgents fetch
2261 /// this alongside the prompt on the `/v1/worker/prompt` path.
2262 pub async fn bake_worker_system_prompt(
2263 &self,
2264 task_id: &StepId,
2265 attempt: u32,
2266 system: Option<String>,
2267 ) -> Result<(), EngineError> {
2268 let task_id = task_id.clone();
2269 self.with_state("bake_worker_system_prompt", move |s| {
2270 // GH #31: record this agent's most-recently-baked render size
2271 // before `system` is moved into `s.systems.insert` below. Same
2272 // `s.tasks.get(&task_id)` → `.spec.agent` lookup pattern
2273 // `fetch_worker_payload` uses (see its doc for why this keying
2274 // is load-bearing for a later `bp_doctor` route).
2275 if let Some(rendered) = system.as_ref() {
2276 if let Some(agent) = s.tasks.get(&task_id).map(|t| t.spec.agent.clone()) {
2277 s.agent_render_sizes.insert(agent, rendered.len());
2278 }
2279 }
2280 s.systems.insert((task_id, attempt), system);
2281 })
2282 .await?;
2283 Ok(())
2284 }
2285
2286 /// GH #31: the most-recently-baked `system_prompt` render size (in
2287 /// bytes) observed for `agent_name`, if `bake_worker_system_prompt` has
2288 /// ever recorded one — last-write-wins across every `(task_id,
2289 /// attempt)` dispatch of that agent. `None` when no `system_prompt`
2290 /// has ever been baked for this agent name. Read by the `bp_doctor`
2291 /// route this subtask's follow-up adds.
2292 pub async fn agent_last_rendered_size(&self, agent_name: &str) -> Option<usize> {
2293 let agent_name = agent_name.to_string();
2294 self.with_state("agent_last_rendered_size", move |s| {
2295 s.agent_render_sizes.get(&agent_name).copied()
2296 })
2297 .await
2298 .ok()
2299 .flatten()
2300 }
2301
2302 /// GH #31: plain read-through of the baked `system` string for
2303 /// `(task_id, attempt)` from `EngineState.systems`, with no threshold
2304 /// branching. Backs `GET /v1/worker/prompt/system` (the `Http`-mode
2305 /// fetch target `system_ref.uri` points at) — that route needs the
2306 /// exact raw bytes to serve as the response body for the client's
2307 /// sha256 verification, not a `WorkerPayload`-wrapped value.
2308 ///
2309 /// Distinct from `apply_system_ref_threshold` (private, mutates an
2310 /// already-built `WorkerPayload` in place after full construction):
2311 /// this accessor has no threshold logic and is `pub` so
2312 /// `mlua-swarm-server`'s `worker` module can call it directly.
2313 ///
2314 /// Returns `Ok(None)` if no baked system exists for that `(task_id,
2315 /// attempt)` (either the task/attempt has no entry in `s.systems`, or
2316 /// the entry is present but stores `None`) — the caller maps this to
2317 /// a 404.
2318 pub async fn raw_system_prompt(
2319 &self,
2320 task_id: &StepId,
2321 attempt: u32,
2322 ) -> Result<Option<String>, EngineError> {
2323 let task_id = task_id.clone();
2324 self.with_state("raw_system_prompt", move |s| {
2325 s.systems.get(&(task_id, attempt)).cloned().unwrap_or(None)
2326 })
2327 .await
2328 }
2329
2330 /// Fetch an arbitrary named resource previously stored via
2331 /// `set_resource`. Not task-scoped — any valid token with the
2332 /// `FetchData` verb may read any key.
2333 pub async fn fetch_data(&self, token: &CapToken, key: &str) -> Result<Value, EngineError> {
2334 self.verify_token(token, Verb::FetchData).await?;
2335 let key = key.to_string();
2336 self.with_state("fetch_data", move |s| {
2337 s.resources
2338 .get(&key)
2339 .cloned()
2340 .ok_or(EngineError::ResourceNotFound(key))
2341 })
2342 .await?
2343 }
2344
2345 // ───────────────────────────────────────────────────────────────────────
2346 // Output path.
2347 // ───────────────────────────────────────────────────────────────────────
2348
2349 /// Send one output event from inside a `SpawnerAdapter` or worker.
2350 /// Structuring is assumed to be complete by the time we cross the
2351 /// `SpawnerAdapter` boundary; this API just appends to the
2352 /// `OutputStore`, pushes to the `EventLog`, and (for `Final`) emits
2353 /// the `TaskAttemptCompleted` event.
2354 ///
2355 /// This is Domain-side plumbing: it feeds the engine's verdict flow,
2356 /// not the Data-plane store in the `output_store` module. It also
2357 /// does not wake the dispatch path — that is done through the
2358 /// spawner's completion oneshot when the worker terminates.
2359 ///
2360 /// # Submit-time projection sink (subtask-4 / ST2 rework)
2361 ///
2362 /// A `Final` event additionally fans out to the submit-time projection
2363 /// sink ([`Self::materialize_final_submission`]): (a) when
2364 /// [`Self::set_output_store`] has wired a Data-plane
2365 /// [`crate::store::output::OutputStore`], the event is dual-written
2366 /// there (`producer_agent` = `TaskState.spec.agent`, resolved to its
2367 /// GH #23 canonical projection name — see below), and (b) when this
2368 /// task's spawn ran through `AgentContextMiddleware` (so
2369 /// `EngineState.agent_ctx` has a `.view.work_dir` / `.view.project_root`
2370 /// for it), the value is additionally materialized to the
2371 /// [`crate::core::projection_placement::ProjectionPlacement`]
2372 /// resolver's target (byte-compat default layout
2373 /// `<root>/workspace/tasks/<task_id>/ctx/<canonical_agent>.md`) — see
2374 /// `crate::core::projection`'s module doc.
2375 ///
2376 /// **GH #23 subtask-2 (canonical sink):** both writes above key off the
2377 /// canonical name — `Engine::step_naming_for(task_id)`'s
2378 /// `StepNaming::canonical_of_producer(producer_agent)` when a table was
2379 /// snapshotted for this task (`EngineDispatcher::with_step_naming`),
2380 /// else `producer_agent` unchanged (fail-open, byte-identical to
2381 /// pre-GH-#23 behavior — see [`crate::core::step_naming`]'s module
2382 /// doc).
2383 ///
2384 /// **Invariants** (Subtask 4): (1) this sink is fail-open — an
2385 /// unresolved root, an unconfigured `OutputStore`, or either one
2386 /// erroring, only logs a `tracing::warn!` and never turns this
2387 /// `Ok(())` into an `Err`; (2) the wired `OutputStore` stays the single
2388 /// source of truth for cross-step queries — the materialized file is a
2389 /// projection of it, not a second store; (3) core does not depend on
2390 /// `mlua-swarm-server` — everything this sink touches
2391 /// (`crate::store::output` / `crate::core::projection`) already lives
2392 /// in this crate.
2393 ///
2394 /// # `Artifact` dual-write (GH #34 subtask-3 gap fix)
2395 ///
2396 /// An `Artifact` event ALSO fans out to the Data-plane, via
2397 /// [`Self::materialize_artifact_submission`] — general-form: every
2398 /// `Artifact` submitted through this API dual-writes, no name-prefix
2399 /// gate. Unlike `Final`, the dual-write key is the artifact's own
2400 /// `name` field, verbatim — NOT resolved through the GH #23 canonical
2401 /// `StepNaming` table. An artifact's `name` IS its identity (mirrors
2402 /// [`crate::store::output::OutputStore::get_latest_by_name`]'s doc),
2403 /// so no canonicalization applies. Same fail-open discipline as
2404 /// `Final` (Invariant 1 above), but `Artifact` does NOT drive the
2405 /// file-materialize half (b) — artifact findings (e.g.
2406 /// `AfterRunAuditMiddleware`'s `"audit:<step_ref>"`) are observational
2407 /// sidecar data, not a step's own submission a work_dir/project_root
2408 /// projection needs to track. `Progress` / `Partial` events are
2409 /// unaffected — no behavior change.
2410 pub async fn submit_output(
2411 &self,
2412 token: &crate::types::CapToken,
2413 task_id: &StepId,
2414 attempt: u32,
2415 event: crate::worker::output::OutputEvent,
2416 ) -> Result<(), EngineError> {
2417 self.verify_token_for_task(token, crate::types::Verb::EmitOutput, task_id)
2418 .await?;
2419 // GH #51 — completion-time verdict-contract enforcement, embedded
2420 // choke point 2 of 2 (see `Self::verdict_contract_completion_check`'s
2421 // doc). Guarded to `Final` only — the ONLY `OutputEvent` variant a
2422 // verdict contract's completion can meaningfully address; this
2423 // guard is defensive (this function is empirically called with
2424 // `Final` only today, both from `worker.rs`'s `worker_result` and
2425 // from `operator.rs`'s WS fallback) but costs nothing and protects
2426 // against a future non-`Final` caller. Runs BEFORE the
2427 // `output_tail` write immediately below: on `Err`, this returns
2428 // immediately and the write never happens — a rejected value
2429 // never reaches `output_tail` / the flow ctx.
2430 if let crate::worker::output::OutputEvent::Final { content, ok } = &event {
2431 let comparable_value = content_ref_to_comparable_string(content.clone());
2432 self.verdict_contract_completion_check(task_id, attempt, *ok, &comparable_value)
2433 .await?;
2434 }
2435 let task_id_for_apply = task_id.clone();
2436 let event_clone = event.clone();
2437 self.with_state("submit_output", move |s| {
2438 s.output_store
2439 .entry((task_id_for_apply.clone(), attempt))
2440 .or_default()
2441 .push(event_clone.clone());
2442 s.push_event(crate::core::state::Event::WorkerOutput {
2443 task_id: task_id_for_apply,
2444 attempt,
2445 event: event_clone,
2446 });
2447 })
2448 .await?;
2449 match &event {
2450 crate::worker::output::OutputEvent::Final { content, ok } => {
2451 self.materialize_final_submission(task_id, attempt, content, *ok)
2452 .await?;
2453 }
2454 crate::worker::output::OutputEvent::Artifact { name, content } => {
2455 self.materialize_artifact_submission(task_id, attempt, name, content)
2456 .await?;
2457 }
2458 _ => {}
2459 }
2460 Ok(())
2461 }
2462
2463 /// Submit-time projection sink (subtask-4 / ST2 rework) shared by
2464 /// [`Self::submit_output`] and [`Self::submit_worker_result_trusted`].
2465 /// Best-effort / fail-open throughout (see `submit_output`'s doc
2466 /// Invariants): every failure path only `tracing::warn!`s and returns.
2467 ///
2468 /// Reads `(producer_agent, view)` via one read-only [`Self::with_state`]
2469 /// call — `producer_agent` off `TaskState.spec.agent`, `view` (the
2470 /// full [`crate::core::agent_context::AgentContextView`]) off
2471 /// `EngineState.agent_ctx[(task_id, attempt)]`, the same snapshot
2472 /// `crate::middleware::agent_context::AgentContextMiddleware` writes at
2473 /// spawn time — then does its actual (dual-write / file-write) work
2474 /// *outside* that lock, so a slow disk write or Data-plane store call
2475 /// never holds up unrelated `Engine::with_state` callers. `root` itself
2476 /// is resolved from `view` AFTER the lock via
2477 /// [`crate::core::projection_placement::ProjectionPlacement::resolve_root`]
2478 /// (GH #27, follow-up to #23) — the SAME resolver
2479 /// [`Self::step_naming_for`]'s sibling accessor
2480 /// [`Self::projection_placement_for`] snapshotted at dispatch time, so
2481 /// this sink's root-preference / fallback order is identical to the
2482 /// server read-back and the spawn-time pointer.
2483 async fn materialize_final_submission(
2484 &self,
2485 task_id: &StepId,
2486 attempt: u32,
2487 content: &crate::worker::output::ContentRef,
2488 ok: bool,
2489 ) -> Result<(), EngineError> {
2490 let server_policy = self.cfg().check_policy;
2491 let task_id_for_lookup = task_id.clone();
2492 let lookup = self
2493 .with_state("materialize_final_submission.lookup", move |s| {
2494 let entry = s.tasks.get(&task_id_for_lookup);
2495 let producer_agent = entry.map(|t| t.spec.agent.clone());
2496 let task_policy = entry.and_then(|t| t.spec.check_policy);
2497 let view = s
2498 .agent_ctx
2499 .get(&(task_id_for_lookup.clone(), attempt))
2500 .map(|e| e.view.clone());
2501 (producer_agent, task_policy, view)
2502 })
2503 .await;
2504 // Per-task `TaskSpec.check_policy` (ST1c) wins
2505 // over the server-wide `EngineCfg.check_policy` when set — a
2506 // per-run override forwarded from the launch entry point (see
2507 // `TaskLaunchRequest.check_policy` /
2508 // `TaskLaunchInput.check_policy`). `None` leaves the server
2509 // default in effect (backward compat).
2510 let policy = lookup
2511 .as_ref()
2512 .ok()
2513 .and_then(|(_, tp, _)| *tp)
2514 .unwrap_or(server_policy);
2515 let (producer_agent, view) = match lookup.map(|(pa, _, view)| (pa, view)) {
2516 Ok(pair) => pair,
2517 Err(err) => {
2518 if !matches!(policy, crate::core::config::CheckPolicy::Silent) {
2519 tracing::warn!(
2520 %task_id,
2521 error = %err,
2522 "submit-time projection sink: state lookup failed; skipping (fail-open)"
2523 );
2524 }
2525 apply_check_policy(
2526 policy,
2527 "submit-time projection sink: state lookup",
2528 "state lookup failed; skipping (fail-open)",
2529 )?;
2530 return Ok(());
2531 }
2532 };
2533 let Some(producer_agent) = producer_agent else {
2534 // Defensive only: `task_id` is always a just-looked-up task at
2535 // every real call site. No task, no addressable producer name
2536 // — nothing to project. Not gated by `CheckPolicy` — a missing
2537 // task is an intentional early-exit path, not a fail-open
2538 // condition to surface.
2539 return Ok(());
2540 };
2541 let placement = self
2542 .projection_placement_for(task_id)
2543 .await
2544 .unwrap_or_default();
2545 let root = view.and_then(|v| placement.resolve_root(&v));
2546
2547 // GH #23 subtask-2: resolve `producer_agent` to its canonical
2548 // projection name via the Blueprint-wide `StepNaming` table
2549 // snapshotted at dispatch time (`Engine::step_naming_for`). Both
2550 // write paths below ((a) data-plane, (b) file stem) use the
2551 // *canonical* name — `StepNaming::canonical_of_producer` returns
2552 // `producer_agent` unchanged for undeclared steps (byte-identical
2553 // to pre-GH-#23 behavior), and `None` (no table for this
2554 // `task_id`, e.g. a spawn that never went through
2555 // `EngineDispatcher::with_step_naming`) is a defensive fail-open
2556 // to the raw `producer_agent`, same discipline as the rest of this
2557 // sink.
2558 let canonical_agent = self
2559 .step_naming_for(task_id)
2560 .await
2561 .and_then(|naming| {
2562 naming
2563 .canonical_of_producer(&producer_agent)
2564 .map(str::to_string)
2565 })
2566 .unwrap_or_else(|| producer_agent.clone());
2567
2568 // (a) Data-plane dual-write, when an OutputStore backend is wired.
2569 if let Some(store) = self.output_store_backend() {
2570 if let Err(err) = store
2571 .append(
2572 task_id.as_str(),
2573 attempt,
2574 &canonical_agent,
2575 crate::worker::output::OutputEvent::Final {
2576 content: content.clone(),
2577 ok,
2578 },
2579 Vec::new(),
2580 )
2581 .await
2582 {
2583 if !matches!(policy, crate::core::config::CheckPolicy::Silent) {
2584 tracing::warn!(
2585 %task_id,
2586 agent = %producer_agent,
2587 canonical = %canonical_agent,
2588 error = %err,
2589 "submit-time projection sink: OutputStore dual-write failed (fail-open)"
2590 );
2591 }
2592 apply_check_policy(
2593 policy,
2594 "submit-time projection sink: OutputStore dual-write",
2595 "OutputStore dual-write failed (fail-open)",
2596 )?;
2597 }
2598 }
2599
2600 // (b) File materialize, when a root resolved.
2601 let Some(root) = root else {
2602 if !matches!(policy, crate::core::config::CheckPolicy::Silent) {
2603 tracing::warn!(
2604 %task_id,
2605 agent = %producer_agent,
2606 canonical = %canonical_agent,
2607 "submit-time projection sink: no work_dir/project_root resolved; skipping file materialize (fail-open)"
2608 );
2609 }
2610 apply_check_policy(
2611 policy,
2612 "submit-time projection sink: file materialize",
2613 "no work_dir/project_root resolved; skipping file materialize (fail-open)",
2614 )?;
2615 return Ok(());
2616 };
2617 let value = match content {
2618 crate::worker::output::ContentRef::Inline { value } => value.clone(),
2619 crate::worker::output::ContentRef::FileRef {
2620 path,
2621 mime,
2622 size_hint,
2623 } => serde_json::json!({
2624 "file_ref": path.to_string_lossy(),
2625 "mime": mime,
2626 "size_hint": size_hint,
2627 }),
2628 };
2629 let key = crate::core::projection::ProjectionKey {
2630 task_id: task_id.to_string(),
2631 run_id: None,
2632 step: Some(canonical_agent.clone()),
2633 path: None,
2634 };
2635 let adapter = crate::core::projection::FileProjectionAdapter::with_placement(
2636 root,
2637 (*placement).clone(),
2638 );
2639 if let Err(err) = adapter.materialize_submission(&key, &value, attempt, ok) {
2640 if !matches!(policy, crate::core::config::CheckPolicy::Silent) {
2641 tracing::warn!(
2642 %task_id,
2643 agent = %producer_agent,
2644 canonical = %canonical_agent,
2645 error = %err,
2646 "submit-time projection sink: file materialize failed (fail-open)"
2647 );
2648 }
2649 apply_check_policy(
2650 policy,
2651 "submit-time projection sink: file materialize",
2652 "file materialize failed (fail-open)",
2653 )?;
2654 }
2655 Ok(())
2656 }
2657
2658 /// Submit-time projection sink for `OutputEvent::Artifact` (GH #34
2659 /// subtask-3, later extended to drive the file half too). Two halves, the
2660 /// [`Self::materialize_final_submission`] mirror for staged named parts:
2661 ///
2662 /// - **Data-plane dual-write** — when [`Self::set_output_store`] has
2663 /// wired a [`crate::store::output::OutputStore`], the artifact
2664 /// dual-writes there under its own `name`, verbatim (general form:
2665 /// every `Artifact` staged via [`Self::submit_output`] /
2666 /// [`Self::stage_worker_artifact_trusted`] materializes this way, no
2667 /// name-prefix gate).
2668 /// - **File materialize** — when a `root` resolves off the spawn-time
2669 /// [`crate::core::agent_context::AgentContextView`], the part's
2670 /// content is written raw to `<ctx-dir>/<name>` via
2671 /// [`crate::core::projection::FileProjectionAdapter::materialize_part`].
2672 /// That file is the IN file the *next* Agent step reads: materializing
2673 /// a Step's OUTPUT to disk is the
2674 /// [`crate::core::projection::FileProjectionAdapter`]'s
2675 /// responsibility, and a staged named part is as much an OUTPUT the
2676 /// next step consumes as a `Final` is — so the sink materializes it
2677 /// too, rather than leaving parts Data-plane-only.
2678 ///
2679 /// Unlike the Final sink, no `StepNaming` canonicalization is applied:
2680 /// an artifact's `name` already IS the key both halves address (it
2681 /// names the file directly, extension included — `plan.md` — so
2682 /// `materialize_part` writes it verbatim, not through the `<stem>.md`
2683 /// synthesis the Final sink's canonical-agent path uses).
2684 ///
2685 /// Fail-open throughout, the same `check_policy` cascade as
2686 /// [`Self::materialize_final_submission`]: a per-task lookup error falls
2687 /// back to the server default (and a `None` view ⇒ the file half's
2688 /// unresolved-root path), an unconfigured `OutputStore` skips the
2689 /// dual-write, an unresolved root skips the file half, and a
2690 /// dual-write / file-write / name-guard error only `tracing::warn!`s
2691 /// (`Silent` suppresses even that) before applying [`apply_check_policy`]
2692 /// (`Strict` surfaces an [`EngineError`], `Warn` / `Silent` return
2693 /// `Ok(())`) — a staged part never turns a would-have-succeeded submit
2694 /// into a failure under the default policy.
2695 async fn materialize_artifact_submission(
2696 &self,
2697 task_id: &StepId,
2698 attempt: u32,
2699 name: &str,
2700 content: &crate::worker::output::ContentRef,
2701 ) -> Result<(), EngineError> {
2702 // Per-task `TaskSpec.check_policy` override + the `AgentContextView`
2703 // snapshot, resolved in ONE read-only `with_state` (the same lock
2704 // the policy lookup already needed — no extra `with_state` for the
2705 // view). Silent per-task lookup failure (`with_state` error) falls
2706 // back to the server-wide default and a `None` view (⇒ the file
2707 // half's own unresolved-root fail-open path); this sink never
2708 // surfaces the lookup error itself as a step failure.
2709 let server_policy = self.cfg().check_policy;
2710 let task_id_for_lookup = task_id.clone();
2711 let lookup = self
2712 .with_state("materialize_artifact_submission.lookup", move |s| {
2713 let task_policy = s
2714 .tasks
2715 .get(&task_id_for_lookup)
2716 .and_then(|t| t.spec.check_policy);
2717 let view = s
2718 .agent_ctx
2719 .get(&(task_id_for_lookup.clone(), attempt))
2720 .map(|e| e.view.clone());
2721 (task_policy, view)
2722 })
2723 .await
2724 .ok();
2725 let policy = lookup
2726 .as_ref()
2727 .and_then(|(tp, _)| *tp)
2728 .unwrap_or(server_policy);
2729 let view = lookup.and_then(|(_, view)| view);
2730
2731 // (a) Data-plane dual-write, when an OutputStore backend is wired —
2732 // the artifact's own `name` is its Data-plane key (no
2733 // canonicalization, unlike the Final sink's `StepNaming`
2734 // resolution).
2735 if let Some(store) = self.output_store_backend() {
2736 if let Err(err) = store
2737 .append(
2738 task_id.as_str(),
2739 attempt,
2740 name,
2741 crate::worker::output::OutputEvent::Artifact {
2742 name: name.to_string(),
2743 content: content.clone(),
2744 },
2745 Vec::new(),
2746 )
2747 .await
2748 {
2749 if !matches!(policy, crate::core::config::CheckPolicy::Silent) {
2750 tracing::warn!(
2751 %task_id,
2752 artifact = %name,
2753 error = %err,
2754 "submit-time projection sink: OutputStore dual-write failed for Artifact (fail-open)"
2755 );
2756 }
2757 apply_check_policy(
2758 policy,
2759 "submit-time projection sink: Artifact OutputStore dual-write",
2760 "OutputStore dual-write failed for Artifact (fail-open)",
2761 )?;
2762 }
2763 }
2764
2765 // (b) File materialize, when a root resolved — writes the staged
2766 // part raw to `<ctx-dir>/<name>`, the IN file the next Agent step
2767 // reads (see `FileProjectionAdapter::materialize_part`'s doc for
2768 // why raw / why the name is verbatim). A name-guard violation lands
2769 // on the same fail-open path as any other write error below.
2770 let placement = self
2771 .projection_placement_for(task_id)
2772 .await
2773 .unwrap_or_default();
2774 let Some(root) = view.and_then(|v| placement.resolve_root(&v)) else {
2775 if !matches!(policy, crate::core::config::CheckPolicy::Silent) {
2776 tracing::warn!(
2777 %task_id,
2778 artifact = %name,
2779 "submit-time projection sink: no work_dir/project_root resolved; skipping part file materialize (fail-open)"
2780 );
2781 }
2782 apply_check_policy(
2783 policy,
2784 "submit-time projection sink: part file materialize",
2785 "no work_dir/project_root resolved; skipping part file materialize (fail-open)",
2786 )?;
2787 return Ok(());
2788 };
2789 let value = match content {
2790 crate::worker::output::ContentRef::Inline { value } => value.clone(),
2791 crate::worker::output::ContentRef::FileRef {
2792 path,
2793 mime,
2794 size_hint,
2795 } => serde_json::json!({
2796 "file_ref": path.to_string_lossy(),
2797 "mime": mime,
2798 "size_hint": size_hint,
2799 }),
2800 };
2801 let adapter = crate::core::projection::FileProjectionAdapter::with_placement(
2802 root,
2803 (*placement).clone(),
2804 );
2805 if let Err(err) = adapter.materialize_part(task_id.as_str(), name, &value) {
2806 if !matches!(policy, crate::core::config::CheckPolicy::Silent) {
2807 tracing::warn!(
2808 %task_id,
2809 artifact = %name,
2810 error = %err,
2811 "submit-time projection sink: part file materialize failed (fail-open)"
2812 );
2813 }
2814 apply_check_policy(
2815 policy,
2816 "submit-time projection sink: part file materialize",
2817 "part file materialize failed (fail-open)",
2818 )?;
2819 }
2820 Ok(())
2821 }
2822
2823 /// Snapshot the entire output tail for a given `(task_id, attempt)`.
2824 /// Used by the dispatch path when pulling `Final`, and by observers
2825 /// reading the trace.
2826 pub async fn output_tail(
2827 &self,
2828 task_id: &StepId,
2829 attempt: u32,
2830 ) -> Vec<crate::worker::output::OutputEvent> {
2831 let key = (task_id.clone(), attempt);
2832 self.with_state("output_tail", move |s| {
2833 s.output_store.get(&key).cloned().unwrap_or_default()
2834 })
2835 .await
2836 .unwrap_or_default()
2837 }
2838
2839 /// Record an interim `last_result` for `task_id` without changing its
2840 /// `status`. Distinct from the terminal `Final` output event handled
2841 /// through `submit_output` / `dispatch_attempt_with`.
2842 pub async fn post_result(
2843 &self,
2844 token: &CapToken,
2845 task_id: &StepId,
2846 result: Value,
2847 ) -> Result<(), EngineError> {
2848 self.verify_token_for_task(token, Verb::PostResult, task_id)
2849 .await?;
2850 let task_id = task_id.clone();
2851 let result_clone = result.clone();
2852 self.with_state("post_result", move |s| {
2853 let task = s
2854 .tasks
2855 .get_mut(&task_id)
2856 .ok_or_else(|| EngineError::TaskNotFound(task_id.to_string()))?;
2857 task.last_result = Some(result_clone);
2858 task.updated_at = now_unix();
2859 Ok::<(), EngineError>(())
2860 })
2861 .await??;
2862 Ok(())
2863 }
2864
2865 /// Store a named resource value, retrievable later via `fetch_data`.
2866 /// No token is required — this is a server-side/admin-style setter
2867 /// (mirrors `bake_worker_system_prompt`).
2868 pub async fn set_resource(
2869 &self,
2870 key: impl Into<String>,
2871 value: Value,
2872 ) -> Result<(), EngineError> {
2873 let key = key.into();
2874 self.with_state("set_resource", move |s| {
2875 s.resources.insert(key, value);
2876 })
2877 .await?;
2878 Ok(())
2879 }
2880
2881 // ═══════════════════════════════════════════════════════════════════════
2882 // Senior suspend / resume
2883 // ═══════════════════════════════════════════════════════════════════════
2884
2885 /// Ask a question of the Senior, mark the task `Suspended`, and
2886 /// return a `ResumeKey`. The suspended state persists until another
2887 /// task calls `resume(key, answer)`.
2888 ///
2889 /// Resume-side waiting is `Notify`-based, so a caller (typically
2890 /// MainAI) can detach, reattach from a different process, and still
2891 /// pull the answer out via `await_resume(key, timeout)` — the answer
2892 /// is stored inside `EngineState`.
2893 pub async fn query_senior(
2894 &self,
2895 token: &CapToken,
2896 task_id: &StepId,
2897 question: Value,
2898 ) -> Result<ResumeKey, EngineError> {
2899 self.verify_token(token, Verb::QuerySenior).await?;
2900 let task_id = task_id.clone();
2901 let key = ResumeKey::for_senior(&task_id);
2902 let task_notify = self
2903 .with_state("query_senior.notify_ensure", |s| {
2904 s.ensure_task_notify(&task_id)
2905 })
2906 .await?;
2907
2908 let key_clone = key.clone();
2909 let task_id_inner = task_id.clone();
2910 let question_clone = question.clone();
2911 self.with_state("query_senior.suspend", move |s| {
2912 let task = s
2913 .tasks
2914 .get_mut(&task_id_inner)
2915 .ok_or_else(|| EngineError::TaskNotFound(task_id_inner.to_string()))?;
2916 task.status = TaskStatus::Suspended;
2917 task.suspended_on = Some(key_clone.clone());
2918 task.updated_at = now_unix();
2919 s.pending_resumes
2920 .insert(key_clone.clone(), ResumePending::new());
2921 s.push_event(Event::SeniorQueried {
2922 task_id: task_id_inner.clone(),
2923 question: question_clone.clone(),
2924 });
2925 s.push_event(Event::TaskSuspended {
2926 task_id: task_id_inner.clone(),
2927 key: key_clone.clone(),
2928 });
2929 Ok::<(), EngineError>(())
2930 })
2931 .await??;
2932
2933 // Notify callers waiting for a task status change (Running → Suspended).
2934 task_notify.notify_waiters();
2935
2936 let _ = self
2937 .inner
2938 .event_tx
2939 .send(Event::SeniorQueried { task_id, question });
2940 Ok(key)
2941 }
2942
2943 /// Store the answer for a `ResumeKey` in `EngineState` and wake the
2944 /// waiting caller via `Notify`. Also flips the suspended task's
2945 /// status back to `Running` and fires the per-task notifier.
2946 pub async fn resume(&self, key: ResumeKey, answer: Value) -> Result<(), EngineError> {
2947 let answer_for_state = answer.clone();
2948 let answer_for_event = answer.clone();
2949 let key_clone = key.clone();
2950 let (notify, task_notify, task_id_opt) = self
2951 .with_state("resume.set", move |s| {
2952 let pending = s
2953 .pending_resumes
2954 .get_mut(&key_clone)
2955 .ok_or(EngineError::ResumeKeyNotFound)?;
2956 pending.answer = Some(answer_for_state);
2957 let notify = pending.notify.clone();
2958
2959 let task_id = s
2960 .tasks
2961 .iter()
2962 .find(|(_, t)| t.suspended_on.as_ref() == Some(&key_clone))
2963 .map(|(id, _)| id.clone());
2964
2965 let task_notify = task_id.as_ref().map(|tid| s.ensure_task_notify(tid));
2966
2967 if let Some(tid) = &task_id {
2968 if let Some(task) = s.tasks.get_mut(tid) {
2969 task.suspended_on = None;
2970 task.status = TaskStatus::Running;
2971 task.updated_at = now_unix();
2972 }
2973 s.push_event(Event::TaskResumed {
2974 task_id: tid.clone(),
2975 key: key_clone.clone(),
2976 });
2977 s.push_event(Event::SeniorAnswered {
2978 task_id: tid.clone(),
2979 answer: answer_for_event.clone(),
2980 });
2981 }
2982 Ok::<_, EngineError>((notify, task_notify, task_id))
2983 })
2984 .await??;
2985
2986 // Outside the lock: notify_waiters for both the ResumePending and task-status waits.
2987 notify.notify_waiters();
2988 if let Some(n) = task_notify {
2989 n.notify_waiters();
2990 }
2991
2992 if let Some(tid) = task_id_opt {
2993 let _ = self
2994 .inner
2995 .event_tx
2996 .send(Event::TaskResumed { task_id: tid, key });
2997 }
2998 Ok(())
2999 }
3000
3001 /// Wait for the resume answer. Even if the caller (an Operator)
3002 /// detached and reattached, the answer is available immediately here
3003 /// — if it was already stored, this returns without waiting on the
3004 /// notifier.
3005 ///
3006 /// `timeout = Duration::ZERO` performs an instant check without
3007 /// waiting.
3008 pub async fn await_resume(
3009 &self,
3010 key: ResumeKey,
3011 timeout: Duration,
3012 ) -> Result<Value, EngineError> {
3013 // (1) Under the lock: clone the notify handle and check for an existing answer.
3014 let key_clone = key.clone();
3015 let (notify, existing) = self
3016 .with_state("await_resume.snapshot", move |s| {
3017 let pending = s
3018 .pending_resumes
3019 .get(&key_clone)
3020 .ok_or(EngineError::ResumeKeyNotFound)?;
3021 Ok::<_, EngineError>((pending.notify.clone(), pending.answer.clone()))
3022 })
3023 .await??;
3024
3025 // (2) If an answer has already been stored, return immediately (detach / reattach pattern).
3026 if let Some(v) = existing {
3027 return Ok(v);
3028 }
3029
3030 // (3) Outside the lock: wait on the notify with a timeout.
3031 if timeout.is_zero() {
3032 return Err(EngineError::PollTimeout);
3033 }
3034 let waited = tokio::time::timeout(timeout, notify.notified()).await;
3035 if waited.is_err() {
3036 return Err(EngineError::PollTimeout);
3037 }
3038
3039 // (4) Under the lock: re-read the answer (should be present now that we were notified).
3040 let key_clone = key.clone();
3041 self.with_state("await_resume.read", move |s| {
3042 let pending = s
3043 .pending_resumes
3044 .get(&key_clone)
3045 .ok_or(EngineError::ResumeKeyNotFound)?;
3046 pending
3047 .answer
3048 .clone()
3049 .ok_or_else(|| EngineError::Internal("notified but answer missing".into()))
3050 })
3051 .await?
3052 }
3053
3054 // ═══════════════════════════════════════════════════════════════════════
3055 // poll_task — the "wait" path that waits for task-status changes (works for long-poll and regular wait).
3056 // ═══════════════════════════════════════════════════════════════════════
3057
3058 /// Wait until the task's status **transitions to terminal or
3059 /// `Suspended`**, then return the latest `TaskState`. Returns
3060 /// immediately if the task is already in a terminal state.
3061 /// Exceeding the timeout returns `EngineError::PollTimeout`.
3062 ///
3063 /// A `hold` of `Duration::from_secs(0)` returns a snapshot immediately
3064 /// (no wait). Larger holds — tens of minutes up to days — are fine;
3065 /// the wait state is kept in memory inside the engine and does not
3066 /// degrade.
3067 pub async fn poll_task(
3068 &self,
3069 token: &CapToken,
3070 task_id: &StepId,
3071 hold: Duration,
3072 ) -> Result<TaskState, EngineError> {
3073 self.verify_token_for_task(token, Verb::PollTask, task_id)
3074 .await?;
3075 let task_id_inner = task_id.clone();
3076
3077 // (1) Under the lock: take a snapshot and clone task_notify.
3078 let (state, notify) = self
3079 .with_state("poll_task.snapshot", move |s| {
3080 let task = s
3081 .tasks
3082 .get(&task_id_inner)
3083 .cloned()
3084 .ok_or_else(|| EngineError::TaskNotFound(task_id_inner.to_string()))?;
3085 let notify = s.ensure_task_notify(&task_id_inner);
3086 Ok::<_, EngineError>((task, notify))
3087 })
3088 .await??;
3089
3090 // (2) Immediate-return condition: already terminal / Suspended (nothing left to wait on).
3091 if matches!(
3092 state.status,
3093 TaskStatus::Pass | TaskStatus::Blocked | TaskStatus::Cancelled | TaskStatus::Suspended
3094 ) {
3095 return Ok(state);
3096 }
3097 if hold.is_zero() {
3098 return Ok(state);
3099 }
3100
3101 // (3) Outside the lock: wait on Notify with a timeout.
3102 let waited = tokio::time::timeout(hold, notify.notified()).await;
3103 if waited.is_err() {
3104 return Err(EngineError::PollTimeout);
3105 }
3106
3107 // (4) Under the lock: take a fresh snapshot.
3108 let task_id_inner = task_id.clone();
3109 self.with_state("poll_task.reread", move |s| {
3110 s.tasks
3111 .get(&task_id_inner)
3112 .cloned()
3113 .ok_or_else(|| EngineError::TaskNotFound(task_id_inner.to_string()))
3114 })
3115 .await?
3116 }
3117
3118 // ═══════════════════════════════════════════════════════════════════════
3119 // Background: heartbeat miss → detach loop
3120 // ═══════════════════════════════════════════════════════════════════════
3121
3122 /// Background loop that scans sessions every `heartbeat_interval` and
3123 /// flips `attached = false` on any session whose `last_seen` exceeds
3124 /// `heartbeat_miss_threshold * interval`.
3125 ///
3126 /// The tasks themselves are kept (assuming
3127 /// `keepalive_on_idle = true`), so another client can reattach with
3128 /// the same token and resume immediately. Dropping the returned
3129 /// `JoinHandle` does not stop the loop — the handle exists so callers
3130 /// who want to abort can hold onto it.
3131 pub fn start_detach_loop(&self) -> tokio::task::JoinHandle<()> {
3132 let engine = self.clone();
3133 let cfg = self.inner.cfg.long_hold.clone();
3134 let interval = cfg.heartbeat_interval;
3135 let miss_secs = cfg.heartbeat_interval.as_secs() * cfg.heartbeat_miss_threshold as u64;
3136
3137 tokio::spawn(async move {
3138 let mut ticker = tokio::time::interval(interval);
3139 ticker.tick().await; // first tick is immediate
3140 loop {
3141 ticker.tick().await;
3142 let now = now_unix();
3143 let detached = engine
3144 .with_state("detach_loop.scan", |s| {
3145 let mut detached = Vec::new();
3146 for (sid, sess) in s.sessions.iter_mut() {
3147 if !sess.attached {
3148 continue;
3149 }
3150 if now.saturating_sub(sess.last_seen) >= miss_secs {
3151 sess.attached = false;
3152 detached.push(sid.clone());
3153 }
3154 }
3155 for sid in &detached {
3156 s.push_event(Event::SessionDetached {
3157 session_id: sid.clone(),
3158 });
3159 }
3160 detached
3161 })
3162 .await
3163 .unwrap_or_default();
3164 for sid in detached {
3165 let _ = engine
3166 .inner
3167 .event_tx
3168 .send(Event::SessionDetached { session_id: sid });
3169 }
3170 }
3171 })
3172 }
3173
3174 /// Helper: wake a task whose status has changed. Called from the
3175 /// method body outside the lock.
3176 async fn wake_task(&self, task_id: &StepId) -> Result<(), EngineError> {
3177 let task_id = task_id.clone();
3178 let notify_opt = self
3179 .with_state("wake_task.get_notify", move |s| {
3180 s.task_notifies.get(&task_id).cloned()
3181 })
3182 .await?;
3183 if let Some(n) = notify_opt {
3184 n.notify_waiters();
3185 }
3186 Ok(())
3187 }
3188}
3189
3190/// Decide what a submit-time projection sink should do at a fail-open
3191/// branch given the configured [`crate::core::config::CheckPolicy`].
3192///
3193/// Returns `Ok(())` under [`CheckPolicy::Silent`] and
3194/// [`CheckPolicy::Warn`] — the caller continues with fail-open. Returns
3195/// [`EngineError::CheckPolicyStrict`] under [`CheckPolicy::Strict`],
3196/// carrying the caller-supplied `context` (call-site identifier) and
3197/// `message` (the pre-existing warn-log message literal, preserved
3198/// verbatim for log parse compatibility).
3199///
3200/// This helper deliberately does **not** call `tracing::warn!` itself —
3201/// the caller is responsible for firing the existing warn! (with its
3202/// full structured-field payload — `%task_id`, `agent`, `canonical`,
3203/// `error`, etc.) under `Warn` mode, and for skipping the warn! under
3204/// `Silent` mode. Keeping the warn! at the call site preserves the
3205/// exact structured-field shape every existing log-parse consumer sees;
3206/// forwarding it through the helper would either drop those fields or
3207/// require a macro (deferred, see subtask-1b).
3208///
3209/// Design intent: the fail-open discipline of every submit-time
3210/// projection sink is byte-identical to the pre-`CheckPolicy` behaviour
3211/// under the default [`CheckPolicy::Warn`]. `Silent` is a per-run opt-in
3212/// to suppress noise (e.g., a caller that has already verified upstream
3213/// invariants); `Strict` is a per-run opt-in to fail loudly (e.g., a
3214/// caller that requires all parts to materialize). See
3215/// [`crate::core::config::CheckPolicy`] for the "state dirty on fail"
3216/// semantics of `Strict`.
3217pub(crate) fn apply_check_policy(
3218 policy: crate::core::config::CheckPolicy,
3219 context: &str,
3220 message: &str,
3221) -> Result<(), EngineError> {
3222 match policy {
3223 crate::core::config::CheckPolicy::Silent | crate::core::config::CheckPolicy::Warn => Ok(()),
3224 crate::core::config::CheckPolicy::Strict => Err(EngineError::CheckPolicyStrict {
3225 context: context.to_string(),
3226 message: message.to_string(),
3227 }),
3228 }
3229}
3230
3231#[cfg(test)]
3232mod check_policy_helper_tests {
3233 use super::apply_check_policy;
3234 use crate::core::config::CheckPolicy;
3235 use crate::core::errors::EngineError;
3236
3237 /// `Silent` returns `Ok(())` without producing an error. Log
3238 /// suppression (the "no `tracing::warn!`" half of the semantics) is
3239 /// enforced at the call site, not inside the helper — see the
3240 /// helper's doc comment for why.
3241 #[test]
3242 fn silent_returns_ok() {
3243 let result = apply_check_policy(CheckPolicy::Silent, "call/site", "sink message");
3244 assert!(matches!(result, Ok(())));
3245 }
3246
3247 /// `Warn` (the default) returns `Ok(())` — the caller continues
3248 /// with fail-open, having already fired its own `tracing::warn!`
3249 /// with the full structured-field payload.
3250 #[test]
3251 fn warn_returns_ok() {
3252 let result = apply_check_policy(CheckPolicy::Warn, "call/site", "sink message");
3253 assert!(matches!(result, Ok(())));
3254 }
3255
3256 /// `Strict` returns
3257 /// [`EngineError::CheckPolicyStrict`] with `context` and `message`
3258 /// copied verbatim from the caller — the completion route surfaces
3259 /// this as a step / launch error so a caller that has opted in can
3260 /// fail fast instead of proceeding with a partially-realized
3261 /// submission.
3262 #[test]
3263 fn strict_returns_error_with_context_and_message() {
3264 let result = apply_check_policy(
3265 CheckPolicy::Strict,
3266 "submit-time projection sink: file materialize",
3267 "no work_dir/project_root resolved; skipping file materialize (fail-open)",
3268 );
3269 match result {
3270 Err(EngineError::CheckPolicyStrict { context, message }) => {
3271 assert_eq!(context, "submit-time projection sink: file materialize");
3272 assert_eq!(
3273 message,
3274 "no work_dir/project_root resolved; skipping file materialize (fail-open)"
3275 );
3276 }
3277 other => panic!("expected CheckPolicyStrict, got {:?}", other),
3278 }
3279 }
3280}
3281
3282// ─── UT: issue #14 — token store keyed by fingerprint, not nonce ────────────
3283#[cfg(test)]
3284mod token_fingerprint_store_tests {
3285 use super::*;
3286
3287 /// A token that was never attached fails verify with a `TokenNotFound`
3288 /// that carries the fingerprint — never the nonce. The error string can
3289 /// surface in HTTP error bodies, so this is the secret-hygiene contract.
3290 #[tokio::test]
3291 async fn verify_unknown_token_reports_fingerprint_not_nonce() {
3292 let engine = Engine::new(EngineCfg::default());
3293 // Signed by the engine's own signer (sig passes) but never inserted
3294 // into the store — verify must fail at step (4), the store lookup.
3295 let token = engine.signer().session(
3296 "ghost",
3297 Role::Operator,
3298 vec!["*".into()],
3299 Duration::from_secs(60),
3300 );
3301 let err = engine
3302 .verify_token(&token, Verb::ReadTaskState)
3303 .await
3304 .expect_err("token is not in the store");
3305 let msg = err.to_string();
3306 assert!(
3307 msg.contains(&token.fingerprint()),
3308 "error must carry the fingerprint: {msg}"
3309 );
3310 assert!(
3311 !msg.contains(&token.nonce),
3312 "error must not leak the nonce: {msg}"
3313 );
3314 }
3315
3316 /// attach → verify → heartbeat → detach all resolve the session /
3317 /// token record through fingerprint keys (mint/verify lifecycle
3318 /// regression guard for the issue #14 key migration).
3319 #[tokio::test]
3320 async fn attach_verify_heartbeat_detach_cycle_with_fp_keying() {
3321 let engine = Engine::new(EngineCfg::default());
3322 let token = engine
3323 .attach("op-1", Role::Operator, Duration::from_secs(60))
3324 .await
3325 .expect("attach");
3326 engine
3327 .verify_token(&token, Verb::ReadTaskState)
3328 .await
3329 .expect("verify consumes via fp key");
3330 engine
3331 .heartbeat(&token)
3332 .await
3333 .expect("heartbeat finds the session by fp");
3334 engine
3335 .detach(&token)
3336 .await
3337 .expect("detach finds the session by fp");
3338 }
3339}
3340
3341// ─── UT: `OperatorKind` "Runtime Global" tier — `Option` semantics ─────────
3342//
3343// Regression coverage for the "explicit Automate is indistinguishable from
3344// unspecified" defect: `OperatorSession.operator_kind` (and the
3345// `attach_with_ids` `kind` parameter it stores) is `Option<OperatorKind>`,
3346// so `Some(Automate)` is an explicit Runtime Global request that must
3347// outrank `bp_global`, while `None` must let `bp_global` decide. Exercises
3348// the real `resolve_operator_info` cascade path (not just
3349// `collapse_operator_kind` in isolation), attaching via `attach_with_ids`
3350// exactly as `TaskLaunchService::launch` does.
3351#[cfg(test)]
3352mod resolve_operator_info_runtime_global_tests {
3353 use super::*;
3354
3355 async fn attach_and_resolve(
3356 runtime_global: Option<OperatorKind>,
3357 bp_global: Option<OperatorKind>,
3358 ) -> OperatorInfo {
3359 let engine = Engine::new(EngineCfg::default());
3360 let token = engine
3361 .attach_with_ids(
3362 "ut-op",
3363 Role::Operator,
3364 Duration::from_secs(30),
3365 runtime_global,
3366 None,
3367 None,
3368 None,
3369 HashMap::new(),
3370 HashMap::new(),
3371 bp_global,
3372 )
3373 .await
3374 .expect("attach_with_ids ok");
3375 let session = engine
3376 .with_state("test.find_session", |s| {
3377 s.sessions
3378 .values()
3379 .find(|sess| sess.token_fp == token.fingerprint())
3380 .cloned()
3381 })
3382 .await
3383 .expect("with_state ok")
3384 .expect("session present after attach_with_ids");
3385 engine.resolve_operator_info(&session, "agent-x").await
3386 }
3387
3388 #[tokio::test]
3389 async fn explicit_some_automate_outranks_bp_global_main_ai() {
3390 // Runtime Global explicitly requests Automate; bp_global is MainAi.
3391 // The explicit `Some(Automate)` must win — this is exactly the case
3392 // the old `== OperatorKind::default()` convention got wrong (it
3393 // could not tell "explicitly Automate" from "unspecified" and would
3394 // have let `bp_global` (MainAi) take over instead).
3395 let info =
3396 attach_and_resolve(Some(OperatorKind::Automate), Some(OperatorKind::MainAi)).await;
3397 assert_eq!(
3398 info.kind,
3399 OperatorKind::Automate,
3400 "explicit Some(Automate) runtime_global must outrank bp_global MainAi"
3401 );
3402 }
3403
3404 #[tokio::test]
3405 async fn none_lets_bp_global_main_ai_win() {
3406 // Runtime Global left unspecified (`None`); bp_global is MainAi.
3407 // With nothing more specific set, `bp_global` must decide.
3408 let info = attach_and_resolve(None, Some(OperatorKind::MainAi)).await;
3409 assert_eq!(
3410 info.kind,
3411 OperatorKind::MainAi,
3412 "None runtime_global must let bp_global MainAi win"
3413 );
3414 }
3415}
3416
3417/// issue #13 run_id propagation: `dispatch_attempt_with`'s `run_id` param
3418/// must land in `Ctx.meta.runtime["run_id"]` (the same slot pattern as the
3419/// pre-existing `worker_handle`), or be omitted entirely when `None`. Same
3420/// `CtxProbe` shape as `middleware::worker_binding`'s test module — an
3421/// inner `SpawnerAdapter` that snapshots the `Ctx` it was called with and
3422/// fails the spawn (only the ctx snapshot matters here).
3423#[cfg(test)]
3424mod dispatch_attempt_with_run_id_tests {
3425 use super::*;
3426 use crate::worker::adapter::{SpawnError, SpawnerAdapter};
3427 use crate::worker::Worker;
3428 use std::sync::Mutex as StdMutex;
3429
3430 struct CtxProbe {
3431 seen: Arc<StdMutex<Option<Ctx>>>,
3432 }
3433
3434 #[async_trait::async_trait]
3435 impl SpawnerAdapter for CtxProbe {
3436 async fn spawn(
3437 &self,
3438 _engine: &Engine,
3439 ctx: &Ctx,
3440 _task_id: StepId,
3441 _attempt: u32,
3442 _token: CapToken,
3443 ) -> Result<Box<dyn Worker>, SpawnError> {
3444 *self.seen.lock().unwrap() = Some(ctx.clone());
3445 Err(SpawnError::Internal("probe stop".into()))
3446 }
3447 }
3448
3449 async fn dispatch_with_probe(run_id: Option<&RunId>) -> Ctx {
3450 let engine = Engine::new(EngineCfg::default());
3451 let token = engine
3452 .attach("ut-op", Role::Operator, Duration::from_secs(30))
3453 .await
3454 .expect("attach");
3455 let tid = engine
3456 .start_task(
3457 &token,
3458 TaskSpec {
3459 agent: "probe".into(),
3460 initial_directive: "hi".into(),
3461 step_ctx: None,
3462 check_policy: None,
3463 },
3464 )
3465 .await
3466 .expect("start_task");
3467 let seen: Arc<StdMutex<Option<Ctx>>> = Arc::new(StdMutex::new(None));
3468 let spawner: Arc<dyn SpawnerAdapter> = Arc::new(CtxProbe { seen: seen.clone() });
3469 // The probe always errors the spawn (`SpawnError::Internal`); we
3470 // only care about the `Ctx` snapshot it captured, so the dispatch
3471 // outcome itself (`Err`) is discarded.
3472 let _ = engine
3473 .dispatch_attempt_with(&token, &tid, &spawner, run_id)
3474 .await;
3475 let captured = seen.lock().unwrap().clone();
3476 captured.expect("inner ctx captured")
3477 }
3478
3479 #[tokio::test]
3480 async fn run_id_lands_in_ctx_meta_runtime_when_some() {
3481 let run_id = RunId::new();
3482 let observed = dispatch_with_probe(Some(&run_id)).await;
3483 assert_eq!(
3484 observed.meta.runtime.get("run_id").and_then(|v| v.as_str()),
3485 Some(run_id.as_str()),
3486 "ctx.meta.runtime[\"run_id\"] must carry the run_id passed to dispatch_attempt_with"
3487 );
3488 }
3489
3490 #[tokio::test]
3491 async fn run_id_key_absent_when_none() {
3492 let observed = dispatch_with_probe(None).await;
3493 assert!(
3494 !observed.meta.runtime.contains_key("run_id"),
3495 "no run_id key must be injected when dispatch_attempt_with is called with None"
3496 );
3497 }
3498}
3499
3500/// GH #21 Phase 2: `TaskSpec.step_ctx` must land in
3501/// `Ctx.meta.runtime[STEP_CTX_KEY]` — re-read from the spec on EVERY
3502/// attempt (the prep closure re-reads `task.spec.step_ctx` every call, not
3503/// caching it once at `start_task`), so a retry (attempt 2) carries it
3504/// too. Same `CtxProbe` shape as `dispatch_attempt_with_run_id_tests`.
3505#[cfg(test)]
3506mod dispatch_attempt_with_step_ctx_tests {
3507 use super::*;
3508 use crate::worker::adapter::{SpawnError, SpawnerAdapter};
3509 use crate::worker::Worker;
3510 use std::sync::Mutex as StdMutex;
3511
3512 struct CtxProbe {
3513 seen: Arc<StdMutex<Option<Ctx>>>,
3514 }
3515
3516 #[async_trait::async_trait]
3517 impl SpawnerAdapter for CtxProbe {
3518 async fn spawn(
3519 &self,
3520 _engine: &Engine,
3521 ctx: &Ctx,
3522 _task_id: StepId,
3523 _attempt: u32,
3524 _token: CapToken,
3525 ) -> Result<Box<dyn Worker>, SpawnError> {
3526 *self.seen.lock().unwrap() = Some(ctx.clone());
3527 Err(SpawnError::Internal("probe stop".into()))
3528 }
3529 }
3530
3531 #[tokio::test]
3532 async fn step_ctx_lands_in_ctx_meta_runtime_on_attempt_1_and_2() {
3533 let engine = Engine::new(EngineCfg::default());
3534 let token = engine
3535 .attach("ut-op", Role::Operator, Duration::from_secs(30))
3536 .await
3537 .expect("attach");
3538 let tid = engine
3539 .start_task(
3540 &token,
3541 TaskSpec {
3542 agent: "probe".into(),
3543 initial_directive: "hi".into(),
3544 step_ctx: Some(serde_json::json!({ "work_dir": "/step" })),
3545 check_policy: None,
3546 },
3547 )
3548 .await
3549 .expect("start_task");
3550 let seen: Arc<StdMutex<Option<Ctx>>> = Arc::new(StdMutex::new(None));
3551 let spawner: Arc<dyn SpawnerAdapter> = Arc::new(CtxProbe { seen: seen.clone() });
3552
3553 // The probe always errors the spawn; only the ctx snapshot matters.
3554 let _ = engine
3555 .dispatch_attempt_with(&token, &tid, &spawner, None)
3556 .await;
3557 let first = seen
3558 .lock()
3559 .unwrap()
3560 .clone()
3561 .expect("attempt 1 ctx captured");
3562 assert_eq!(
3563 first.meta.runtime.get(STEP_CTX_KEY),
3564 Some(&serde_json::json!({ "work_dir": "/step" })),
3565 "attempt 1 must carry TaskSpec.step_ctx in ctx.meta.runtime[STEP_CTX_KEY]"
3566 );
3567
3568 let _ = engine
3569 .dispatch_attempt_with(&token, &tid, &spawner, None)
3570 .await;
3571 let second = seen
3572 .lock()
3573 .unwrap()
3574 .clone()
3575 .expect("attempt 2 ctx captured");
3576 assert_eq!(
3577 second.meta.runtime.get(STEP_CTX_KEY),
3578 Some(&serde_json::json!({ "work_dir": "/step" })),
3579 "attempt 2 (retry) must ALSO carry TaskSpec.step_ctx — prep re-reads the spec every attempt"
3580 );
3581 }
3582
3583 #[tokio::test]
3584 async fn step_ctx_key_absent_when_none() {
3585 let engine = Engine::new(EngineCfg::default());
3586 let token = engine
3587 .attach("ut-op", Role::Operator, Duration::from_secs(30))
3588 .await
3589 .expect("attach");
3590 let tid = engine
3591 .start_task(
3592 &token,
3593 TaskSpec {
3594 agent: "probe".into(),
3595 initial_directive: "hi".into(),
3596 step_ctx: None,
3597 check_policy: None,
3598 },
3599 )
3600 .await
3601 .expect("start_task");
3602 let seen: Arc<StdMutex<Option<Ctx>>> = Arc::new(StdMutex::new(None));
3603 let spawner: Arc<dyn SpawnerAdapter> = Arc::new(CtxProbe { seen: seen.clone() });
3604 let _ = engine
3605 .dispatch_attempt_with(&token, &tid, &spawner, None)
3606 .await;
3607 let observed = seen.lock().unwrap().clone().expect("ctx captured");
3608 assert!(
3609 !observed.meta.runtime.contains_key(STEP_CTX_KEY),
3610 "no step_ctx key must be injected when TaskSpec.step_ctx is None"
3611 );
3612 }
3613}
3614
3615// ─── issue #18: `TaskSpec.initial_directive` `Value` pass-through ──────────
3616#[cfg(test)]
3617mod initial_directive_value_passthrough_tests {
3618 use super::*;
3619
3620 async fn seeded_engine(initial_directive: Value) -> (Engine, CapToken, StepId) {
3621 let engine = Engine::new(EngineCfg::default());
3622 let op_token = engine
3623 .attach("ut-op", Role::Operator, Duration::from_secs(30))
3624 .await
3625 .expect("attach");
3626 let task_id = engine
3627 .start_task(
3628 &op_token,
3629 TaskSpec {
3630 agent: "planner".to_string(),
3631 initial_directive,
3632 step_ctx: None,
3633 check_policy: None,
3634 },
3635 )
3636 .await
3637 .expect("start_task");
3638 (engine, op_token, task_id)
3639 }
3640
3641 /// Mint + register a `Role::Worker` token the same way
3642 /// `dispatch_attempt_with` does — `fetch_prompt` is worker-verb-gated.
3643 async fn mint_worker_token(engine: &Engine, task_id: &StepId) -> CapToken {
3644 let worker_token = engine.signer().session(
3645 format!("worker-of-{task_id}"),
3646 Role::Worker,
3647 vec!["*".into()],
3648 Duration::from_secs(600),
3649 );
3650 let fp = worker_token.fingerprint();
3651 let record = CapTokenRecord::from_worker_token(worker_token.clone(), task_id.clone());
3652 engine
3653 .with_state("test.mint_worker", move |s| {
3654 s.tokens.insert(fp, record);
3655 })
3656 .await
3657 .expect("mint worker token");
3658 worker_token
3659 }
3660
3661 /// `EngineDispatcher::dispatch` no longer stringifies the evaluated
3662 /// `Step.in` value before seeding `TaskSpec.initial_directive` — an
3663 /// Object seed must round-trip through `start_task` /
3664 /// `read_task_state` byte-for-byte as the same `Value::Object`, not a
3665 /// JSON-stringified `Value::String`.
3666 #[tokio::test]
3667 async fn object_seed_passes_through_task_spec_unchanged() {
3668 let seed = serde_json::json!({"key": "value"});
3669 let (engine, token, task_id) = seeded_engine(seed.clone()).await;
3670 let state = engine
3671 .read_task_state(&token, &task_id)
3672 .await
3673 .expect("read_task_state");
3674 assert_eq!(
3675 state.spec.initial_directive, seed,
3676 "TaskSpec.initial_directive must equal the raw Object seed, not a stringified copy"
3677 );
3678 }
3679
3680 /// `Engine::fetch_prompt` returns the `Value` end-to-end (issue #18):
3681 /// an Object seed stays a `Value::Object` and is not stringified in
3682 /// the engine layer. The Worker HTTP boundary
3683 /// (`fetch_worker_payload*`) is what performs the render down to a
3684 /// JSON literal `String` for `WorkerPayload.prompt`.
3685 #[tokio::test]
3686 async fn object_seed_passes_through_fetch_prompt_as_value() {
3687 let seed = serde_json::json!({"key": "value"});
3688 let (engine, _token, task_id) = seeded_engine(seed.clone()).await;
3689 let worker_token = mint_worker_token(&engine, &task_id).await;
3690 let prompt = engine
3691 .fetch_prompt(&worker_token, &task_id)
3692 .await
3693 .expect("fetch_prompt");
3694 assert_eq!(
3695 prompt, seed,
3696 "fetch_prompt must return the raw Object Value, not a stringified copy"
3697 );
3698 }
3699
3700 /// The Worker HTTP boundary is the render point: `fetch_worker_payload*`
3701 /// coerces the stored `Value` down to `WorkerPayload.prompt: String`
3702 /// (JSON-literal shape for non-strings). Verifies the boundary render
3703 /// stays intact for an Object seed.
3704 #[tokio::test]
3705 async fn object_seed_renders_as_json_literal_at_worker_payload_boundary() {
3706 let seed = serde_json::json!({"key": "value"});
3707 let (engine, _token, task_id) = seeded_engine(seed).await;
3708 let worker_token = mint_worker_token(&engine, &task_id).await;
3709 let payload = engine
3710 .fetch_worker_payload(&worker_token, &task_id)
3711 .await
3712 .expect("fetch_worker_payload");
3713 assert_eq!(
3714 payload.prompt, r#"{"key":"value"}"#,
3715 "WorkerPayload.prompt must be the JSON literal String render of the Value seed"
3716 );
3717 }
3718
3719 /// A `String` seed is unaffected — still passes through verbatim, both
3720 /// as the `TaskSpec.initial_directive` `Value` and as the Worker
3721 /// `fetch_prompt` return (issue #18 Invariant 2).
3722 #[tokio::test]
3723 async fn string_seed_passes_through_unchanged() {
3724 let (engine, token, task_id) = seeded_engine(serde_json::json!("do the thing")).await;
3725 let state = engine
3726 .read_task_state(&token, &task_id)
3727 .await
3728 .expect("read_task_state");
3729 assert_eq!(
3730 state.spec.initial_directive,
3731 serde_json::json!("do the thing")
3732 );
3733 let worker_token = mint_worker_token(&engine, &task_id).await;
3734 let prompt = engine
3735 .fetch_prompt(&worker_token, &task_id)
3736 .await
3737 .expect("fetch_prompt");
3738 assert_eq!(prompt, serde_json::json!("do the thing"));
3739 }
3740}
3741
3742/// GH #31: `fetch_worker_payload{,_trusted}`'s size-threshold branch
3743/// between inline (`WorkerPayload.system`) and by-reference
3744/// (`WorkerPayload.system_ref`) delivery, plus the `bake_worker_system_prompt`
3745/// `agent_render_sizes` bookkeeping that feeds `agent_last_rendered_size`.
3746#[cfg(test)]
3747mod system_ref_threshold_tests {
3748 use super::*;
3749
3750 async fn seeded_engine_with_cfg(cfg: EngineCfg) -> (Engine, CapToken, StepId) {
3751 let engine = Engine::new(cfg);
3752 let op_token = engine
3753 .attach("ut-op", Role::Operator, Duration::from_secs(30))
3754 .await
3755 .expect("attach");
3756 let task_id = engine
3757 .start_task(
3758 &op_token,
3759 TaskSpec {
3760 agent: "planner".to_string(),
3761 initial_directive: serde_json::json!("do the thing"),
3762 step_ctx: None,
3763 check_policy: None,
3764 },
3765 )
3766 .await
3767 .expect("start_task");
3768 (engine, op_token, task_id)
3769 }
3770
3771 /// Same worker-token-minting fixture as
3772 /// `initial_directive_value_passthrough_tests::mint_worker_token`
3773 /// (kept local to this module — the two `mod`s do not share private
3774 /// helpers across `cfg(test)` boundaries).
3775 async fn mint_worker_token(engine: &Engine, task_id: &StepId) -> CapToken {
3776 let worker_token = engine.signer().session(
3777 format!("worker-of-{task_id}"),
3778 Role::Worker,
3779 vec!["*".into()],
3780 Duration::from_secs(600),
3781 );
3782 let fp = worker_token.fingerprint();
3783 let record = CapTokenRecord::from_worker_token(worker_token.clone(), task_id.clone());
3784 engine
3785 .with_state("test.mint_worker", move |s| {
3786 s.tokens.insert(fp, record);
3787 })
3788 .await
3789 .expect("mint worker token");
3790 worker_token
3791 }
3792
3793 /// Under-threshold: `system` stays inline, `system_ref` stays `None`.
3794 #[tokio::test]
3795 async fn under_threshold_stays_inline() {
3796 let (engine, _op_token, task_id) = seeded_engine_with_cfg(EngineCfg::default()).await;
3797 let worker_token = mint_worker_token(&engine, &task_id).await;
3798 let rendered = "a short system prompt".to_string();
3799 engine
3800 .bake_worker_system_prompt(&task_id, 1, Some(rendered.clone()))
3801 .await
3802 .expect("bake");
3803 let payload = engine
3804 .fetch_worker_payload(&worker_token, &task_id)
3805 .await
3806 .expect("fetch_worker_payload");
3807 assert_eq!(payload.system, Some(rendered));
3808 assert!(payload.system_ref.is_none());
3809 }
3810
3811 /// Over-threshold: `system` is cleared and `system_ref` is populated
3812 /// with a `sha256` matching the known input string. Exercises
3813 /// `fetch_worker_payload_trusted` (the `_trusted` sibling must be
3814 /// behaviorally identical to `fetch_worker_payload`).
3815 #[tokio::test]
3816 async fn over_threshold_switches_to_system_ref_with_matching_sha256() {
3817 let mut cfg = EngineCfg::default();
3818 cfg.system_ref.threshold_bytes = 16;
3819 cfg.system_ref.mode = crate::types::SystemRefMode::File;
3820 cfg.system_ref.store_dir =
3821 std::env::temp_dir().join(format!("mse-system-ref-test-{}", crate::types::now_unix()));
3822 let (engine, _op_token, task_id) = seeded_engine_with_cfg(cfg).await;
3823 let rendered =
3824 "this system prompt is deliberately longer than the 16 byte threshold".to_string();
3825 engine
3826 .bake_worker_system_prompt(&task_id, 1, Some(rendered.clone()))
3827 .await
3828 .expect("bake");
3829 let payload = engine
3830 .fetch_worker_payload_trusted(&task_id)
3831 .await
3832 .expect("fetch_worker_payload_trusted");
3833 assert!(
3834 payload.system.is_none(),
3835 "over-threshold response must not also inline `system`"
3836 );
3837 let system_ref = payload
3838 .system_ref
3839 .expect("over-threshold response must populate system_ref");
3840 assert_eq!(system_ref.size_bytes, rendered.len() as u64);
3841 assert_eq!(system_ref.mode, crate::types::SystemRefMode::File);
3842 use sha2::Digest;
3843 let expected_sha256 = hex::encode(sha2::Sha256::digest(rendered.as_bytes()));
3844 assert_eq!(system_ref.sha256, expected_sha256);
3845 assert!(system_ref.uri.starts_with("file://"));
3846 let written = tokio::fs::read_to_string(system_ref.uri.trim_start_matches("file://"))
3847 .await
3848 .expect("File mode must have written the referenced path");
3849 assert_eq!(written, rendered);
3850 }
3851
3852 /// `Http` mode never writes a file — `system_ref.uri` is the bare path
3853 /// the engine can construct on its own, scheme/host-free.
3854 #[tokio::test]
3855 async fn over_threshold_http_mode_constructs_path_only_uri() {
3856 let mut cfg = EngineCfg::default();
3857 cfg.system_ref.threshold_bytes = 16;
3858 cfg.system_ref.mode = crate::types::SystemRefMode::Http;
3859 let (engine, _op_token, task_id) = seeded_engine_with_cfg(cfg).await;
3860 let worker_token = mint_worker_token(&engine, &task_id).await;
3861 let rendered =
3862 "this system prompt is deliberately longer than the 16 byte threshold".to_string();
3863 engine
3864 .bake_worker_system_prompt(&task_id, 1, Some(rendered))
3865 .await
3866 .expect("bake");
3867 let payload = engine
3868 .fetch_worker_payload(&worker_token, &task_id)
3869 .await
3870 .expect("fetch_worker_payload");
3871 let system_ref = payload.system_ref.expect("system_ref must be populated");
3872 assert_eq!(system_ref.mode, crate::types::SystemRefMode::Http);
3873 assert_eq!(
3874 system_ref.uri,
3875 format!("/v1/worker/prompt/system?task_id={task_id}&attempt=1")
3876 );
3877 }
3878
3879 /// `bake_worker_system_prompt` records the render size keyed by agent
3880 /// name (last-write-wins), readable via `agent_last_rendered_size`.
3881 #[tokio::test]
3882 async fn bake_records_agent_render_size_last_write_wins() {
3883 let (engine, _op_token, task_id) = seeded_engine_with_cfg(EngineCfg::default()).await;
3884 assert_eq!(engine.agent_last_rendered_size("planner").await, None);
3885 engine
3886 .bake_worker_system_prompt(&task_id, 1, Some("a".repeat(10)))
3887 .await
3888 .expect("bake 1");
3889 assert_eq!(engine.agent_last_rendered_size("planner").await, Some(10));
3890 engine
3891 .bake_worker_system_prompt(&task_id, 2, Some("b".repeat(20)))
3892 .await
3893 .expect("bake 2");
3894 assert_eq!(
3895 engine.agent_last_rendered_size("planner").await,
3896 Some(20),
3897 "most-recently-observed size wins, not the largest"
3898 );
3899 }
3900
3901 /// GH #83: `materialize_system_file` writes the baked system prompt
3902 /// unconditionally — a system well UNDER `threshold_bytes` still
3903 /// lands on disk, because a `{system_file}` template reference needs
3904 /// a real path regardless of size.
3905 #[tokio::test]
3906 async fn materialize_system_file_writes_under_threshold_system() {
3907 let mut cfg = EngineCfg::default();
3908 cfg.system_ref.store_dir =
3909 std::env::temp_dir().join(format!("mse-system-file-test-{}", crate::types::now_unix()));
3910 assert!(cfg.system_ref.threshold_bytes > 64, "fixture premise");
3911 let (engine, _op_token, task_id) = seeded_engine_with_cfg(cfg).await;
3912 let rendered = "a short system prompt".to_string();
3913 engine
3914 .bake_worker_system_prompt(&task_id, 1, Some(rendered.clone()))
3915 .await
3916 .expect("bake");
3917 let path = engine
3918 .materialize_system_file(&task_id, 1)
3919 .await
3920 .expect("materialize_system_file")
3921 .expect("baked system must yield a path");
3922 let written = tokio::fs::read_to_string(&path)
3923 .await
3924 .expect("materialized path must exist");
3925 assert_eq!(written, rendered);
3926 }
3927
3928 /// GH #83: no baked system → `Ok(None)` (the Subprocess spawn path
3929 /// turns this into a fail-loud `SpawnError` when `{system_file}` is
3930 /// actually referenced).
3931 #[tokio::test]
3932 async fn materialize_system_file_none_when_nothing_baked() {
3933 let (engine, _op_token, task_id) = seeded_engine_with_cfg(EngineCfg::default()).await;
3934 let path = engine
3935 .materialize_system_file(&task_id, 1)
3936 .await
3937 .expect("materialize_system_file");
3938 assert!(path.is_none());
3939 }
3940}
3941
3942/// subtask-4 / ST2 rework: `submit_output` / `submit_worker_result_trusted`'s
3943/// submit-time projection sink (`Engine::materialize_final_submission`) —
3944/// the Data-plane `OutputStore` dual-write plus the
3945/// `FileProjectionAdapter`-backed file materialize, both fail-open. See
3946/// the subtask-4 Tests this module covers inline on each test.
3947#[cfg(test)]
3948mod submit_time_projection_sink_tests {
3949 use super::*;
3950 use crate::core::agent_context::AgentContextView;
3951 use crate::store::output::{ContentRef, InMemoryOutputStore, OutputEvent};
3952
3953 /// Starts a task under `agent`, returning `(engine, op_token, task_id,
3954 /// worker_token)` — same helper shape as the sibling test modules
3955 /// above (`initial_directive_value_passthrough_tests::seeded_engine` /
3956 /// `mint_worker_token`), duplicated locally per this file's
3957 /// established per-module convention.
3958 async fn seeded_task(agent: &str) -> (Engine, CapToken, StepId, CapToken) {
3959 let engine = Engine::new(EngineCfg::default());
3960 let op_token = engine
3961 .attach("ut-op", Role::Operator, Duration::from_secs(30))
3962 .await
3963 .expect("attach");
3964 let task_id = engine
3965 .start_task(
3966 &op_token,
3967 TaskSpec {
3968 agent: agent.to_string(),
3969 initial_directive: Value::String("go".into()),
3970 step_ctx: None,
3971 check_policy: None,
3972 },
3973 )
3974 .await
3975 .expect("start_task");
3976 let worker_token = engine.signer().session(
3977 format!("worker-of-{task_id}"),
3978 Role::Worker,
3979 vec!["*".into()],
3980 Duration::from_secs(600),
3981 );
3982 let fp = worker_token.fingerprint();
3983 let record = CapTokenRecord::from_worker_token(worker_token.clone(), task_id.clone());
3984 engine
3985 .with_state("test.mint_worker", move |s| {
3986 s.tokens.insert(fp, record);
3987 })
3988 .await
3989 .expect("mint worker token");
3990 (engine, op_token, task_id, worker_token)
3991 }
3992
3993 /// Sibling of [`seeded_task`] that lets a caller pin the engine's
3994 /// `EngineCfg.check_policy` before the engine is constructed — used
3995 /// by the `check_policy_*` regression tests below to exercise the
3996 /// three [`crate::core::config::CheckPolicy`] modes without touching
3997 /// the shared `seeded_task` helper (which every unrelated sink test
3998 /// depends on).
3999 async fn seeded_task_with_policy(
4000 agent: &str,
4001 policy: crate::core::config::CheckPolicy,
4002 ) -> (Engine, CapToken, StepId, CapToken) {
4003 let cfg = EngineCfg {
4004 check_policy: policy,
4005 ..EngineCfg::default()
4006 };
4007 let engine = Engine::new(cfg);
4008 let op_token = engine
4009 .attach("ut-op", Role::Operator, Duration::from_secs(30))
4010 .await
4011 .expect("attach");
4012 let task_id = engine
4013 .start_task(
4014 &op_token,
4015 TaskSpec {
4016 agent: agent.to_string(),
4017 initial_directive: Value::String("go".into()),
4018 step_ctx: None,
4019 check_policy: None,
4020 },
4021 )
4022 .await
4023 .expect("start_task");
4024 let worker_token = engine.signer().session(
4025 format!("worker-of-{task_id}"),
4026 Role::Worker,
4027 vec!["*".into()],
4028 Duration::from_secs(600),
4029 );
4030 let fp = worker_token.fingerprint();
4031 let record = CapTokenRecord::from_worker_token(worker_token.clone(), task_id.clone());
4032 engine
4033 .with_state("test.mint_worker", move |s| {
4034 s.tokens.insert(fp, record);
4035 })
4036 .await
4037 .expect("mint worker token");
4038 (engine, op_token, task_id, worker_token)
4039 }
4040
4041 /// Seeds `EngineState.agent_ctx[(task_id, attempt)].view` directly —
4042 /// the same snapshot `AgentContextMiddleware` writes at spawn time
4043 /// (see its module doc), stood up here without the full spawner
4044 /// stack so these tests can exercise `submit_output` in isolation.
4045 async fn seed_agent_context(engine: &Engine, task_id: &StepId, attempt: u32, work_dir: &str) {
4046 let task_id = task_id.clone();
4047 let work_dir = work_dir.to_string();
4048 engine
4049 .with_state("test.seed_agent_context", move |s| {
4050 s.agent_ctx.insert(
4051 (task_id, attempt),
4052 crate::core::state::AgentCtxEntry {
4053 view: AgentContextView {
4054 work_dir: Some(work_dir),
4055 ..Default::default()
4056 },
4057 policy: Default::default(),
4058 },
4059 );
4060 })
4061 .await
4062 .expect("seed agent_ctx");
4063 }
4064
4065 /// GH #27 (follow-up to #23): seeds `EngineState.agent_ctx` with an
4066 /// arbitrary `work_dir` / `project_root` pair (either may be `None`),
4067 /// unlike [`seed_agent_context`] (which only ever sets `work_dir`) —
4068 /// lets these tests exercise `ProjectionPlacement::resolve_root`'s
4069 /// fallback in both directions.
4070 async fn seed_agent_context_roots(
4071 engine: &Engine,
4072 task_id: &StepId,
4073 attempt: u32,
4074 work_dir: Option<&str>,
4075 project_root: Option<&str>,
4076 ) {
4077 let task_id = task_id.clone();
4078 let work_dir = work_dir.map(str::to_string);
4079 let project_root = project_root.map(str::to_string);
4080 engine
4081 .with_state("test.seed_agent_context_roots", move |s| {
4082 s.agent_ctx.insert(
4083 (task_id, attempt),
4084 crate::core::state::AgentCtxEntry {
4085 view: AgentContextView {
4086 work_dir,
4087 project_root,
4088 ..Default::default()
4089 },
4090 policy: Default::default(),
4091 },
4092 );
4093 })
4094 .await
4095 .expect("seed agent_ctx");
4096 }
4097
4098 /// GH #27 (follow-up to #23): seeds `EngineState.projection_placements`
4099 /// directly — the same snapshot `EngineDispatcher::dispatch` stashes
4100 /// at dispatch time (mirroring [`seed_step_naming`]'s contract) — so
4101 /// these tests can exercise a declared `ProjectionPlacement` without
4102 /// driving a real `Compiler::compile`.
4103 async fn seed_projection_placement(
4104 engine: &Engine,
4105 task_id: &StepId,
4106 placement: crate::core::projection_placement::ProjectionPlacement,
4107 ) {
4108 let task_id = task_id.clone();
4109 let placement = Arc::new(placement);
4110 engine
4111 .with_state("test.seed_projection_placement", move |s| {
4112 s.projection_placements.insert(task_id, placement);
4113 })
4114 .await
4115 .expect("seed projection_placements");
4116 }
4117
4118 /// GH #23 subtask-2: builds a fixture
4119 /// [`crate::core::step_naming::StepNaming`] table declaring `producer`
4120 /// → `canonical` (`AgentMeta.projection_name`), then seeds it into
4121 /// `EngineState.step_namings` for `task_id` — the same snapshot
4122 /// `EngineDispatcher::dispatch` stashes at dispatch time
4123 /// (`blueprint.rs`'s "construct once, read many" contract), stood up
4124 /// here without the full Blueprint-compile path so these tests can
4125 /// exercise the canonical-sink resolution in isolation.
4126 async fn seed_step_naming(engine: &Engine, task_id: &StepId, producer: &str, canonical: &str) {
4127 use crate::blueprint::{
4128 current_schema_version, AgentDef, AgentKind, AgentMeta, Blueprint, BlueprintMetadata,
4129 CompilerHints, CompilerStrategy,
4130 };
4131 use crate::core::step_naming::StepNaming;
4132 use mlua_flow_ir::{Expr, Node};
4133
4134 let flow = Node::Step {
4135 ref_: producer.to_string(),
4136 in_: Expr::Path {
4137 at: "$.in".parse().expect("literal test path: $.in"),
4138 },
4139 out: Expr::Path {
4140 at: format!("$.{producer}_out")
4141 .parse()
4142 .expect("literal test path"),
4143 },
4144 };
4145 let bp = Blueprint {
4146 schema_version: current_schema_version(),
4147 id: "sink-canonical-ut".into(),
4148 flow,
4149 agents: vec![AgentDef {
4150 name: producer.to_string(),
4151 kind: AgentKind::RustFn,
4152 spec: serde_json::json!({ "fn_id": producer }),
4153 profile: None,
4154 meta: Some(AgentMeta {
4155 projection_name: Some(canonical.to_string()),
4156 ..Default::default()
4157 }),
4158 runner: None,
4159 runner_ref: None,
4160 verdict: None,
4161 }],
4162 operators: vec![],
4163 metas: vec![],
4164 hints: CompilerHints::default(),
4165 strategy: CompilerStrategy::default(),
4166 metadata: BlueprintMetadata::default(),
4167 spawner_hints: Default::default(),
4168 default_agent_kind: AgentKind::Operator,
4169 default_operator_kind: None,
4170 default_init_ctx: None,
4171 default_agent_ctx: None,
4172 default_context_policy: None,
4173 projection_placement: None,
4174 audits: vec![],
4175 degradation_policy: None,
4176 runners: vec![],
4177 default_runner: None,
4178 subprocesses: vec![],
4179 check_policy: None,
4180 blueprint_ref_includes: Vec::new(),
4181 };
4182 let (naming, warnings) = StepNaming::from_blueprint(&bp).expect("no collision");
4183 assert!(warnings.is_empty(), "single-step fixture has no collisions");
4184 let naming = Arc::new(naming);
4185 let task_id = task_id.clone();
4186 engine
4187 .with_state("test.seed_step_naming", move |s| {
4188 s.step_namings.insert(task_id, naming);
4189 })
4190 .await
4191 .expect("seed step_namings");
4192 }
4193
4194 fn final_event(value: Value, ok: bool) -> crate::worker::output::OutputEvent {
4195 crate::worker::output::OutputEvent::Final {
4196 content: crate::worker::output::ContentRef::Inline { value },
4197 ok,
4198 }
4199 }
4200
4201 /// Subtask 4 Test #2: `submit_output`'s `Final` writes
4202 /// `<root>/workspace/tasks/<task_id>/ctx/<agent>.md`, content matching
4203 /// the submitted value.
4204 #[tokio::test]
4205 async fn submit_output_final_materializes_file_when_work_dir_resolved() {
4206 let dir = tempfile::TempDir::new().unwrap();
4207 let (engine, _op, task_id, worker_token) = seeded_task("planner").await;
4208 seed_agent_context(&engine, &task_id, 1, &dir.path().to_string_lossy()).await;
4209
4210 engine
4211 .submit_output(
4212 &worker_token,
4213 &task_id,
4214 1,
4215 final_event(serde_json::json!({"plan": "do it"}), true),
4216 )
4217 .await
4218 .expect("submit_output");
4219
4220 let expected_file = dir
4221 .path()
4222 .join("workspace/tasks")
4223 .join(task_id.as_str())
4224 .join("ctx/planner.md");
4225 assert!(
4226 expected_file.exists(),
4227 "materialized submission file missing at {expected_file:?}"
4228 );
4229 let body = std::fs::read_to_string(expected_file).unwrap();
4230 assert!(body.contains(r#""plan": "do it""#), "body: {body}");
4231 }
4232
4233 /// Subtask 4 Test #3: `work_dir` unresolved (no `agent_ctx`
4234 /// snapshot for this `(task_id, attempt)`) — submit still succeeds,
4235 /// fail-open, no file.
4236 #[tokio::test]
4237 async fn submit_output_final_skips_file_when_root_unresolved() {
4238 let (engine, _op, task_id, worker_token) = seeded_task("planner").await;
4239 // No seed_agent_context call — root is unresolved.
4240
4241 let result = engine
4242 .submit_output(
4243 &worker_token,
4244 &task_id,
4245 1,
4246 final_event(serde_json::json!("hi"), true),
4247 )
4248 .await;
4249 assert!(
4250 result.is_ok(),
4251 "submit must succeed even with no resolvable root (fail-open, Invariant 1)"
4252 );
4253 }
4254
4255 /// Regression for the check_policy cascade: the default
4256 /// [`crate::core::config::CheckPolicy::Warn`] preserves the
4257 /// pre-`CheckPolicy` fail-open semantics — a submit whose root is
4258 /// unresolved still succeeds. Byte-compat with
4259 /// `submit_output_final_skips_file_when_root_unresolved`; this test
4260 /// pins the mode explicitly so a future default change to
4261 /// `Strict` (silent breakage) is caught here.
4262 #[tokio::test]
4263 async fn submit_output_final_check_policy_warn_preserves_fail_open() {
4264 let (engine, _op, task_id, worker_token) =
4265 seeded_task_with_policy("planner", crate::core::config::CheckPolicy::Warn).await;
4266
4267 let result = engine
4268 .submit_output(
4269 &worker_token,
4270 &task_id,
4271 1,
4272 final_event(serde_json::json!("hi"), true),
4273 )
4274 .await;
4275 assert!(
4276 result.is_ok(),
4277 "Warn mode preserves fail-open: submit must succeed when root unresolved"
4278 );
4279 }
4280
4281 /// Regression for the check_policy cascade:
4282 /// [`crate::core::config::CheckPolicy::Strict`] surfaces the "no
4283 /// work_dir/project_root resolved" fail-open condition as an
4284 /// [`EngineError::CheckPolicyStrict`], letting a caller who has
4285 /// opted in fail fast instead of proceeding with a partially-
4286 /// realized submission. The error's `context` identifies the call
4287 /// site (`"file materialize"`), and `message` preserves the
4288 /// pre-`CheckPolicy` warn literal verbatim (log-parse compat).
4289 #[tokio::test]
4290 async fn submit_output_final_check_policy_strict_surfaces_error_when_root_unresolved() {
4291 let (engine, _op, task_id, worker_token) =
4292 seeded_task_with_policy("planner", crate::core::config::CheckPolicy::Strict).await;
4293
4294 let err = engine
4295 .submit_output(
4296 &worker_token,
4297 &task_id,
4298 1,
4299 final_event(serde_json::json!("hi"), true),
4300 )
4301 .await
4302 .expect_err("Strict mode must return an error when root unresolved");
4303 match err {
4304 EngineError::CheckPolicyStrict { context, message } => {
4305 assert!(
4306 context.contains("file materialize"),
4307 "context must identify the call site: {context}"
4308 );
4309 assert!(
4310 message.contains("no work_dir/project_root resolved"),
4311 "message must preserve the warn-log literal for log-parse compat: {message}"
4312 );
4313 }
4314 other => panic!(
4315 "expected EngineError::CheckPolicyStrict, got a different variant: {other:?}"
4316 ),
4317 }
4318 }
4319
4320 /// Regression for the check_policy cascade:
4321 /// [`crate::core::config::CheckPolicy::Silent`] returns `Ok(())` (
4322 /// like `Warn`) without surfacing an error. The log-suppression side
4323 /// of `Silent` (no `tracing::warn!`) is enforced at the call site
4324 /// via the `if !matches!(policy, Silent) { warn!(...) }` guard —
4325 /// verifying tracing output shape here would couple the test to a
4326 /// subscriber setup, so the assertion is limited to the error-
4327 /// return semantics (matches the helper unit tests in
4328 /// `check_policy_helper_tests`).
4329 #[tokio::test]
4330 async fn submit_output_final_check_policy_silent_returns_ok_when_root_unresolved() {
4331 let (engine, _op, task_id, worker_token) =
4332 seeded_task_with_policy("planner", crate::core::config::CheckPolicy::Silent).await;
4333
4334 let result = engine
4335 .submit_output(
4336 &worker_token,
4337 &task_id,
4338 1,
4339 final_event(serde_json::json!("hi"), true),
4340 )
4341 .await;
4342 assert!(
4343 result.is_ok(),
4344 "Silent mode returns Ok(()) at the error surface: submit must succeed"
4345 );
4346 }
4347
4348 /// Subtask 4 Test #4 (file half): re-submitting under the same
4349 /// `(task_id, agent)` overwrites the materialized file with the
4350 /// latest value.
4351 #[tokio::test]
4352 async fn resubmit_overwrites_materialized_file_with_latest() {
4353 let dir = tempfile::TempDir::new().unwrap();
4354 let (engine, _op, task_id, worker_token) = seeded_task("planner").await;
4355 seed_agent_context(&engine, &task_id, 1, &dir.path().to_string_lossy()).await;
4356
4357 engine
4358 .submit_output(
4359 &worker_token,
4360 &task_id,
4361 1,
4362 final_event(serde_json::json!("first"), true),
4363 )
4364 .await
4365 .expect("first submit");
4366 engine
4367 .submit_output(
4368 &worker_token,
4369 &task_id,
4370 1,
4371 final_event(serde_json::json!("second"), true),
4372 )
4373 .await
4374 .expect("second submit");
4375
4376 let expected_file = dir
4377 .path()
4378 .join("workspace/tasks")
4379 .join(task_id.as_str())
4380 .join("ctx/planner.md");
4381 let body = std::fs::read_to_string(expected_file).unwrap();
4382 assert!(body.contains("second"), "body must reflect latest: {body}");
4383 assert!(
4384 !body.contains("first"),
4385 "body must not carry the stale value: {body}"
4386 );
4387 }
4388
4389 /// GH #27 (follow-up to #23): the byte-compat default
4390 /// `ProjectionPlacement` (`root_preference = WorkDir`) falls back to
4391 /// `project_root` when `work_dir` is absent — the same fallback
4392 /// [`crate::core::projection_placement::ProjectionPlacement::resolve_root`]
4393 /// now performs for every one of the "3 path" call sites, this one
4394 /// exercised at the submit-sink layer.
4395 #[tokio::test]
4396 async fn submit_output_final_falls_back_to_project_root_when_work_dir_absent() {
4397 let dir = tempfile::TempDir::new().unwrap();
4398 let (engine, _op, task_id, worker_token) = seeded_task("planner").await;
4399 seed_agent_context_roots(
4400 &engine,
4401 &task_id,
4402 1,
4403 None,
4404 Some(&dir.path().to_string_lossy()),
4405 )
4406 .await;
4407
4408 engine
4409 .submit_output(
4410 &worker_token,
4411 &task_id,
4412 1,
4413 final_event(serde_json::json!({"plan": "via project_root"}), true),
4414 )
4415 .await
4416 .expect("submit_output");
4417
4418 let expected_file = dir
4419 .path()
4420 .join("workspace/tasks")
4421 .join(task_id.as_str())
4422 .join("ctx/planner.md");
4423 assert!(
4424 expected_file.exists(),
4425 "materialized submission file missing at {expected_file:?} \
4426 (work_dir absent must fall back to project_root)"
4427 );
4428 }
4429
4430 /// GH #27 (follow-up to #23): a declared `ProjectionPlacement`
4431 /// (`root_preference = ProjectRoot`, custom `dir_template`) changes
4432 /// BOTH which root is preferred (project_root wins even though
4433 /// work_dir is also present) AND the target directory layout — proof
4434 /// the submit sink consults the snapshotted resolver rather than a
4435 /// hardcoded layout.
4436 #[tokio::test]
4437 async fn submit_output_final_uses_declared_projection_placement() {
4438 let work_dir = tempfile::TempDir::new().unwrap();
4439 let project_root = tempfile::TempDir::new().unwrap();
4440 let (engine, _op, task_id, worker_token) = seeded_task("planner").await;
4441 seed_agent_context_roots(
4442 &engine,
4443 &task_id,
4444 1,
4445 Some(&work_dir.path().to_string_lossy()),
4446 Some(&project_root.path().to_string_lossy()),
4447 )
4448 .await;
4449 seed_projection_placement(
4450 &engine,
4451 &task_id,
4452 crate::core::projection_placement::ProjectionPlacement {
4453 root_preference: crate::core::projection_placement::RootPreference::ProjectRoot,
4454 dir_template: "custom/{task_id}/out".to_string(),
4455 },
4456 )
4457 .await;
4458
4459 engine
4460 .submit_output(
4461 &worker_token,
4462 &task_id,
4463 1,
4464 final_event(serde_json::json!({"plan": "via custom placement"}), true),
4465 )
4466 .await
4467 .expect("submit_output");
4468
4469 let expected_file = project_root
4470 .path()
4471 .join("custom")
4472 .join(task_id.as_str())
4473 .join("out/planner.md");
4474 assert!(
4475 expected_file.exists(),
4476 "materialized submission file missing at custom placement target {expected_file:?}"
4477 );
4478 let unexpected_file = work_dir
4479 .path()
4480 .join("workspace/tasks")
4481 .join(task_id.as_str())
4482 .join("ctx/planner.md");
4483 assert!(
4484 !unexpected_file.exists(),
4485 "declared root_preference=ProjectRoot must not fall back to work_dir: {unexpected_file:?}"
4486 );
4487 }
4488
4489 /// Subtask 4 Invariant 3 / crux requirement #3: when
4490 /// [`Engine::set_output_store`] wires a Data-plane [`crate::store::output::OutputStore`],
4491 /// `submit_output`'s `Final` dual-writes into it under
4492 /// `producer_agent = TaskState.spec.agent` — the store becomes
4493 /// queryable via `get_latest_by_name`, independent of whether a root
4494 /// resolved for the file half.
4495 #[tokio::test]
4496 async fn submit_output_final_dual_writes_into_configured_output_store() {
4497 let (engine, _op, task_id, worker_token) = seeded_task("reviewer").await;
4498 let data_store: Arc<dyn crate::store::output::OutputStore> =
4499 Arc::new(InMemoryOutputStore::new());
4500 engine.set_output_store(data_store.clone());
4501
4502 engine
4503 .submit_output(
4504 &worker_token,
4505 &task_id,
4506 1,
4507 final_event(serde_json::json!({"verdict": "pass"}), true),
4508 )
4509 .await
4510 .expect("submit_output");
4511
4512 let record = data_store
4513 .get_latest_by_name("reviewer")
4514 .await
4515 .expect("dual-written record");
4516 match record.event {
4517 OutputEvent::Final { content, ok } => {
4518 assert!(ok);
4519 match content {
4520 ContentRef::Inline { value } => {
4521 assert_eq!(value, serde_json::json!({"verdict": "pass"}));
4522 }
4523 other => panic!("expected Inline content, got {other:?}"),
4524 }
4525 }
4526 other => panic!("expected Final event, got {other:?}"),
4527 }
4528 }
4529
4530 /// GH #34 subtask-3 gap fix: an `Artifact` event submitted via
4531 /// `submit_output` dual-writes into a wired Data-plane `OutputStore`
4532 /// under its OWN `name`, verbatim — mirrors
4533 /// `submit_output_final_dual_writes_into_configured_output_store`
4534 /// above, but for the `Artifact` variant.
4535 #[tokio::test]
4536 async fn submit_output_artifact_dual_writes_into_configured_output_store() {
4537 let (engine, _op, task_id, worker_token) = seeded_task("echo").await;
4538 let data_store: Arc<dyn crate::store::output::OutputStore> =
4539 Arc::new(InMemoryOutputStore::new());
4540 engine.set_output_store(data_store.clone());
4541
4542 engine
4543 .submit_output(
4544 &worker_token,
4545 &task_id,
4546 1,
4547 OutputEvent::Artifact {
4548 name: "audit:echo".to_string(),
4549 content: ContentRef::Inline {
4550 value: serde_json::json!({"finding": "clean"}),
4551 },
4552 },
4553 )
4554 .await
4555 .expect("submit_output");
4556
4557 let record = data_store
4558 .get_latest_by_name("audit:echo")
4559 .await
4560 .expect("dual-written artifact record");
4561 match record.event {
4562 OutputEvent::Artifact { name, content } => {
4563 assert_eq!(name, "audit:echo");
4564 match content {
4565 ContentRef::Inline { value } => {
4566 assert_eq!(value, serde_json::json!({"finding": "clean"}));
4567 }
4568 other => panic!("expected Inline content, got {other:?}"),
4569 }
4570 }
4571 other => panic!("expected Artifact event, got {other:?}"),
4572 }
4573 // The `Artifact` dual-write must never collide with / overwrite
4574 // the producing step's own `Final` name — `submit_output` never
4575 // materialized a `Final` here, so `"echo"` must stay unresolved.
4576 assert!(
4577 data_store.get_latest_by_name("echo").await.is_err(),
4578 "artifact write must not fabricate a record under the raw producer_agent name"
4579 );
4580 }
4581
4582 /// Invariant 1 (fail-open) for `Artifact`, mirroring
4583 /// `submit_output_final_skips_file_when_root_unresolved`'s Final-side
4584 /// coverage: no `OutputStore` wired at all — submit still succeeds.
4585 #[tokio::test]
4586 async fn submit_output_artifact_is_fail_open_when_no_output_store_configured() {
4587 let (engine, _op, task_id, worker_token) = seeded_task("echo").await;
4588
4589 let result = engine
4590 .submit_output(
4591 &worker_token,
4592 &task_id,
4593 1,
4594 OutputEvent::Artifact {
4595 name: "audit:echo".to_string(),
4596 content: ContentRef::Inline {
4597 value: serde_json::json!("finding"),
4598 },
4599 },
4600 )
4601 .await;
4602 assert!(
4603 result.is_ok(),
4604 "submit must succeed even with no OutputStore wired (fail-open, Invariant 1)"
4605 );
4606 }
4607
4608 /// `submit_worker_result_trusted` (the `/v1/worker/submit` short-handle
4609 /// path) triggers the exact same sink as `submit_output` — parity
4610 /// across both worker-submit entry points.
4611 #[tokio::test]
4612 async fn submit_worker_result_trusted_also_triggers_projection_sink() {
4613 let dir = tempfile::TempDir::new().unwrap();
4614 let (engine, _op, task_id, _worker_token) = seeded_task("planner").await;
4615 seed_agent_context(&engine, &task_id, 1, &dir.path().to_string_lossy()).await;
4616 let data_store: Arc<dyn crate::store::output::OutputStore> =
4617 Arc::new(InMemoryOutputStore::new());
4618 engine.set_output_store(data_store.clone());
4619
4620 engine
4621 .submit_worker_result_trusted(
4622 &task_id,
4623 1,
4624 serde_json::json!("trusted-value"),
4625 SubmitOutcome::Pass,
4626 )
4627 .await
4628 .expect("submit_worker_result_trusted");
4629
4630 let expected_file = dir
4631 .path()
4632 .join("workspace/tasks")
4633 .join(task_id.as_str())
4634 .join("ctx/planner.md");
4635 assert!(expected_file.exists());
4636 let record = data_store
4637 .get_latest_by_name("planner")
4638 .await
4639 .expect("dual-written record");
4640 assert!(matches!(record.event, OutputEvent::Final { ok: true, .. }));
4641 }
4642
4643 /// GH #23 subtask-2 (canonical sink): a declared `projection_name`
4644 /// (`AgentMeta.projection_name`, surfaced via `StepNaming`) redirects
4645 /// `submit_output`'s Final canonical sink — both the Data-plane
4646 /// dual-write name and the materialized file stem resolve to the
4647 /// canonical name, not the raw `producer_agent`.
4648 #[tokio::test]
4649 async fn submit_output_final_uses_canonical_name_when_step_naming_declares_one() {
4650 let dir = tempfile::TempDir::new().unwrap();
4651 let (engine, _op, task_id, worker_token) = seeded_task("reviewer").await;
4652 seed_agent_context(&engine, &task_id, 1, &dir.path().to_string_lossy()).await;
4653 seed_step_naming(&engine, &task_id, "reviewer", "verdict-final").await;
4654 let data_store: Arc<dyn crate::store::output::OutputStore> =
4655 Arc::new(InMemoryOutputStore::new());
4656 engine.set_output_store(data_store.clone());
4657
4658 engine
4659 .submit_output(
4660 &worker_token,
4661 &task_id,
4662 1,
4663 final_event(serde_json::json!({"verdict": "pass"}), true),
4664 )
4665 .await
4666 .expect("submit_output");
4667
4668 let record = data_store
4669 .get_latest_by_name("verdict-final")
4670 .await
4671 .expect("dual-written record under canonical name");
4672 assert!(matches!(record.event, OutputEvent::Final { ok: true, .. }));
4673 assert!(
4674 data_store.get_latest_by_name("reviewer").await.is_err(),
4675 "raw producer_agent name must not be written once canonical resolves"
4676 );
4677
4678 let expected_file = dir
4679 .path()
4680 .join("workspace/tasks")
4681 .join(task_id.as_str())
4682 .join("ctx/verdict-final.md");
4683 assert!(
4684 expected_file.exists(),
4685 "materialized file stem must be canonical at {expected_file:?}"
4686 );
4687 }
4688
4689 /// GH #23 subtask-2: no `StepNaming` table snapshotted for this
4690 /// `task_id` (the pre-GH-#23 / no-`with_step_naming` path) is a
4691 /// defensive fail-open — the canonical sink falls back to the raw
4692 /// `producer_agent`, byte-identical to
4693 /// `submit_output_final_dual_writes_into_configured_output_store`
4694 /// above (which never calls `seed_step_naming`).
4695 #[tokio::test]
4696 async fn submit_output_final_falls_back_to_producer_agent_when_no_step_naming_table() {
4697 let (engine, _op, task_id, worker_token) = seeded_task("reviewer").await;
4698 let data_store: Arc<dyn crate::store::output::OutputStore> =
4699 Arc::new(InMemoryOutputStore::new());
4700 engine.set_output_store(data_store.clone());
4701
4702 engine
4703 .submit_output(
4704 &worker_token,
4705 &task_id,
4706 1,
4707 final_event(serde_json::json!({"verdict": "pass"}), true),
4708 )
4709 .await
4710 .expect("submit_output");
4711
4712 let record = data_store
4713 .get_latest_by_name("reviewer")
4714 .await
4715 .expect("fail-open dual-write under raw producer_agent name");
4716 assert!(matches!(record.event, OutputEvent::Final { ok: true, .. }));
4717 }
4718
4719 /// GH #23 subtask-2 (Layer 2): `OutputStore::get_latest_by_name_in_run`
4720 /// resolves the value `submit_output` dual-wrote for this exact
4721 /// `(task_id, attempt)` run, independent of `get_latest_by_name`'s
4722 /// cross-Run race (two Runs sharing a producer name never bleed into
4723 /// each other through the Run-scoped accessor).
4724 #[tokio::test]
4725 async fn submit_output_final_is_resolvable_via_run_scoped_lookup() {
4726 let (engine, _op, task_id, worker_token) = seeded_task("reviewer").await;
4727 let data_store: Arc<dyn crate::store::output::OutputStore> =
4728 Arc::new(InMemoryOutputStore::new());
4729 engine.set_output_store(data_store.clone());
4730
4731 engine
4732 .submit_output(
4733 &worker_token,
4734 &task_id,
4735 1,
4736 final_event(serde_json::json!({"verdict": "pass"}), true),
4737 )
4738 .await
4739 .expect("submit_output");
4740
4741 let record = data_store
4742 .get_latest_by_name_in_run(task_id.as_str(), 1, "reviewer")
4743 .await
4744 .expect("run-scoped lookup resolves the dual-written record");
4745 assert!(matches!(record.event, OutputEvent::Final { ok: true, .. }));
4746
4747 // A different attempt of the same task must not resolve — the
4748 // Run-scoped lookup does not fall back across attempts.
4749 assert!(
4750 data_store
4751 .get_latest_by_name_in_run(task_id.as_str(), 2, "reviewer")
4752 .await
4753 .is_err(),
4754 "a different attempt must not resolve the same-named record"
4755 );
4756 }
4757
4758 // ─── staged part file materialize ───
4759
4760 /// Staging a part with a resolved `work_dir` writes
4761 /// `<work_dir>/workspace/tasks/<task_id>/ctx/<name>` with the part's
4762 /// content RAW (no front matter / fenced wrapper).
4763 #[tokio::test]
4764 async fn stage_artifact_materializes_part_file_when_work_dir_resolved() {
4765 let dir = tempfile::TempDir::new().unwrap();
4766 let (engine, _op, task_id, _worker_token) = seeded_task("planner").await;
4767 seed_agent_context(&engine, &task_id, 1, &dir.path().to_string_lossy()).await;
4768
4769 engine
4770 .stage_worker_artifact_trusted(
4771 &task_id,
4772 1,
4773 "plan.md".to_string(),
4774 serde_json::json!("# Plan\n\nstep one\n"),
4775 )
4776 .await
4777 .expect("stage artifact");
4778
4779 let expected_file = dir
4780 .path()
4781 .join("workspace/tasks")
4782 .join(task_id.as_str())
4783 .join("ctx/plan.md");
4784 assert!(
4785 expected_file.exists(),
4786 "materialized part file missing at {expected_file:?}"
4787 );
4788 let body = std::fs::read_to_string(expected_file).unwrap();
4789 // Raw — no YAML front matter / fenced-json wrapper.
4790 assert_eq!(body, "# Plan\n\nstep one\n");
4791 }
4792
4793 /// No resolvable root + `Warn` — staging still
4794 /// succeeds (fail-open), and no part file is written.
4795 #[tokio::test]
4796 async fn stage_artifact_check_policy_warn_skips_part_file_when_root_unresolved() {
4797 let dir = tempfile::TempDir::new().unwrap();
4798 let (engine, _op, task_id, _worker_token) =
4799 seeded_task_with_policy("planner", crate::core::config::CheckPolicy::Warn).await;
4800 // No seed_agent_context — root unresolved.
4801
4802 let result = engine
4803 .stage_worker_artifact_trusted(
4804 &task_id,
4805 1,
4806 "plan.md".to_string(),
4807 serde_json::json!("x"),
4808 )
4809 .await;
4810 assert!(
4811 result.is_ok(),
4812 "Warn mode preserves fail-open: stage must succeed when root unresolved"
4813 );
4814 assert!(
4815 !dir.path().join("workspace").exists(),
4816 "no part file may be materialized when root is unresolved"
4817 );
4818 }
4819
4820 /// No resolvable root + `Strict` — staging surfaces
4821 /// the fail-open condition as an [`EngineError::CheckPolicyStrict`],
4822 /// its message identifying the "part file materialize" call site.
4823 #[tokio::test]
4824 async fn stage_artifact_check_policy_strict_surfaces_error_when_root_unresolved() {
4825 let (engine, _op, task_id, _worker_token) =
4826 seeded_task_with_policy("planner", crate::core::config::CheckPolicy::Strict).await;
4827
4828 let err = engine
4829 .stage_worker_artifact_trusted(
4830 &task_id,
4831 1,
4832 "plan.md".to_string(),
4833 serde_json::json!("x"),
4834 )
4835 .await
4836 .expect_err("Strict mode must return an error when root unresolved");
4837 match err {
4838 EngineError::CheckPolicyStrict { context, message } => {
4839 assert!(
4840 context.contains("part file materialize"),
4841 "context must identify the call site: {context}"
4842 );
4843 assert!(
4844 message.contains("part file materialize"),
4845 "message must identify the part-file sink: {message}"
4846 );
4847 assert!(
4848 message.contains("no work_dir/project_root resolved"),
4849 "message must preserve the warn-log literal: {message}"
4850 );
4851 }
4852 other => panic!(
4853 "expected EngineError::CheckPolicyStrict, got a different variant: {other:?}"
4854 ),
4855 }
4856 }
4857
4858 /// A path-traversal `name` (`../evil.md`) with a
4859 /// resolved root — the name guard fails the write, but fail-open keeps
4860 /// the stage succeeding, and nothing is written outside the ctx dir.
4861 #[tokio::test]
4862 async fn stage_artifact_traversal_name_is_fail_open_and_writes_nothing() {
4863 let dir = tempfile::TempDir::new().unwrap();
4864 let (engine, _op, task_id, _worker_token) = seeded_task("planner").await;
4865 seed_agent_context(&engine, &task_id, 1, &dir.path().to_string_lossy()).await;
4866
4867 let result = engine
4868 .stage_worker_artifact_trusted(
4869 &task_id,
4870 1,
4871 "../evil.md".to_string(),
4872 serde_json::json!("pwned"),
4873 )
4874 .await;
4875 assert!(
4876 result.is_ok(),
4877 "default (Warn) policy is fail-open even on a rejected part name"
4878 );
4879 // The escaped target (ctx dir's parent) must not have been written.
4880 let escaped = dir
4881 .path()
4882 .join("workspace/tasks")
4883 .join(task_id.as_str())
4884 .join("evil.md");
4885 assert!(
4886 !escaped.exists(),
4887 "a traversal name must never write outside the ctx dir: {escaped:?}"
4888 );
4889 }
4890}
4891
4892/// GH #36 ST1: named multi-part worker output. Covers (a) the pure
4893/// `fold_final_and_parts` assembly `dispatch_attempt_with`'s Final-pull
4894/// delegates to, (b) `stage_worker_artifact_trusted`'s per-attempt
4895/// isolation on `EngineState.output_store` / `.worker_artifact_names` (the
4896/// same `HashMap<(StepId, u32), _>` key shape `submit_worker_result_trusted`
4897/// uses — a fresh attempt is a fresh key, so nothing to explicitly "clean
4898/// up"), and (c) the allowlist behavior that keeps a non-opt-in `Artifact`
4899/// producer (e.g. `AfterRunAuditMiddleware`) from being folded in.
4900#[cfg(test)]
4901mod named_multi_part_worker_output_tests {
4902 use super::*;
4903 use crate::worker::output::{ContentRef, OutputEvent};
4904
4905 fn artifact(name: &str, value: Value) -> OutputEvent {
4906 OutputEvent::Artifact {
4907 name: name.to_string(),
4908 content: ContentRef::Inline { value },
4909 }
4910 }
4911
4912 fn final_ev(value: Value, ok: bool) -> OutputEvent {
4913 OutputEvent::Final {
4914 content: ContentRef::Inline { value },
4915 ok,
4916 }
4917 }
4918
4919 fn names(list: &[&str]) -> Vec<String> {
4920 list.iter().map(|s| s.to_string()).collect()
4921 }
4922
4923 /// Two staged parts (both in `staged_names`) + a `Final` fold into
4924 /// `{"out", "parts"}`, each value carried through verbatim.
4925 #[test]
4926 fn fold_final_and_parts_assembles_out_and_parts_shape() {
4927 let tail = vec![
4928 artifact("summary", serde_json::json!("the summary")),
4929 artifact("diff", serde_json::json!({"lines": 3})),
4930 final_ev(serde_json::json!("final text"), true),
4931 ];
4932 let staged = names(&["summary", "diff"]);
4933 let (value, ok) = fold_final_and_parts(&tail, &staged).expect("Final present");
4934 assert!(ok);
4935 assert_eq!(
4936 value,
4937 serde_json::json!({
4938 "out": "final text",
4939 "parts": {
4940 "summary": "the summary",
4941 "diff": {"lines": 3},
4942 }
4943 })
4944 );
4945 }
4946
4947 /// Zero staged parts: the value is exactly the plain `Final` value — no
4948 /// `{"out", "parts"}` wrapping. This is the back-compat guarantee (GH
4949 /// #36 must not change the shape for a worker that never POSTs to
4950 /// `/v1/worker/artifact`).
4951 #[test]
4952 fn fold_final_and_parts_with_no_parts_returns_plain_final_value() {
4953 let tail = vec![final_ev(serde_json::json!("plain value"), true)];
4954 let (value, ok) = fold_final_and_parts(&tail, &[]).expect("Final present");
4955 assert!(ok);
4956 assert_eq!(value, serde_json::json!("plain value"));
4957 }
4958
4959 /// The same staged part `name` appearing twice in one attempt: the
4960 /// LATER (tail-order) value wins — `parts` is a `Map`, not an
4961 /// accumulating list.
4962 #[test]
4963 fn fold_final_and_parts_same_name_twice_last_write_wins() {
4964 let tail = vec![
4965 artifact("a", serde_json::json!("first")),
4966 artifact("a", serde_json::json!("second")),
4967 final_ev(serde_json::json!("f"), true),
4968 ];
4969 let staged = names(&["a"]);
4970 let (value, _ok) = fold_final_and_parts(&tail, &staged).expect("Final present");
4971 assert_eq!(
4972 value,
4973 serde_json::json!({"out": "f", "parts": {"a": "second"}})
4974 );
4975 }
4976
4977 /// No `Final` anywhere in the tail (only staged parts, e.g. the worker
4978 /// crashed before submitting) — `None`, the caller's pre-existing "no
4979 /// Final in output_tail" error path.
4980 #[test]
4981 fn fold_final_and_parts_returns_none_when_no_final_present() {
4982 let tail = vec![artifact("a", serde_json::json!("v"))];
4983 let staged = names(&["a"]);
4984 assert!(fold_final_and_parts(&tail, &staged).is_none());
4985 }
4986
4987 /// An `Artifact` on the tail whose name is NOT in `staged_names` (e.g.
4988 /// `AfterRunAuditMiddleware`'s `"audit:<step_ref>"` sidecar finding on
4989 /// an audited step's own tail) must NOT be folded into `"parts"` — the
4990 /// value stays the plain `Final` value, exactly the pre-GH-#36
4991 /// behavior for every producer that isn't the worker's own
4992 /// `/v1/worker/artifact` staging. This is the regression this fold was
4993 /// almost shipped without (see `dispatch_attempt_with`'s doc).
4994 #[test]
4995 fn fold_final_and_parts_ignores_artifacts_outside_the_staged_allowlist() {
4996 let tail = vec![
4997 final_ev(serde_json::json!({"echoed": "hi"}), true),
4998 artifact("audit:echo", serde_json::json!({"finding": "clean"})),
4999 ];
5000 // `staged_names` empty: the worker itself never staged anything —
5001 // the audit sidecar Artifact must be ignored.
5002 let (value, ok) = fold_final_and_parts(&tail, &[]).expect("Final present");
5003 assert!(ok);
5004 assert_eq!(value, serde_json::json!({"echoed": "hi"}));
5005 }
5006
5007 /// Mixed tail: one staged (allowlisted) part and one non-staged
5008 /// (audit-style) `Artifact` — only the staged one is folded in.
5009 #[test]
5010 fn fold_final_and_parts_folds_only_the_staged_subset_of_a_mixed_tail() {
5011 let tail = vec![
5012 artifact("summary", serde_json::json!("s")),
5013 artifact("audit:echo", serde_json::json!({"finding": "clean"})),
5014 final_ev(serde_json::json!("f"), true),
5015 ];
5016 let staged = names(&["summary"]);
5017 let (value, _ok) = fold_final_and_parts(&tail, &staged).expect("Final present");
5018 assert_eq!(
5019 value,
5020 serde_json::json!({"out": "f", "parts": {"summary": "s"}})
5021 );
5022 }
5023
5024 /// `stage_worker_artifact_trusted` writes onto the `(task_id, attempt)`
5025 /// key exactly like `submit_worker_result_trusted` does — a part staged
5026 /// under attempt N is invisible to an `output_tail` / allowlist read of
5027 /// attempt N+1 (a fresh attempt starts empty; nothing carries over).
5028 #[tokio::test]
5029 async fn stage_worker_artifact_trusted_is_isolated_per_attempt() {
5030 let engine = Engine::new(EngineCfg::default());
5031 let task_id = StepId::new();
5032
5033 engine
5034 .stage_worker_artifact_trusted(&task_id, 1, "a".to_string(), serde_json::json!("v1"))
5035 .await
5036 .expect("stage attempt 1");
5037
5038 let attempt_1_tail = engine.output_tail(&task_id, 1).await;
5039 assert_eq!(attempt_1_tail.len(), 1);
5040 assert!(matches!(
5041 &attempt_1_tail[0],
5042 OutputEvent::Artifact { name, .. } if name == "a"
5043 ));
5044 assert_eq!(
5045 engine.worker_artifact_names_for(&task_id, 1).await,
5046 vec!["a".to_string()]
5047 );
5048
5049 let attempt_2_tail = engine.output_tail(&task_id, 2).await;
5050 assert!(
5051 attempt_2_tail.is_empty(),
5052 "attempt 2 must not see attempt 1's staged part"
5053 );
5054 assert!(
5055 engine
5056 .worker_artifact_names_for(&task_id, 2)
5057 .await
5058 .is_empty(),
5059 "attempt 2's allowlist must not see attempt 1's staged name"
5060 );
5061 }
5062}
5063
5064// ─── GH #50 (Subtask 2): `Engine::register_verdict_contracts` /
5065// `Engine::verdict_contract_for_task` ────────────────────────────────────
5066#[cfg(test)]
5067mod verdict_contract_registry_tests {
5068 use super::*;
5069
5070 async fn seeded_engine(agent: &str) -> (Engine, StepId) {
5071 let engine = Engine::new(EngineCfg::default());
5072 let op_token = engine
5073 .attach("ut-op", Role::Operator, Duration::from_secs(30))
5074 .await
5075 .expect("attach");
5076 let task_id = engine
5077 .start_task(
5078 &op_token,
5079 TaskSpec {
5080 agent: agent.to_string(),
5081 initial_directive: serde_json::json!("x"),
5082 step_ctx: None,
5083 check_policy: None,
5084 },
5085 )
5086 .await
5087 .expect("start_task");
5088 (engine, task_id)
5089 }
5090
5091 /// An agent with no registered contract at all → `None` (the opt-in
5092 /// default; every pre-GH-#50 `Engine`).
5093 #[tokio::test]
5094 async fn returns_none_when_no_contract_registered_for_the_agent() {
5095 let (engine, task_id) = seeded_engine("gate").await;
5096 assert_eq!(engine.verdict_contract_for_task(&task_id).await, None);
5097 }
5098
5099 /// A registered contract for the running task's agent is returned
5100 /// verbatim.
5101 #[tokio::test]
5102 async fn returns_the_registered_contract_for_the_running_agent() {
5103 let (engine, task_id) = seeded_engine("gate").await;
5104 let contract = mlua_swarm_schema::VerdictContract {
5105 channel: mlua_swarm_schema::VerdictChannel::Body,
5106 values: vec!["PASS".to_string(), "BLOCKED".to_string()],
5107 };
5108 engine.register_verdict_contracts(HashMap::from([("gate".to_string(), contract.clone())]));
5109 assert_eq!(
5110 engine.verdict_contract_for_task(&task_id).await,
5111 Some(contract)
5112 );
5113 }
5114
5115 /// A registered contract for a DIFFERENT agent name never leaks onto
5116 /// an unrelated task.
5117 #[tokio::test]
5118 async fn does_not_leak_a_contract_registered_for_a_different_agent() {
5119 let (engine, task_id) = seeded_engine("gate").await;
5120 engine.register_verdict_contracts(HashMap::from([(
5121 "other-agent".to_string(),
5122 mlua_swarm_schema::VerdictContract {
5123 channel: mlua_swarm_schema::VerdictChannel::Body,
5124 values: vec!["PASS".to_string()],
5125 },
5126 )]));
5127 assert_eq!(engine.verdict_contract_for_task(&task_id).await, None);
5128 }
5129
5130 /// An unknown `task_id` → `None`, not a panic / error.
5131 #[tokio::test]
5132 async fn returns_none_for_an_unknown_task_id() {
5133 let engine = Engine::new(EngineCfg::default());
5134 let unknown = StepId::new();
5135 assert_eq!(engine.verdict_contract_for_task(&unknown).await, None);
5136 }
5137
5138 /// `register_verdict_contracts` is additive (`HashMap::extend`): a
5139 /// second call registering a DIFFERENT agent does not clobber the
5140 /// first call's entry.
5141 #[tokio::test]
5142 async fn register_verdict_contracts_is_additive_across_calls() {
5143 let (engine, task_id) = seeded_engine("gate").await;
5144 let contract = mlua_swarm_schema::VerdictContract {
5145 channel: mlua_swarm_schema::VerdictChannel::Part,
5146 values: vec!["ALLOW".to_string()],
5147 };
5148 engine.register_verdict_contracts(HashMap::from([("gate".to_string(), contract.clone())]));
5149 engine.register_verdict_contracts(HashMap::from([(
5150 "unrelated-agent".to_string(),
5151 mlua_swarm_schema::VerdictContract {
5152 channel: mlua_swarm_schema::VerdictChannel::Body,
5153 values: vec!["X".to_string()],
5154 },
5155 )]));
5156 assert_eq!(
5157 engine.verdict_contract_for_task(&task_id).await,
5158 Some(contract)
5159 );
5160 }
5161}
5162
5163// ─── GH #51: completion-time verdict-contract enforcement — the shared
5164// `Engine::verdict_contract_completion_check` choke point embedded inside
5165// `submit_worker_result_trusted` / `submit_output`, exercised here at the
5166// `submit_output` level (the WS Operator fallback route's own unit-test
5167// coverage — see `crates/mlua-swarm-server/tests/verdict_contract.rs` for
5168// the HTTP-round-trip coverage of the other 2 routes) ───────────────────
5169#[cfg(test)]
5170mod verdict_contract_completion_tests {
5171 use super::*;
5172
5173 /// Seeds a `Pending` task bound to `agent` and mints a bound
5174 /// `Role::Worker` token for it — the same mint-and-register pattern
5175 /// `initial_directive_value_passthrough_tests::mint_worker_token`
5176 /// uses (duplicated here: that helper is private to its own sibling
5177 /// `#[cfg(test)]` module, not reachable via `super::*` from this one).
5178 async fn seeded_task_with_worker_token(agent: &str) -> (Engine, CapToken, StepId) {
5179 let engine = Engine::new(EngineCfg::default());
5180 let op_token = engine
5181 .attach("ut-op", Role::Operator, Duration::from_secs(30))
5182 .await
5183 .expect("attach");
5184 let task_id = engine
5185 .start_task(
5186 &op_token,
5187 TaskSpec {
5188 agent: agent.to_string(),
5189 initial_directive: serde_json::json!("x"),
5190 step_ctx: None,
5191 check_policy: None,
5192 },
5193 )
5194 .await
5195 .expect("start_task");
5196 let worker_token = engine.signer().session(
5197 format!("worker-of-{task_id}"),
5198 Role::Worker,
5199 vec!["*".into()],
5200 Duration::from_secs(600),
5201 );
5202 let fp = worker_token.fingerprint();
5203 let record = CapTokenRecord::from_worker_token(worker_token.clone(), task_id.clone());
5204 engine
5205 .with_state("test.mint_worker", move |s| {
5206 s.tokens.insert(fp, record);
5207 })
5208 .await
5209 .expect("mint worker token");
5210 (engine, worker_token, task_id)
5211 }
5212
5213 fn body_contract(values: &[&str]) -> mlua_swarm_schema::VerdictContract {
5214 mlua_swarm_schema::VerdictContract {
5215 channel: mlua_swarm_schema::VerdictChannel::Body,
5216 values: values.iter().map(|v| v.to_string()).collect(),
5217 }
5218 }
5219
5220 fn part_contract(values: &[&str]) -> mlua_swarm_schema::VerdictContract {
5221 mlua_swarm_schema::VerdictContract {
5222 channel: mlua_swarm_schema::VerdictChannel::Part,
5223 values: values.iter().map(|v| v.to_string()).collect(),
5224 }
5225 }
5226
5227 fn final_event(value: Value, ok: bool) -> crate::worker::output::OutputEvent {
5228 crate::worker::output::OutputEvent::Final {
5229 content: crate::worker::output::ContentRef::Inline { value },
5230 ok,
5231 }
5232 }
5233
5234 /// Route 3 (WS Operator fallback, `submit_output` level) — a
5235 /// `channel: "part"` contract's attempt completes via a plain
5236 /// `Final` without ever staging a `"verdict"` artifact: rejected
5237 /// with `EngineError::VerdictPartMissing`, and nothing lands on
5238 /// `output_tail` — the rejected value never reaches the flow ctx.
5239 #[tokio::test]
5240 async fn submit_output_rejects_missing_verdict_part() {
5241 let (engine, token, task_id) = seeded_task_with_worker_token("gate").await;
5242 engine.register_verdict_contracts(HashMap::from([(
5243 "gate".to_string(),
5244 part_contract(&["PASS", "BLOCKED"]),
5245 )]));
5246
5247 let err = engine
5248 .submit_output(
5249 &token,
5250 &task_id,
5251 1,
5252 final_event(serde_json::json!("anything"), true),
5253 )
5254 .await
5255 .expect_err("missing staged verdict part must be rejected");
5256 assert!(
5257 matches!(err, EngineError::VerdictPartMissing { .. }),
5258 "unexpected error variant: {err:?}"
5259 );
5260
5261 let tail = engine.output_tail(&task_id, 1).await;
5262 assert!(
5263 !tail
5264 .iter()
5265 .any(|ev| matches!(ev, crate::worker::output::OutputEvent::Final { .. })),
5266 "a rejected completion must not write a Final onto output_tail"
5267 );
5268 }
5269
5270 /// Route 3 — a `channel: "part"` contract completes normally when the
5271 /// worker DID stage a matching `"verdict"` artifact first (defense in
5272 /// depth: presence AND membership both hold).
5273 #[tokio::test]
5274 async fn submit_output_accepts_when_verdict_part_is_staged_and_a_member() {
5275 let (engine, token, task_id) = seeded_task_with_worker_token("gate").await;
5276 engine.register_verdict_contracts(HashMap::from([(
5277 "gate".to_string(),
5278 part_contract(&["PASS", "BLOCKED"]),
5279 )]));
5280 engine
5281 .stage_worker_artifact_trusted(
5282 &task_id,
5283 1,
5284 "verdict".to_string(),
5285 serde_json::json!("PASS"),
5286 )
5287 .await
5288 .expect("stage verdict part");
5289
5290 engine
5291 .submit_output(
5292 &token,
5293 &task_id,
5294 1,
5295 final_event(serde_json::json!("full report"), true),
5296 )
5297 .await
5298 .expect("staged + member verdict part must be accepted");
5299
5300 let tail = engine.output_tail(&task_id, 1).await;
5301 assert!(
5302 tail.iter()
5303 .any(|ev| matches!(ev, crate::worker::output::OutputEvent::Final { .. })),
5304 "an accepted completion must write its Final onto output_tail"
5305 );
5306 }
5307
5308 /// Route 3 — a `channel: "body"` contract's completing value is NOT a
5309 /// member of `values`: rejected with
5310 /// `EngineError::VerdictValueRejected`, no `Final` written.
5311 #[tokio::test]
5312 async fn submit_output_rejects_body_value_outside_contract() {
5313 let (engine, token, task_id) = seeded_task_with_worker_token("gate").await;
5314 engine.register_verdict_contracts(HashMap::from([(
5315 "gate".to_string(),
5316 body_contract(&["PASS", "BLOCKED"]),
5317 )]));
5318
5319 let err = engine
5320 .submit_output(
5321 &token,
5322 &task_id,
5323 1,
5324 final_event(serde_json::json!("UNKNOWN"), true),
5325 )
5326 .await
5327 .expect_err("out-of-contract body value must be rejected");
5328 match err {
5329 EngineError::VerdictValueRejected { value, allowed } => {
5330 assert_eq!(value, "UNKNOWN");
5331 assert_eq!(allowed, vec!["PASS".to_string(), "BLOCKED".to_string()]);
5332 }
5333 other => panic!("unexpected error variant: {other:?}"),
5334 }
5335
5336 let tail = engine.output_tail(&task_id, 1).await;
5337 assert!(
5338 !tail
5339 .iter()
5340 .any(|ev| matches!(ev, crate::worker::output::OutputEvent::Final { .. })),
5341 "a rejected completion must not write a Final onto output_tail"
5342 );
5343 }
5344
5345 /// `ok=false` bypasses the completion-time check entirely, regardless
5346 /// of channel or membership — the exemption acceptance criterion,
5347 /// exercised at the `submit_output` choke point.
5348 #[tokio::test]
5349 async fn submit_output_ok_false_bypasses_the_check() {
5350 let (engine, token, task_id) = seeded_task_with_worker_token("gate").await;
5351 engine.register_verdict_contracts(HashMap::from([(
5352 "gate".to_string(),
5353 body_contract(&["PASS", "BLOCKED"]),
5354 )]));
5355
5356 engine
5357 .submit_output(
5358 &token,
5359 &task_id,
5360 1,
5361 final_event(serde_json::json!("UNKNOWN"), false),
5362 )
5363 .await
5364 .expect("ok=false must bypass the verdict contract check entirely");
5365
5366 let tail = engine.output_tail(&task_id, 1).await;
5367 assert!(
5368 tail.iter()
5369 .any(|ev| matches!(ev, crate::worker::output::OutputEvent::Final { .. })),
5370 "an ok=false completion is exempt, not rejected — its Final must still land"
5371 );
5372 }
5373
5374 /// `staged_verdict_value_for` mirrors `fold_final_and_parts`'s
5375 /// last-write-wins semantics: staging `"verdict"` twice within the
5376 /// same attempt returns the LAST value, not the first.
5377 #[tokio::test]
5378 async fn staged_verdict_value_for_is_last_write_wins() {
5379 let (engine, _token, task_id) = seeded_task_with_worker_token("gate").await;
5380 engine
5381 .stage_worker_artifact_trusted(
5382 &task_id,
5383 1,
5384 "verdict".to_string(),
5385 serde_json::json!("PASS"),
5386 )
5387 .await
5388 .expect("stage first verdict part");
5389 engine
5390 .stage_worker_artifact_trusted(
5391 &task_id,
5392 1,
5393 "verdict".to_string(),
5394 serde_json::json!("BLOCKED"),
5395 )
5396 .await
5397 .expect("stage second verdict part");
5398
5399 assert_eq!(
5400 engine.staged_verdict_value_for(&task_id, 1).await,
5401 Some("BLOCKED".to_string())
5402 );
5403 }
5404
5405 /// `staged_verdict_value_for` ignores artifacts staged under any name
5406 /// OTHER than the literal `"verdict"` — mirrors `channel: "part"`
5407 /// contracts only ever addressing that one part.
5408 #[tokio::test]
5409 async fn staged_verdict_value_for_ignores_other_artifact_names() {
5410 let (engine, _token, task_id) = seeded_task_with_worker_token("gate").await;
5411 engine
5412 .stage_worker_artifact_trusted(
5413 &task_id,
5414 1,
5415 "notes".to_string(),
5416 serde_json::json!("irrelevant"),
5417 )
5418 .await
5419 .expect("stage unrelated part");
5420
5421 assert_eq!(engine.staged_verdict_value_for(&task_id, 1).await, None);
5422 }
5423
5424 /// `staged_verdict_value_for` → `None` when nothing was ever staged —
5425 /// the normal case the completion check turns into
5426 /// `EngineError::VerdictPartMissing`.
5427 #[tokio::test]
5428 async fn staged_verdict_value_for_returns_none_when_nothing_staged() {
5429 let (engine, _token, task_id) = seeded_task_with_worker_token("gate").await;
5430 assert_eq!(engine.staged_verdict_value_for(&task_id, 1).await, None);
5431 }
5432}
5433
5434// ─── GH #76 Skip tier: DispatchOutcome::Skip tier + SubmitOutcome API ────────────
5435#[cfg(test)]
5436mod skip_tier_tests {
5437 use super::*;
5438 use crate::blueprint::compiler::{RustFnInProcessSpawnerFactory, SpawnerFactory};
5439 use crate::blueprint::EngineDispatcher;
5440 use crate::core::state::{
5441 is_skip_marker, unwrap_skip_marker, wrap_skip_marker, SubmitOutcome, SKIP_MARKER_KEY,
5442 };
5443 use crate::store::run::{InMemoryRunStore, RunContext, RunRecord, RunStatus, RunStore};
5444 use crate::types::{RunId, TaskId};
5445 use crate::worker::adapter::WorkerResult;
5446 use mlua_flow_ir::AsyncDispatcher;
5447 use mlua_swarm_schema::{AgentDef, AgentKind};
5448 use serde_json::json;
5449
5450 /// `DispatchOutcome::Skip(v)` roundtrips through serde JSON without
5451 /// loss — the enum is serialized with the default externally-tagged
5452 /// form (same as `Pass`/`Blocked`), so no `#[serde(...)]` tuning is
5453 /// needed for the new variant.
5454 #[test]
5455 fn dispatch_outcome_skip_variant_serializes_roundtrip() {
5456 let outcome = DispatchOutcome::Skip(json!({ "verdict": "SKIP", "reason": "n/a" }));
5457 let serialized = serde_json::to_string(&outcome).expect("serialize");
5458 let round: DispatchOutcome = serde_json::from_str(&serialized).expect("deserialize");
5459 match round {
5460 DispatchOutcome::Skip(v) => {
5461 assert_eq!(v, json!({ "verdict": "SKIP", "reason": "n/a" }));
5462 }
5463 other => panic!("expected Skip after roundtrip, got {other:?}"),
5464 }
5465 }
5466
5467 /// The `is_skip_marker` / `unwrap_skip_marker` / `wrap_skip_marker`
5468 /// helper triangle round-trips consistently and rejects plain
5469 /// payloads. Pinning the reserved-key contract in a unit test guards
5470 /// against a future edit accidentally renaming the sentinel key
5471 /// (which would silently break every downstream reader).
5472 #[test]
5473 fn skip_marker_helpers_wrap_detect_and_unwrap() {
5474 assert!(!is_skip_marker(&json!("plain string")));
5475 assert!(!is_skip_marker(&json!({ "verdict": "PASS" })));
5476 assert!(!is_skip_marker(&json!(null)));
5477
5478 let inner = json!({ "reason": "not applicable" });
5479 let wrapped = wrap_skip_marker(inner.clone());
5480 assert!(is_skip_marker(&wrapped));
5481 assert_eq!(wrapped[SKIP_MARKER_KEY], json!(true));
5482 assert_eq!(unwrap_skip_marker(&wrapped), Some(inner));
5483
5484 // A malformed sentinel (marker key present but `value` absent) is
5485 // still a Skip signal, defaulting the carried payload to Null so
5486 // downstream match arms never observe `None` on a marker match.
5487 let malformed = json!({ SKIP_MARKER_KEY: true });
5488 assert!(is_skip_marker(&malformed));
5489 assert_eq!(unwrap_skip_marker(&malformed), Some(Value::Null));
5490
5491 // Plain payloads → `unwrap_skip_marker` returns `None` (the
5492 // caller falls back to the ordinary Pass/Blocked path).
5493 assert_eq!(unwrap_skip_marker(&json!("plain")), None);
5494 }
5495
5496 /// The `SubmitOutcome::Skip` mapping wraps the payload in the
5497 /// skip-marker sentinel AND records `Final.ok = true` — matching the
5498 /// invariant in the outcome mapping table in
5499 /// `submit_worker_result_trusted`'s doc. This is the wire shape
5500 /// `dispatch_attempt_with*` reads back to route into
5501 /// `DispatchOutcome::Skip`.
5502 #[tokio::test]
5503 async fn submit_worker_result_trusted_skip_outcome_records_final_ok_true_with_sentinel() {
5504 use crate::worker::output::OutputEvent;
5505 let engine = Engine::new(EngineCfg::default());
5506 let op_token = engine
5507 .attach("ut-op", Role::Operator, Duration::from_secs(30))
5508 .await
5509 .expect("attach");
5510 let task_id = engine
5511 .start_task(
5512 &op_token,
5513 TaskSpec {
5514 agent: "analyst".into(),
5515 initial_directive: json!("go"),
5516 step_ctx: None,
5517 check_policy: None,
5518 },
5519 )
5520 .await
5521 .expect("start_task");
5522
5523 let inner_verdict = json!({ "verdict": "SKIP", "reason": "migration=no" });
5524 engine
5525 .submit_worker_result_trusted(&task_id, 1, inner_verdict.clone(), SubmitOutcome::Skip)
5526 .await
5527 .expect("submit with Skip outcome");
5528
5529 let tail = engine.output_tail(&task_id, 1).await;
5530 let final_ev = tail
5531 .iter()
5532 .rev()
5533 .find_map(|ev| match ev {
5534 OutputEvent::Final { content, ok } => Some((content.clone(), *ok)),
5535 _ => None,
5536 })
5537 .expect("Final present after Skip submit");
5538 assert!(
5539 final_ev.1,
5540 "Skip records Final.ok = true (flow-continuation)"
5541 );
5542 let stored_value = super::content_ref_to_value(final_ev.0);
5543 assert!(
5544 is_skip_marker(&stored_value),
5545 "Skip wraps the payload in the sentinel: got {stored_value}"
5546 );
5547 assert_eq!(unwrap_skip_marker(&stored_value), Some(inner_verdict));
5548 }
5549
5550 /// The new `SubmitOutcome::Pass` / `SubmitOutcome::Blocked` arms
5551 /// preserve byte-for-byte the pre-#76 wire shape (Final.ok mirrors
5552 /// the tier; the value is not wrapped). Regression against a future
5553 /// edit that accidentally routes Pass/Blocked through the Skip
5554 /// wrapper.
5555 #[tokio::test]
5556 async fn submit_worker_result_trusted_pass_and_blocked_wire_unchanged() {
5557 use crate::worker::output::OutputEvent;
5558 let engine = Engine::new(EngineCfg::default());
5559 let op_token = engine
5560 .attach("ut-op", Role::Operator, Duration::from_secs(30))
5561 .await
5562 .expect("attach");
5563
5564 // Pass path.
5565 let pass_task = engine
5566 .start_task(
5567 &op_token,
5568 TaskSpec {
5569 agent: "worker".into(),
5570 initial_directive: json!("go"),
5571 step_ctx: None,
5572 check_policy: None,
5573 },
5574 )
5575 .await
5576 .expect("start_task pass");
5577 engine
5578 .submit_worker_result_trusted(&pass_task, 1, json!("pass-value"), SubmitOutcome::Pass)
5579 .await
5580 .expect("submit Pass");
5581 let pass_tail = engine.output_tail(&pass_task, 1).await;
5582 let (pass_content, pass_ok) = pass_tail
5583 .iter()
5584 .rev()
5585 .find_map(|ev| match ev {
5586 OutputEvent::Final { content, ok } => Some((content.clone(), *ok)),
5587 _ => None,
5588 })
5589 .expect("Final present");
5590 assert!(pass_ok);
5591 assert_eq!(
5592 super::content_ref_to_value(pass_content),
5593 json!("pass-value"),
5594 "Pass value must not be wrapped"
5595 );
5596
5597 // Blocked path.
5598 let blocked_task = engine
5599 .start_task(
5600 &op_token,
5601 TaskSpec {
5602 agent: "worker".into(),
5603 initial_directive: json!("go"),
5604 step_ctx: None,
5605 check_policy: None,
5606 },
5607 )
5608 .await
5609 .expect("start_task blocked");
5610 engine
5611 .submit_worker_result_trusted(
5612 &blocked_task,
5613 1,
5614 json!("blocked-value"),
5615 SubmitOutcome::Blocked,
5616 )
5617 .await
5618 .expect("submit Blocked");
5619 let blocked_tail = engine.output_tail(&blocked_task, 1).await;
5620 let (blocked_content, blocked_ok) = blocked_tail
5621 .iter()
5622 .rev()
5623 .find_map(|ev| match ev {
5624 OutputEvent::Final { content, ok } => Some((content.clone(), *ok)),
5625 _ => None,
5626 })
5627 .expect("Final present");
5628 assert!(!blocked_ok);
5629 assert_eq!(
5630 super::content_ref_to_value(blocked_content),
5631 json!("blocked-value"),
5632 "Blocked value must not be wrapped"
5633 );
5634 }
5635
5636 /// End-to-end (engine layer): a worker that returns a skip-marker
5637 /// sentinel value via `WorkerResult { value: wrap_skip_marker(inner),
5638 /// ok: true }` — which is what a Skip-aware caller of
5639 /// `submit_worker_result_trusted(..., SubmitOutcome::Skip)` places on
5640 /// the wire — is folded by `dispatch_attempt_with_run_ctx` into
5641 /// `DispatchOutcome::Skip(inner)`. Proves the sentinel → outcome
5642 /// routing that the flow-ir binding boundary depends on.
5643 #[tokio::test]
5644 async fn dispatcher_folds_skip_sentinel_into_skip_outcome() {
5645 let inner_verdict = json!({ "verdict": "SKIP", "reason": "not applicable" });
5646 let inner_for_worker = inner_verdict.clone();
5647 let factory = RustFnInProcessSpawnerFactory::new().register_fn("analyst", move |_inv| {
5648 let value = wrap_skip_marker(inner_for_worker.clone());
5649 async move { Ok(WorkerResult { value, ok: true }) }
5650 });
5651 let def = AgentDef {
5652 name: "analyst".into(),
5653 kind: AgentKind::RustFn,
5654 spec: json!({ "fn_id": "analyst" }),
5655 profile: None,
5656 meta: None,
5657 runner: None,
5658 runner_ref: None,
5659 verdict: None,
5660 };
5661 let spawner = factory.build(&def, None).expect("build");
5662
5663 let engine = Engine::new(EngineCfg::default());
5664 let op_token = engine
5665 .attach("ut-op", Role::Operator, Duration::from_secs(30))
5666 .await
5667 .expect("attach");
5668 let task_id = engine
5669 .start_task(
5670 &op_token,
5671 TaskSpec {
5672 agent: "analyst".into(),
5673 initial_directive: json!("go"),
5674 step_ctx: None,
5675 check_policy: None,
5676 },
5677 )
5678 .await
5679 .expect("start_task");
5680
5681 let outcome = engine
5682 .dispatch_attempt_with_run_ctx(&op_token, &task_id, &spawner, None)
5683 .await
5684 .expect("dispatch ok");
5685
5686 match outcome {
5687 DispatchOutcome::Skip(v) => {
5688 assert_eq!(v, inner_verdict, "Skip carries the unwrapped inner verdict");
5689 }
5690 other => panic!("expected DispatchOutcome::Skip, got {other:?}"),
5691 }
5692 }
5693
5694 /// `EngineDispatcher::dispatch` (the `AsyncDispatcher` impl flow-ir
5695 /// invokes) maps `DispatchOutcome::Skip(v)` to `Ok(wrap_skip_marker(v))`
5696 /// — a successful return whose Value carries the sentinel across the
5697 /// flow-ir boundary. Pinning this mapping in a test guards the arm
5698 /// order (a wildcard `Ok(other) =>` arm accidentally placed BEFORE the
5699 /// Skip arm would route Skip to `EvalError::DispatcherError` and
5700 /// abort the flow — the exact failure mode this tier prevents).
5701 #[tokio::test]
5702 async fn engine_dispatcher_maps_skip_outcome_to_ok_sentinel_value() {
5703 let inner_verdict = json!({ "verdict": "SKIP", "reason": "not applicable" });
5704 let inner_for_worker = inner_verdict.clone();
5705 let factory = RustFnInProcessSpawnerFactory::new().register_fn("analyst", move |_inv| {
5706 let value = wrap_skip_marker(inner_for_worker.clone());
5707 async move { Ok(WorkerResult { value, ok: true }) }
5708 });
5709 let def = AgentDef {
5710 name: "analyst".into(),
5711 kind: AgentKind::RustFn,
5712 spec: json!({ "fn_id": "analyst" }),
5713 profile: None,
5714 meta: None,
5715 runner: None,
5716 runner_ref: None,
5717 verdict: None,
5718 };
5719 let spawner = factory.build(&def, None).expect("build");
5720
5721 let engine = Engine::new(EngineCfg::default());
5722 let op_token = engine
5723 .attach("ut-op", Role::Operator, Duration::from_secs(30))
5724 .await
5725 .expect("attach");
5726 let dispatcher = EngineDispatcher::with_spawner(engine.clone(), op_token, spawner);
5727
5728 let out = dispatcher
5729 .dispatch("analyst", json!("go"))
5730 .await
5731 .expect("dispatch returns Ok for Skip tier (not EvalError::DispatcherError)");
5732
5733 assert!(
5734 is_skip_marker(&out),
5735 "returned value must carry the skip-marker sentinel across the flow-ir boundary: got {out}"
5736 );
5737 assert_eq!(unwrap_skip_marker(&out), Some(inner_verdict));
5738 }
5739
5740 /// `EngineDispatcher::dispatch`'s `RunContext` step-entry log records
5741 /// `status = "skipped"` for a Skip completion (distinct from
5742 /// `"passed"` / `"blocked"`), so post-run inspection of
5743 /// `RunRecord.step_entries` can distinguish flow-continuation-with-
5744 /// binding-write from flow-continuation-without-binding-write.
5745 #[tokio::test]
5746 async fn engine_dispatcher_step_entry_status_is_skipped_for_skip_outcome() {
5747 let inner_verdict = json!({ "verdict": "SKIP" });
5748 let inner_for_worker = inner_verdict.clone();
5749 let factory = RustFnInProcessSpawnerFactory::new().register_fn("analyst", move |_inv| {
5750 let value = wrap_skip_marker(inner_for_worker.clone());
5751 async move { Ok(WorkerResult { value, ok: true }) }
5752 });
5753 let def = AgentDef {
5754 name: "analyst".into(),
5755 kind: AgentKind::RustFn,
5756 spec: json!({ "fn_id": "analyst" }),
5757 profile: None,
5758 meta: None,
5759 runner: None,
5760 runner_ref: None,
5761 verdict: None,
5762 };
5763 let spawner = factory.build(&def, None).expect("build");
5764
5765 let engine = Engine::new(EngineCfg::default());
5766 let op_token = engine
5767 .attach("ut-op", Role::Operator, Duration::from_secs(30))
5768 .await
5769 .expect("attach");
5770
5771 // Seed a RunContext with an InMemoryRunStore so the dispatcher
5772 // appends a step_entry we can then read back.
5773 let run_id = RunId::new();
5774 let run_store: Arc<dyn RunStore> = Arc::new(InMemoryRunStore::new());
5775 run_store
5776 .create(RunRecord {
5777 id: run_id.clone(),
5778 task_id: TaskId::new(),
5779 status: RunStatus::Running,
5780 step_entries: Vec::new(),
5781 degradations: Vec::new(),
5782 operator_sid: None,
5783 result_ref: None,
5784 input_json: None,
5785 created_at: 0,
5786 updated_at: 0,
5787 })
5788 .await
5789 .expect("create run record");
5790 let run_ctx = RunContext::new(run_id.clone(), run_store.clone());
5791
5792 let dispatcher =
5793 EngineDispatcher::with_spawner(engine.clone(), op_token, spawner).with_run(run_ctx);
5794
5795 let out = dispatcher
5796 .dispatch("analyst", json!("go"))
5797 .await
5798 .expect("dispatch ok");
5799 assert!(is_skip_marker(&out));
5800
5801 let record = run_store.get(&run_id).await.expect("run record present");
5802 let step = record
5803 .step_entries
5804 .first()
5805 .expect("at least one step_entry appended for the dispatched step");
5806 assert_eq!(
5807 step.status.as_deref(),
5808 Some("skipped"),
5809 "Skip outcome must record StepEntry.status = \"skipped\""
5810 );
5811 assert_eq!(step.step_ref.as_deref(), Some("analyst"));
5812 }
5813}