mlua_swarm/core/engine.rs
1//! `Engine` — the long-running stateful runtime plus the `with_state`
2//! helper (R1-R4 discipline).
3//!
4//! The engine owns the Domain side of the Data / Domain split:
5//! flow control (dispatch / verdict), state (`EngineState`), and the
6//! `submit_output` / `output_tail` surface that feeds it. Data-plane
7//! traffic (Big Response bodies) is delegated to the `output_store` module
8//! plus its paired `SpawnerLayer`s and passes through here without the
9//! engine core needing to grow.
10
11use crate::core::agent_context::{RUN_ID_KEY, STEP_CTX_KEY};
12use crate::core::config::EngineCfg;
13use crate::core::ctx::{Ctx, OperatorInfo, OperatorKind, SeniorBridge, SpawnHook};
14use crate::core::errors::EngineError;
15use crate::core::state::{
16 unwrap_skip_marker, wrap_skip_marker, CapTokenRecord, DispatchOutcome, EngineState, Event,
17 EventStream, OperatorSession, ResumeKey, ResumePending, SubmitOutcome, TaskSpec, TaskState,
18 TaskStatus,
19};
20use crate::store::replay::{hash_input_value, ReplayEntry};
21use crate::store::run::RunContext;
22use crate::types::{
23 default_role_verb_table, now_unix, CapToken, Role, RoleVerbGate, RunId, SessionId, StepId,
24 TokenSigner, Verb,
25};
26use crate::worker::adapter::SpawnerAdapter;
27use serde_json::Value;
28use std::collections::HashMap;
29use std::sync::Arc;
30use std::time::{Duration, Instant};
31use tokio::sync::{broadcast, Mutex};
32
33/// Process-wide long-running runtime. Cheap to `clone()` — an `Arc`
34/// lives inside.
35#[derive(Clone)]
36pub struct Engine {
37 inner: Arc<EngineInner>,
38}
39
40struct EngineInner {
41 state: Mutex<EngineState>,
42 cfg: EngineCfg,
43 signer: TokenSigner,
44 gate: RoleVerbGate,
45 event_tx: broadcast::Sender<Event>,
46 /// ID-keyed bridge registry (register-by-ID design). `SeniorBridge`
47 /// and `SpawnHook` are registered by ID; sessions bind to those IDs
48 /// only. Persistence stores just the ID, and on reattach the caller
49 /// re-registers under the same ID to restore presence.
50 senior_bridges: tokio::sync::RwLock<HashMap<String, Arc<dyn SeniorBridge>>>,
51 spawn_hooks: tokio::sync::RwLock<HashMap<String, Arc<dyn SpawnHook>>>,
52 /// ID registry for full-spawn Operator backends (backends that take the
53 /// entire spawn via `execute`). Sibling to `senior_bridges` /
54 /// `spawn_hooks`. `OperatorDelegateMiddleware` looks these up via
55 /// `ctx` and, when `kind = MainAi` / `Composite`, bypasses
56 /// `inner.spawn` and calls `operator.execute` instead.
57 operators: tokio::sync::RwLock<HashMap<String, Arc<dyn crate::operator::Operator>>>,
58 /// Base and hint layer factories for the `SpawnerStack`. At
59 /// `service::linker::link` time, `compiled.router` is wrapped with
60 /// the base factories plus the hint factories resolved from
61 /// `blueprint.spawner_hints.layers`. This is the engine-side
62 /// counterpart to the discipline "Flow / Blueprint doesn't spell out
63 /// middleware implementations — it declares the capabilities it needs
64 /// as hint keys".
65 layer_registry: crate::middleware::LayerRegistry,
66 /// Optional Data-plane `OutputStore` backend (subtask-4 / ST2 rework —
67 /// see `submit_output`'s doc). `None` (the default) preserves
68 /// pre-subtask-4 behavior exactly: `submit_output` /
69 /// `submit_worker_result_trusted` only touch the Domain-plane
70 /// `EngineState.output_store` HashMap, same as before this was added.
71 /// `Some` additionally dual-writes every `Final` event into this store
72 /// via [`crate::store::output::OutputStore::append`], making it
73 /// queryable (e.g. by `mlua-swarm-server`'s `GET /v1/tasks/:id/ctx`)
74 /// even for an in-flight run. A plain `std::sync::RwLock` (not
75 /// `tokio::sync::RwLock`) — set once at boot via [`Engine::set_output_store`]
76 /// from a synchronous call site (`mlua-swarm-server`'s router builder),
77 /// then only ever briefly read (clone the `Option<Arc<..>>`, never held
78 /// across an `.await`) from the async submit path.
79 data_store: std::sync::RwLock<Option<Arc<dyn crate::store::output::OutputStore>>>,
80 /// GH #50 (Subtask 2 — runtime plumbing): agent name → declared
81 /// [`mlua_swarm_schema::VerdictContract`], the Engine-side registry
82 /// [`Self::verdict_contract_for_task`] resolves against. Populated via
83 /// [`Self::register_verdict_contracts`] — same sync-`RwLock`,
84 /// set-outside-the-lock idiom as `data_store` above. Empty by default
85 /// (every pre-GH-#50 `Engine`), which is exactly the opt-in "no
86 /// contract declared" state `verdict_contract_for_task` treats as
87 /// `None`. Populated from a live `Compiler::compile`'s
88 /// `CompiledAgentTable.verdict_contracts` output by
89 /// `TaskLaunchService::launch`, immediately after `compiler.compile`
90 /// succeeds — see [`Self::register_verdict_contracts`]'s doc for the
91 /// overwrite semantics of that merge.
92 verdict_contracts: std::sync::RwLock<HashMap<String, mlua_swarm_schema::VerdictContract>>,
93}
94
95/// Renders a `TaskSpec.initial_directive` / `EngineState.prompts`
96/// `Value` down to the `String` shape that string-consuming boundaries
97/// require (issue #18). Strings pass through verbatim; anything else
98/// (Object / Array / Number / Bool / Null) is serde-stringified. This
99/// is the single canonical rendering — the coercion that used to sit
100/// inside `EngineDispatcher::dispatch` moved here and is invoked only
101/// at consumer boundaries: `WorkerPayload.prompt` (HTTP
102/// `/v1/worker/prompt`), `WorkerInvocation.prompt` (in-process
103/// spawners), the subprocess spawner's directive arg/stdin, and the
104/// WS Spawn frame text render (`operator_ws::session`). Everything
105/// upstream (Blueprint dispatch → engine state → `fetch_prompt` →
106/// `Operator::execute`) keeps the `Value` end-to-end.
107pub(crate) fn render_directive_to_string(v: &Value) -> String {
108 match v {
109 Value::String(s) => s.clone(),
110 other => other.to_string(),
111 }
112}
113
114/// Renders a [`crate::worker::output::ContentRef`] down to the `Value` shape
115/// the BP-chain / `DispatchOutcome` consume. `Inline` passes its `value`
116/// through verbatim; `FileRef` is stringified into the same
117/// `{"file_ref", "mime", "size_hint"}` shape `materialize_final_submission`
118/// uses for its own file-materialize projection — one canonical
119/// stringification, not two independently-maintained copies (GH #36 ST1:
120/// shared by both the `Final`-pull and the `Artifact`-parts fold in
121/// [`Engine::dispatch_attempt_with`]'s doc).
122fn content_ref_to_value(content: crate::worker::output::ContentRef) -> Value {
123 match content {
124 crate::worker::output::ContentRef::Inline { value } => value,
125 crate::worker::output::ContentRef::FileRef {
126 path,
127 mime,
128 size_hint,
129 } => serde_json::json!({
130 "file_ref": path.to_string_lossy(),
131 "mime": mime,
132 "size_hint": size_hint,
133 }),
134 }
135}
136
137/// GH #51 — reduces a [`content_ref_to_value`] result down to the `String`
138/// shape the completion-time verdict-contract check compares against a
139/// declared `VerdictContract.values` token set. A `Value::String` unwraps
140/// to its raw contents (no surrounding JSON quotes) — this mirrors the
141/// pre-GH-#51 `check_verdict_contract` (`mlua-swarm-server`'s
142/// `worker.rs`), which always compared the raw submitted body string
143/// directly, never a JSON-stringified copy. Any OTHER `Value` shape
144/// (`Number` / `Object` / `Array` / `Bool` / `Null` — i.e. a `channel:
145/// "body"` contract whose completing value is not a string at all, or a
146/// `FileRef` content whose `content_ref_to_value` projection is an
147/// object) falls back to `Value::to_string()`'s JSON-encoded form: it can
148/// never collide with a plain declared token like `"PASS"`, so it
149/// naturally fails membership — consistent with the "non-string values
150/// under a body contract are violations" rule (issue #51's Proposal).
151fn content_ref_to_comparable_string(content: crate::worker::output::ContentRef) -> String {
152 let value = content_ref_to_value(content);
153 match value {
154 Value::String(s) => s,
155 other => other.to_string(),
156 }
157}
158
159/// [`Engine::dispatch_attempt_with`]'s Final-pull assembly (GH #36 ST1:
160/// named multi-part worker output), factored out as a pure function of the
161/// output-event tail so it is unit-testable without a live `Engine` /
162/// spawner.
163///
164/// Finds the LAST `Final` event in `tail` (mirrors the pre-GH-#36 pull:
165/// "last Final wins" if more than one was ever appended) and folds every
166/// `Artifact` event in the SAME tail WHOSE NAME APPEARS IN `staged_names`
167/// into a `"parts"` object keyed by `Artifact.name` — walked in tail (=
168/// event-append) order, so a name staged more than once within the attempt
169/// is last-write-wins (`Map` insert semantics, not an accumulating list;
170/// `Engine::stage_worker_artifact_trusted`'s doc). `staged_names` is the
171/// WORKER's own opt-in allowlist (`EngineState.worker_artifact_names`'s
172/// doc) — an `Artifact` on the tail whose name is NOT in `staged_names`
173/// (e.g. `AfterRunAuditMiddleware`'s `"audit:<step_ref>"` sidecar finding)
174/// is left alone, exactly as before GH #36; this is what keeps an audited
175/// step's BP-chain value byte-identical when the worker itself never
176/// staged a part.
177///
178/// At least one matching part: the returned value is `{"out": <final
179/// value>, "parts": {<name>: <value>, ...}}`. Zero matching parts: the
180/// returned value is the plain final value, unchanged from the pre-GH-#36
181/// shape — this is the back-compat guarantee, not an incidental default.
182///
183/// `None` when `tail` carries no `Final` at all (the caller's pre-existing
184/// "no Final in output_tail" error path).
185fn fold_final_and_parts(
186 tail: &[crate::worker::output::OutputEvent],
187 staged_names: &[String],
188) -> Option<(Value, bool)> {
189 let (final_content, ok) = tail.iter().rev().find_map(|ev| match ev {
190 crate::worker::output::OutputEvent::Final { content, ok } => Some((content.clone(), *ok)),
191 _ => None,
192 })?;
193 let final_value = content_ref_to_value(final_content);
194
195 let mut parts = serde_json::Map::new();
196 for ev in tail {
197 if let crate::worker::output::OutputEvent::Artifact { name, content } = ev {
198 if staged_names.iter().any(|staged| staged == name) {
199 parts.insert(name.clone(), content_ref_to_value(content.clone()));
200 }
201 }
202 }
203
204 let value = if parts.is_empty() {
205 final_value
206 } else {
207 serde_json::json!({ "out": final_value, "parts": Value::Object(parts) })
208 };
209 Some((value, ok))
210}
211
212impl Engine {
213 /// Backwards-compatible constructor that starts the engine without a
214 /// layer registry, preserving the signature already used by ~88
215 /// existing call sites. Use this when automatic middleware wrapping
216 /// at bind time is not needed. Callers such as `mlua-swarm-server` go through
217 /// `new_with_layers(cfg, registry)` to enable the hint-resolution path.
218 pub fn new(cfg: EngineCfg) -> Self {
219 Self::new_with_layers(cfg, crate::middleware::LayerRegistry::new())
220 }
221
222 /// Construct an `Engine` with an explicit `LayerRegistry`, enabling
223 /// hint-resolution: `spawner_hints.layers` declared on a `Blueprint`
224 /// are resolved against this registry when the spawner stack is bound
225 /// at `service::linker::link` time.
226 pub fn new_with_layers(
227 cfg: EngineCfg,
228 layer_registry: crate::middleware::LayerRegistry,
229 ) -> Self {
230 let (event_tx, _) = broadcast::channel(256);
231 let signer = TokenSigner::new(&cfg.token_secret);
232 Self {
233 inner: Arc::new(EngineInner {
234 state: Mutex::new(EngineState::new()),
235 cfg,
236 signer,
237 gate: default_role_verb_table(),
238 event_tx,
239 senior_bridges: tokio::sync::RwLock::new(HashMap::new()),
240 spawn_hooks: tokio::sync::RwLock::new(HashMap::new()),
241 operators: tokio::sync::RwLock::new(HashMap::new()),
242 layer_registry,
243 data_store: std::sync::RwLock::new(None),
244 verdict_contracts: std::sync::RwLock::new(HashMap::new()),
245 }),
246 }
247 }
248
249 /// Rebuild this `Engine` with a different `RoleVerbGate`. The gate is
250 /// treated as fixed-at-build-time, so this constructs a fresh
251 /// `EngineInner` (fresh empty `EngineState`) rather than mutating in
252 /// place — mainly a testing convenience for swapping gate rules.
253 pub fn with_gate(self, gate: RoleVerbGate) -> Self {
254 // The gate is fixed at build time — the intent is to build a fresh
255 // instance rather than mutating in place. As a testing convenience we
256 // do allow swapping the inner Arc. Simpler form: just rebuild
257 // Arc<EngineInner>.
258 let inner = Arc::new(EngineInner {
259 state: Mutex::new(EngineState::new()),
260 cfg: self.inner.cfg.clone(),
261 signer: self.inner.signer.clone(),
262 gate,
263 event_tx: self.inner.event_tx.clone(),
264 senior_bridges: tokio::sync::RwLock::new(HashMap::new()),
265 spawn_hooks: tokio::sync::RwLock::new(HashMap::new()),
266 operators: tokio::sync::RwLock::new(HashMap::new()),
267 layer_registry: self.inner.layer_registry.clone(),
268 data_store: std::sync::RwLock::new(None),
269 verdict_contracts: std::sync::RwLock::new(HashMap::new()),
270 });
271 Self { inner }
272 }
273
274 // ═══════════════════════════════════════════════════════════════════════
275 // Accessors. Production code drives execution through compile +
276 // `service::linker::link` + `dispatch_attempt_with(spawner)` inside
277 // `TaskLaunchService`; `Engine` itself is a pure execution surface — it
278 // does not own a BlueprintStore / EnhanceAdapter / Compiler, nor a
279 // global spawner (the spawner is carried per-request, never stashed on
280 // the engine).
281 // ═══════════════════════════════════════════════════════════════════════
282
283 /// Access the `EngineCfg` this engine was built with.
284 pub fn cfg(&self) -> &EngineCfg {
285 &self.inner.cfg
286 }
287
288 /// Expose the internal `LayerRegistry` — used when deriving a
289 /// sub-engine that needs the same registry re-injected. The
290 /// per-request sub-engine in `mlua-swarm-server` reads the parent engine's
291 /// registry through this accessor and passes it to
292 /// `Engine::new_with_layers(cfg, parent.layer_registry().clone())`.
293 pub fn layer_registry(&self) -> &crate::middleware::LayerRegistry {
294 &self.inner.layer_registry
295 }
296
297 /// Access the `TokenSigner` used to mint/verify `CapToken`s.
298 pub fn signer(&self) -> &TokenSigner {
299 &self.inner.signer
300 }
301
302 /// Clone a handle to the process-wide `Event` broadcast sender. Prefer
303 /// `subscribe` for a ready-to-use receiver.
304 pub fn event_tx(&self) -> broadcast::Sender<Event> {
305 self.inner.event_tx.clone()
306 }
307
308 /// Subscribe to the engine's `Event` broadcast stream.
309 pub fn subscribe(&self) -> EventStream {
310 self.inner.event_tx.subscribe()
311 }
312
313 /// Wires the Data-plane [`crate::store::output::OutputStore`] backend
314 /// used by `submit_output` / `submit_worker_result_trusted`'s
315 /// submit-time projection sink (subtask-4 / ST2 rework — see
316 /// `submit_output`'s doc). Synchronous (a plain `std::sync::RwLock`
317 /// write) so a caller can wire it up at boot from a non-`async`
318 /// context (`mlua-swarm-server`'s router builder passes the same
319 /// `Arc` it hands to its `AppState.data_store`, so `POST
320 /// /v1/data/emit` and every worker's ordinary `/v1/worker/submit` land
321 /// in the one store). Calling this more than once replaces the
322 /// previous backend; not calling it at all (the default) preserves
323 /// pre-subtask-4 behavior exactly — `submit_output` only touches the
324 /// Domain-plane `EngineState.output_store` HashMap.
325 pub fn set_output_store(&self, store: Arc<dyn crate::store::output::OutputStore>) {
326 let mut guard = self
327 .inner
328 .data_store
329 .write()
330 .unwrap_or_else(|poisoned| poisoned.into_inner());
331 *guard = Some(store);
332 }
333
334 /// Clones the currently-wired Data-plane store handle, if any. Kept
335 /// private and side-effect-free (no lock held past this call) —
336 /// callers (`materialize_final_submission`) do their actual `.append`
337 /// work outside of any lock.
338 fn output_store_backend(&self) -> Option<Arc<dyn crate::store::output::OutputStore>> {
339 self.inner
340 .data_store
341 .read()
342 .unwrap_or_else(|poisoned| poisoned.into_inner())
343 .clone()
344 }
345
346 /// GH #50 (Subtask 2): merges `contracts` (agent name → declared
347 /// [`mlua_swarm_schema::VerdictContract`]) into the engine's runtime
348 /// verdict-contract registry, later resolved per-task by
349 /// [`Self::verdict_contract_for_task`]. Same sync-write idiom as
350 /// [`Self::set_output_store`] — a plain `std::sync::RwLock` write, so
351 /// this can be called from a non-`async` context. Production call
352 /// site: `TaskLaunchService::launch`, immediately after a successful
353 /// `Compiler::compile`, passing `compiled.router.verdict_contracts.clone()`.
354 ///
355 /// # Overwrite semantics (explicit — read before adding a second call site)
356 ///
357 /// The registry is a single flat `HashMap` **keyed by agent name only**
358 /// (`String`), with process-wide (not per-task, not per-Blueprint,
359 /// not per-launch) scope. Registration is additive via
360 /// `HashMap::extend`: an entry for an agent name NOT already present is
361 /// added; an entry for an agent name ALREADY present is REPLACED
362 /// (last write wins) by the incoming one. Concretely: launching a
363 /// second Blueprint that also declares a `verdict` contract for an
364 /// agent named `"gate"` OVERWRITES whatever contract a first, still
365 /// in-flight, launch registered for an agent of that same name — even
366 /// if the two Blueprints intend it as two semantically different
367 /// agents that merely share a name, and even while the first launch's
368 /// tasks are still running. This is a **known limitation** of the v1
369 /// design; a per-task (or per-`RunId` / per-Blueprint) scoped registry
370 /// is a possible follow-up if two concurrently in-flight Blueprints
371 /// declaring conflicting contracts under the same agent name turns out
372 /// to matter in practice. Calling this with an empty map (or not at
373 /// all — the default) is a no-op, preserving pre-GH-#50 behavior
374 /// exactly (opt-in).
375 pub fn register_verdict_contracts(
376 &self,
377 contracts: HashMap<String, mlua_swarm_schema::VerdictContract>,
378 ) {
379 let mut guard = self
380 .inner
381 .verdict_contracts
382 .write()
383 .unwrap_or_else(|poisoned| poisoned.into_inner());
384 guard.extend(contracts);
385 }
386
387 /// GH #50 (Subtask 2): the declared
388 /// [`mlua_swarm_schema::VerdictContract`] for the agent currently
389 /// running `task_id`, if any. Resolves `task_id` → `TaskState.spec.agent`
390 /// (via `EngineState.tasks`, the same lookup [`Self::task_attempt`]
391 /// performs) and looks that agent name up in the registry
392 /// [`Self::register_verdict_contracts`] populates.
393 ///
394 /// `None` in both of these cases — deliberately collapsed to the same
395 /// value, mirroring [`Self::agent_context_for`]'s `Result`-into-`Option`
396 /// pattern (`.ok().flatten()`; a lookup failure here is never itself an
397 /// error worth surfacing to a caller):
398 /// - `task_id` is unknown (no `TaskState` for it).
399 /// - `task_id` resolves to a known agent, but that agent declared no
400 /// `verdict` contract (the opt-in default).
401 ///
402 /// Callers (`mlua-swarm-server`'s `worker_submit` / `worker_artifact`)
403 /// treat every `None` identically: skip the submit-time verdict gate
404 /// entirely, preserving pre-GH-#50 behavior byte-for-byte.
405 pub async fn verdict_contract_for_task(
406 &self,
407 task_id: &StepId,
408 ) -> Option<mlua_swarm_schema::VerdictContract> {
409 let tid = task_id.clone();
410 let agent = self
411 .with_state("verdict_contract_for_task", move |s| {
412 s.tasks.get(&tid).map(|t| t.spec.agent.clone())
413 })
414 .await
415 .ok()
416 .flatten()?;
417 self.inner
418 .verdict_contracts
419 .read()
420 .unwrap_or_else(|poisoned| poisoned.into_inner())
421 .get(&agent)
422 .cloned()
423 }
424
425 /// GH #51 — the value of the LAST staged `"verdict"` `Artifact` for
426 /// `(task_id, attempt)`, if any. Mirrors [`fold_final_and_parts`]'s
427 /// reverse-scan-of-`output_tail` pattern (last-write-wins per name,
428 /// same as that fold and [`Self::stage_worker_artifact_trusted`]'s
429 /// doc), narrowed to the single literal artifact name
430 /// `channel: "part"` contracts address (Pattern B — see
431 /// `blueprint-authoring.md`'s "Returning verdicts to drive BP flow").
432 ///
433 /// Infallible accessor: `None` is the normal "nothing staged yet"
434 /// case, not an error — the caller
435 /// ([`Self::verdict_contract_completion_check`]) is what converts
436 /// `None` into `Err(EngineError::VerdictPartMissing)`.
437 pub(crate) async fn staged_verdict_value_for(
438 &self,
439 task_id: &StepId,
440 attempt: u32,
441 ) -> Option<String> {
442 let tail = self.output_tail(task_id, attempt).await;
443 tail.iter().rev().find_map(|ev| match ev {
444 crate::worker::output::OutputEvent::Artifact { name, content } if name == "verdict" => {
445 Some(content_ref_to_comparable_string(content.clone()))
446 }
447 _ => None,
448 })
449 }
450
451 /// GH #51 — the single completion-time verdict-contract choke point,
452 /// embedded inside BOTH [`Self::submit_worker_result_trusted`] and
453 /// [`Self::submit_output`] (the two engine-side writes every HTTP/WS
454 /// completion route ultimately passes through). Not duplicated per
455 /// route handler — a future 4th completion route is gated for free
456 /// as long as it funnels through one of those two functions.
457 ///
458 /// `ok=false` is exempt on every route (this single early-return IS
459 /// the exemption, reused identically by both embedding sites — see
460 /// issue #51's "ok=false completions are exempt" acceptance
461 /// criterion). An agent with no declared contract, or a contract for
462 /// the OTHER channel, is untouched (`Ok(())`) — same opt-in,
463 /// byte-for-byte-preserving posture as
464 /// [`Self::verdict_contract_for_task`]'s doc.
465 ///
466 /// - `channel: "body"` — `value` (the completing `Final`'s content,
467 /// already reduced to a comparable string by the caller via
468 /// [`content_ref_to_comparable_string`]) must be a member of
469 /// `contract.values`.
470 /// - `channel: "part"` — [`Self::staged_verdict_value_for`] must find
471 /// a staged `"verdict"` artifact for this attempt (presence,
472 /// defense in depth over the staging-time membership check) AND its
473 /// value must be a member of `contract.values`.
474 async fn verdict_contract_completion_check(
475 &self,
476 task_id: &StepId,
477 attempt: u32,
478 ok: bool,
479 value: &str,
480 ) -> Result<(), EngineError> {
481 if !ok {
482 return Ok(());
483 }
484 let Some(contract) = self.verdict_contract_for_task(task_id).await else {
485 return Ok(());
486 };
487 match contract.channel {
488 mlua_swarm_schema::VerdictChannel::Body => {
489 if contract.values.iter().any(|v| v == value) {
490 Ok(())
491 } else {
492 Err(EngineError::VerdictValueRejected {
493 value: value.to_string(),
494 allowed: contract.values.clone(),
495 })
496 }
497 }
498 mlua_swarm_schema::VerdictChannel::Part => {
499 match self.staged_verdict_value_for(task_id, attempt).await {
500 None => Err(EngineError::VerdictPartMissing {
501 allowed: contract.values.clone(),
502 }),
503 Some(staged) if contract.values.iter().any(|v| v == &staged) => Ok(()),
504 Some(staged) => Err(EngineError::VerdictValueRejected {
505 value: staged,
506 allowed: contract.values.clone(),
507 }),
508 }
509 }
510 }
511 }
512
513 // ═══════════════════════════════════════════════════════════════════════
514 // §7 with_state — single Mutex + R1-R4 (try_lock + bounded retry + max-hold panic)
515 // ═══════════════════════════════════════════════════════════════════════
516
517 /// The closure is a **sync** `FnOnce` — you cannot pass an async
518 /// closure, which enforces R3 at the type level. Exceeding `max_hold`
519 /// panics in debug builds so that R4 violations surface immediately;
520 /// release builds emit a `tracing::warn!` and continue, so a
521 /// load-dependent overrun never unwinds the caller's task.
522 pub async fn with_state<F, R>(&self, op: &'static str, f: F) -> Result<R, EngineError>
523 where
524 F: FnOnce(&mut EngineState) -> R,
525 {
526 let cfg = &self.inner.cfg;
527
528 // R2: try_lock + bounded retry
529 let mut guard_opt = None;
530 for attempt in 0..=cfg.max_retry {
531 match self.inner.state.try_lock() {
532 Ok(g) => {
533 guard_opt = Some(g);
534 break;
535 }
536 Err(_) if cfg.try_only => return Err(EngineError::LockBusy(op)),
537 Err(_) => {
538 let backoff = cfg.backoff_ms_step * (attempt as u64 + 1);
539 tokio::time::sleep(Duration::from_millis(backoff)).await;
540 }
541 }
542 }
543 let mut guard = guard_opt.ok_or(EngineError::LockBusyAfterRetry(op))?;
544
545 // R4: max_hold guard
546 let start = Instant::now();
547 let result = f(&mut guard);
548 let elapsed_ms = start.elapsed().as_millis();
549 drop(guard);
550
551 if elapsed_ms > cfg.max_hold_ms {
552 // R4 violation. In dev/test this is a hard failure so a long op
553 // inside the lock surfaces immediately; in release it must never
554 // take the caller's task down with it (a panic here kills the run
555 // driver future and strands the RunRecord in `Running`).
556 tracing::warn!(
557 op,
558 elapsed_ms = %elapsed_ms,
559 max_hold_ms = %cfg.max_hold_ms,
560 "with_state exceeded max hold — suspected R3 violation (long op inside lock)"
561 );
562 #[cfg(debug_assertions)]
563 panic!(
564 "Engine.with_state('{op}') held {elapsed_ms}ms > max {}ms — suspected R3 violation (long op inside lock)",
565 cfg.max_hold_ms
566 );
567 }
568 Ok(result)
569 }
570
571 // ═══════════════════════════════════════════════════════════════════════
572 // Token verify (= sig + expire + gate + uses_left)
573 // ═══════════════════════════════════════════════════════════════════════
574
575 /// Four steps: (1) signature verify, (2) expiry check, (3) role × verb
576 /// gate, (4) `uses_left` consume.
577 pub async fn verify_token(&self, token: &CapToken, verb: Verb) -> Result<(), EngineError> {
578 // (1) sig
579 if !self.inner.signer.verify_sig(token) {
580 return Err(EngineError::BadSignature);
581 }
582 // (2) expire
583 if token.is_expired(now_unix()) {
584 return Err(EngineError::TokenExpired);
585 }
586 // (3) role × verb gate
587 if !self.inner.gate.is_allowed(token.role, verb) {
588 return Err(EngineError::RoleViolation {
589 role: token.role,
590 verb,
591 });
592 }
593 // (4) server-side uses_left consume
594 let fp = token.fingerprint();
595 self.with_state("token.consume", move |s| {
596 let rec = s
597 .tokens
598 .get_mut(&fp)
599 .ok_or_else(|| EngineError::TokenNotFound(fp.clone()))?;
600 rec.consume()
601 .map_err(|_: crate::core::state::CapTokenConsumeError| {
602 EngineError::TokenUsesExhausted
603 })?;
604 Ok::<(), EngineError>(())
605 })
606 .await??;
607 Ok(())
608 }
609
610 /// `verify_token` plus the **task-ownership gate**.
611 ///
612 /// When a Worker-role token calls a state-touch verb (`fetch_prompt` /
613 /// `post_result` / `read_task_state` / `cancel_task` / `poll_task`),
614 /// the gate checks that `CapTokenRecord.task_id` matches the argument
615 /// `task_id`; a mismatch returns `EngineError::TokenTaskMismatch`.
616 /// Operator / Senior / Observer tokens are outside the ownership gate
617 /// and may touch any task.
618 ///
619 /// **Verbs exempt from the gate.** `start_task` and `dispatch_attempt`
620 /// stay outside so recursive swarming keeps working; depth is capped
621 /// by `max_spawn_depth`.
622 pub async fn verify_token_for_task(
623 &self,
624 token: &CapToken,
625 verb: Verb,
626 task_id: &StepId,
627 ) -> Result<(), EngineError> {
628 self.verify_token(token, verb).await?;
629 if token.role != Role::Worker {
630 return Ok(());
631 }
632 let fp = token.fingerprint();
633 let arg_tid = task_id.clone();
634 self.with_state("token.ownership_gate", move |s| {
635 let bound = s.tokens.get(&fp).and_then(|r| r.task_id.as_ref()).cloned();
636 match bound {
637 Some(t) if t == arg_tid => Ok(()),
638 Some(t) => Err(EngineError::TokenTaskMismatch {
639 bound: t.into_string(),
640 arg: arg_tid.into_string(),
641 }),
642 None => Err(EngineError::TokenNotFound(fp.clone())),
643 }
644 })
645 .await??;
646 Ok(())
647 }
648
649 /// Resolve the bound `task_id` from a Worker-role token. Used on the
650 /// simple `/v1/worker/submit` endpoint, where the worker POSTs with a
651 /// token but no `task_id`. Returns `Err` if the token role is not
652 /// Worker, or if no bound task is set.
653 pub async fn task_id_from_token(&self, token: &CapToken) -> Result<StepId, EngineError> {
654 if token.role != Role::Worker {
655 return Err(EngineError::RoleViolation {
656 role: token.role,
657 verb: Verb::PostResult,
658 });
659 }
660 let fp = token.fingerprint();
661 self.with_state("task_id_from_token", move |s| {
662 s.tokens
663 .get(&fp)
664 .and_then(|r| r.task_id.as_ref())
665 .cloned()
666 .ok_or_else(|| EngineError::TokenNotFound(fp.clone()))
667 })
668 .await?
669 }
670
671 /// Resolve a short worker handle (`wh-XXXXXXXX`) to the bound
672 /// `task_id`. Used on `/v1/worker/submit` when the Bearer is a short
673 /// handle string rather than a full `CapToken` JSON. A missing entry
674 /// returns `TokenNotFound`, i.e. "the handle is not in the store".
675 pub async fn task_id_from_handle(&self, handle: &str) -> Result<StepId, EngineError> {
676 let h = handle.to_string();
677 self.with_state("task_id_from_handle", move |s| {
678 let fp = s
679 .worker_handles
680 .get(&h)
681 .cloned()
682 .ok_or_else(|| EngineError::TokenNotFound(format!("handle={h}")))?;
683 s.tokens
684 .get(&fp)
685 .and_then(|r| r.task_id.as_ref())
686 .cloned()
687 .ok_or_else(|| EngineError::TokenNotFound(format!("fp={fp}")))
688 })
689 .await?
690 }
691
692 /// Submit a worker result via a short handle. Skips token verification
693 /// and updates `output_tail` `Final` + `task.last_result` directly in
694 /// a thin path. The caller is expected to have already resolved
695 /// `task_id` via `task_id_from_handle` — the handle's presence in
696 /// `worker_handles` means it was minted server-side and is therefore
697 /// trusted.
698 ///
699 /// # GH #76 Skip tier: `outcome: SubmitOutcome`
700 ///
701 /// The `outcome` parameter (replacing the pre-#76 `ok: bool`) is the
702 /// caller's tier declaration:
703 ///
704 /// | outcome | `Final.ok` | `Final.content` | verdict-contract check |
705 /// |----------|------------|--------------------------------|------------------------|
706 /// | `Pass` | `true` | `value` verbatim | fires |
707 /// | `Blocked`| `false` | `value` verbatim | exempt (`ok=false`) |
708 /// | `Skip` | `true` | `wrap_skip_marker(value)` | exempt (Skip opt-out) |
709 ///
710 /// The Skip tier is opt-out from the verdict-contract completion check
711 /// on the same rationale [`crate::core::state::SubmitOutcome::Skip`]'s
712 /// doc records: the agent explicitly declared "not applicable", so
713 /// the payload is not a real verdict value to gate.
714 pub async fn submit_worker_result_trusted(
715 &self,
716 task_id: &StepId,
717 attempt: u32,
718 value: Value,
719 outcome: SubmitOutcome,
720 ) -> Result<(), EngineError> {
721 // Resolve outcome into the wire-level (value, ok, run_contract)
722 // triple exactly once, then reuse it below. Keeping the mapping
723 // literal in one place makes the "Skip wraps + skips contract"
724 // invariant grep-visible.
725 let (wire_value, wire_ok, run_contract_check) = match outcome {
726 SubmitOutcome::Pass => (value, true, true),
727 SubmitOutcome::Blocked => (value, false, false),
728 SubmitOutcome::Skip => (wrap_skip_marker(value), true, false),
729 };
730
731 // GH #51 — completion-time verdict-contract enforcement, embedded
732 // choke point 1 of 2 (see `Self::verdict_contract_completion_check`'s
733 // doc). This path always submits a `Final` by construction (there
734 // is no other event kind on `/v1/worker/submit`), so the check
735 // always applies — unlike `submit_output` below, no `if let
736 // OutputEvent::Final { .. }` guard is needed here since there is
737 // no other `OutputEvent` variant this function could be asked to
738 // write. Runs BEFORE the `output_tail` write immediately below:
739 // on `Err`, this returns immediately and neither `with_state` call
740 // in this function executes.
741 //
742 // GH #76 Skip tier: gated on `run_contract_check` — Skip is opt-out
743 // (see the outcome mapping table above), Blocked stays exempt via
744 // `verdict_contract_completion_check`'s existing `ok=false`
745 // early return (redundant flag here for grep locality).
746 if run_contract_check {
747 let comparable_value =
748 content_ref_to_comparable_string(crate::worker::output::ContentRef::Inline {
749 value: wire_value.clone(),
750 });
751 self.verdict_contract_completion_check(task_id, attempt, wire_ok, &comparable_value)
752 .await?;
753 }
754 let task_id_for_apply = task_id.clone();
755 let value_for_event = wire_value.clone();
756 self.with_state("submit_worker_result_trusted.output", move |s| {
757 let ev = crate::worker::output::OutputEvent::Final {
758 content: crate::worker::output::ContentRef::Inline {
759 value: value_for_event,
760 },
761 ok: wire_ok,
762 };
763 s.output_store
764 .entry((task_id_for_apply.clone(), attempt))
765 .or_default()
766 .push(ev.clone());
767 s.push_event(crate::core::state::Event::WorkerOutput {
768 task_id: task_id_for_apply,
769 attempt,
770 event: ev,
771 });
772 })
773 .await?;
774 let task_id_for_result = task_id.clone();
775 let value_for_result = wire_value.clone();
776 self.with_state("submit_worker_result_trusted.last_result", move |s| {
777 if let Some(t) = s.tasks.get_mut(&task_id_for_result) {
778 t.last_result = Some(value_for_result);
779 t.updated_at = now_unix();
780 }
781 })
782 .await?;
783 // subtask-4 / ST2 rework: this path always submits a `Final` (there
784 // is no other event kind on `/v1/worker/submit`), so the
785 // submit-time projection sink always fires — see
786 // `materialize_final_submission`'s doc and `submit_output`'s
787 // Invariants (fail-open, never turns a would-have-succeeded submit
788 // into a failure).
789 let content = crate::worker::output::ContentRef::Inline { value: wire_value };
790 self.materialize_final_submission(task_id, attempt, &content, wire_ok)
791 .await?;
792 Ok(())
793 }
794
795 /// Stage a named `Artifact` from a worker via a short handle (GH #36
796 /// ST1: named multi-part worker output). Trusted analog of
797 /// [`Self::submit_worker_result_trusted`] for `OutputEvent::Artifact`:
798 /// skips token verification for the same reason (the caller already
799 /// resolved `task_id` via `task_id_from_handle`, so the handle's
800 /// presence in `worker_handles` is itself the trust boundary).
801 ///
802 /// Appends to the same per-`(task_id, attempt)` `output_store` tail
803 /// [`Self::dispatch_attempt_with`]'s Final-pull later folds into
804 /// `{"out": <final>, "parts": {<name>: <value>, ...}}` (see that
805 /// method's doc for the fold semantics — event order, last-write-wins
806 /// per name), AND records `name` in `EngineState.worker_artifact_names`
807 /// — the fold's allowlist of the WORKER's own staged parts, as opposed
808 /// to every `Artifact` that happens to land on the shared tail (e.g. an
809 /// audit sidecar finding; see that field's doc). Also dual-writes to
810 /// the Data-plane `OutputStore` the same way [`Self::submit_output`]'s
811 /// `Artifact` arm does, via [`Self::materialize_artifact_submission`]
812 /// (the artifact's own `name` is its Data-plane key, no
813 /// canonicalization — see that method's doc).
814 pub async fn stage_worker_artifact_trusted(
815 &self,
816 task_id: &StepId,
817 attempt: u32,
818 name: String,
819 value: Value,
820 ) -> Result<(), EngineError> {
821 let content = crate::worker::output::ContentRef::Inline { value };
822 let task_id_for_apply = task_id.clone();
823 let name_for_apply = name.clone();
824 let content_for_apply = content.clone();
825 self.with_state("stage_worker_artifact_trusted.output", move |s| {
826 let ev = crate::worker::output::OutputEvent::Artifact {
827 name: name_for_apply.clone(),
828 content: content_for_apply,
829 };
830 s.output_store
831 .entry((task_id_for_apply.clone(), attempt))
832 .or_default()
833 .push(ev.clone());
834 s.record_worker_artifact_name(task_id_for_apply.clone(), attempt, name_for_apply);
835 s.push_event(crate::core::state::Event::WorkerOutput {
836 task_id: task_id_for_apply,
837 attempt,
838 event: ev,
839 });
840 })
841 .await?;
842 self.materialize_artifact_submission(task_id, attempt, &name, &content)
843 .await?;
844 Ok(())
845 }
846
847 /// The in-process lane's half of the "this part is the WORKER's own"
848 /// signal: record `name` in `EngineState.worker_artifact_names` for an
849 /// `Artifact` that already went through [`Self::submit_output`].
850 ///
851 /// The out-of-process lane gets this for free inside
852 /// [`Self::stage_worker_artifact_trusted`] (one `with_state`, tail
853 /// append + name record together). An in-process worker has no HTTP
854 /// route to call: it stages through `WorkerInvocation.sink`, which
855 /// lands on the generic `submit_output` — the same entry point OTHER
856 /// `Artifact` producers use (`AfterRunAuditMiddleware`'s
857 /// `"audit:<step_ref>"` sidecar), so `submit_output` itself must NOT
858 /// record. The distinction lives one layer up, in
859 /// [`crate::worker::output::EngineSink`]: `InProcSpawner::spawn` is
860 /// its sole constructor, so an `Artifact` arriving through that sink
861 /// is by construction the worker's own, and the sink calls this
862 /// immediately after its `submit_output` succeeds.
863 ///
864 /// Without it a `channel: "part"` in-process gate passes the
865 /// completion-time contract check (which reads the tail directly) yet
866 /// its part never folds into `{out, parts}` — so a downstream
867 /// `$.<step>.parts["verdict"]` cond reads `null`, the exact
868 /// half-working state GH #86's sink bridge left behind.
869 ///
870 /// Two calls rather than one atomic `with_state` is deliberate here:
871 /// the tail write must be allowed to fail (contract rejection, strict
872 /// `CheckPolicy`) WITHOUT leaving a phantom name behind, so the record
873 /// is strictly downstream of a successful submit.
874 pub(crate) async fn record_worker_artifact_name(
875 &self,
876 task_id: &StepId,
877 attempt: u32,
878 name: String,
879 ) -> Result<(), EngineError> {
880 let task_id = task_id.clone();
881 self.with_state("record_worker_artifact_name", move |s| {
882 s.record_worker_artifact_name(task_id, attempt, name);
883 })
884 .await
885 }
886
887 /// GH #36 ST1: the set of `Artifact` names staged for `(task_id,
888 /// attempt)` by the worker itself — see
889 /// `EngineState.worker_artifact_names`'s doc for the two lanes that
890 /// populate it. Used by [`Self::dispatch_attempt_with`]'s Final-pull
891 /// to distinguish a worker's own named parts from any other `Artifact`
892 /// producer on the same tail.
893 async fn worker_artifact_names_for(&self, task_id: &StepId, attempt: u32) -> Vec<String> {
894 let key = (task_id.clone(), attempt);
895 self.with_state("worker_artifact_names_for", move |s| {
896 s.worker_artifact_names
897 .get(&key)
898 .cloned()
899 .unwrap_or_default()
900 })
901 .await
902 .unwrap_or_default()
903 }
904
905 /// Mint a short handle and register it in the `worker_handles` map.
906 /// Called immediately after the worker-token mint inside
907 /// `dispatch_attempt_with`, and issues a handle bound to the same
908 /// token fingerprint. Format is `wh-<8 hex chars>` (11 chars total),
909 /// designed to remove the base64 copy-paste failure mode.
910 async fn mint_worker_handle(&self, worker_fp: String) -> Result<String, EngineError> {
911 // The handle is a sole bearer secret on the `/v1/worker/submit`
912 // short-handle path (`submit_worker_result_trusted` skips token
913 // verification), so it must be unguessable — OS RNG, not the
914 // predictable uid counter. 8 hex chars (~4B entropy) keeps the
915 // documented `wh-<8 hex>` wire shape; collision between live
916 // handles is negligible at in-process handle counts.
917 let short = crate::types::secure_hex(4);
918 let handle = format!("wh-{short}");
919 let h = handle.clone();
920 self.with_state("mint_worker_handle", move |s| {
921 s.worker_handles.insert(h, worker_fp);
922 })
923 .await?;
924 Ok(handle)
925 }
926
927 // ═══════════════════════════════════════════════════════════════════════
928 // Session API
929 // ═══════════════════════════════════════════════════════════════════════
930
931 /// Attach a new session with default `OperatorInfo` (`Automate`, no
932 /// bridges/hooks). Shorthand for `attach_with(.., OperatorInfo::default())`.
933 pub async fn attach(
934 &self,
935 operator_id: impl Into<String>,
936 role: Role,
937 ttl: Duration,
938 ) -> Result<CapToken, EngineError> {
939 self.attach_with(
940 operator_id,
941 role,
942 ttl,
943 crate::core::ctx::OperatorInfo::default(),
944 )
945 .await
946 }
947
948 // ═══════════════════════════════════════════════════════════════════════
949 // BridgeRegistry API.
950 // ═══════════════════════════════════════════════════════════════════════
951
952 /// Register a `SeniorBridge` under a name. An existing entry with the
953 /// same name is overwritten. On the persisted-session reattach path,
954 /// the caller re-registers under the same ID beforehand and the
955 /// bridge becomes effective again.
956 pub async fn register_senior_bridge(
957 &self,
958 id: impl Into<String>,
959 bridge: Arc<dyn SeniorBridge>,
960 ) {
961 self.inner
962 .senior_bridges
963 .write()
964 .await
965 .insert(id.into(), bridge);
966 }
967
968 /// Register a `SpawnHook` under a name. An existing entry with the
969 /// same name is overwritten.
970 pub async fn register_spawn_hook(&self, id: impl Into<String>, hook: Arc<dyn SpawnHook>) {
971 self.inner.spawn_hooks.write().await.insert(id.into(), hook);
972 }
973
974 /// Register an `Operator` (a spawn-body backend) under a name. An
975 /// existing entry with the same name is overwritten.
976 /// `OperatorDelegateMiddleware` looks this up via `ctx` and, when
977 /// `kind = MainAi` / `Composite`, bypasses `inner.spawn` and calls
978 /// `operator.execute` instead.
979 pub async fn register_operator(
980 &self,
981 id: impl Into<String>,
982 operator: Arc<dyn crate::operator::Operator>,
983 ) {
984 self.inner
985 .operators
986 .write()
987 .await
988 .insert(id.into(), operator);
989 }
990
991 /// Unregister a `SeniorBridge` by name (e.g. on WebSocket disconnect
992 /// or explicit teardown). A missing ID is a no-op.
993 pub async fn unregister_senior_bridge(&self, id: &str) {
994 self.inner.senior_bridges.write().await.remove(id);
995 }
996
997 /// Unregister a `SpawnHook` by name. A missing ID is a no-op.
998 pub async fn unregister_spawn_hook(&self, id: &str) {
999 self.inner.spawn_hooks.write().await.remove(id);
1000 }
1001
1002 /// Unregister an `Operator` backend by name. A missing ID is a no-op.
1003 pub async fn unregister_operator(&self, id: &str) {
1004 self.inner.operators.write().await.remove(id);
1005 }
1006
1007 /// Snapshot the list of registered `SpawnHook` IDs (for test
1008 /// observation and debugging).
1009 pub async fn list_spawn_hook_ids(&self) -> Vec<String> {
1010 self.inner
1011 .spawn_hooks
1012 .read()
1013 .await
1014 .keys()
1015 .cloned()
1016 .collect()
1017 }
1018
1019 /// Snapshot the list of registered `SeniorBridge` IDs.
1020 pub async fn list_senior_bridge_ids(&self) -> Vec<String> {
1021 self.inner
1022 .senior_bridges
1023 .read()
1024 .await
1025 .keys()
1026 .cloned()
1027 .collect()
1028 }
1029
1030 /// Snapshot the list of registered `Operator` IDs.
1031 pub async fn list_operator_ids(&self) -> Vec<String> {
1032 self.inner.operators.read().await.keys().cloned().collect()
1033 }
1034
1035 /// Attach specifying IDs directly. The caller is expected to have
1036 /// pre-registered them via `register_senior_bridge` /
1037 /// `register_spawn_hook` / `register_operator`. This is the canonical
1038 /// path when persistence is in play.
1039 ///
1040 /// `kind` is the "Runtime Global" tier of the `OperatorKind` cascade
1041 /// (stored verbatim on `OperatorSession.operator_kind`): `Some(_)` is
1042 /// an explicit request (including `Some(OperatorKind::Automate)`) that
1043 /// outranks the BP-level tiers; `None` leaves it unspecified so the
1044 /// BP-level tiers / final default decide. See
1045 /// `crate::core::ctx::collapse_operator_kind`.
1046 #[allow(clippy::too_many_arguments)]
1047 pub async fn attach_with_ids(
1048 &self,
1049 operator_id: impl Into<String>,
1050 role: Role,
1051 ttl: Duration,
1052 kind: Option<OperatorKind>,
1053 bridge_id: Option<String>,
1054 hook_id: Option<String>,
1055 operator_backend_id: Option<String>,
1056 operator_kind_overrides: HashMap<String, OperatorKind>,
1057 bp_agent_kinds: HashMap<String, OperatorKind>,
1058 bp_global_kind: Option<OperatorKind>,
1059 ) -> Result<CapToken, EngineError> {
1060 let operator_id = operator_id.into();
1061 let token = self
1062 .inner
1063 .signer
1064 .session(operator_id.clone(), role, vec!["*".into()], ttl);
1065 let session_id = SessionId::new();
1066 let fp = token.fingerprint();
1067 let now = now_unix();
1068 let token_for_store = token.clone();
1069
1070 self.with_state("attach_with_ids", |s| {
1071 s.tokens
1072 .insert(fp.clone(), CapTokenRecord::from_token(token_for_store));
1073 s.sessions.insert(
1074 session_id.clone(),
1075 OperatorSession {
1076 id: session_id.clone(),
1077 operator_id: operator_id.clone(),
1078 role,
1079 attached_at: now,
1080 last_seen: now,
1081 attached: true,
1082 owned_task_ids: Vec::new(),
1083 token_fp: fp.clone(),
1084 operator_kind: kind,
1085 runtime_agent_kinds: operator_kind_overrides,
1086 bp_agent_kinds,
1087 bp_global_kind,
1088 bridge_id,
1089 hook_id,
1090 operator_backend_id,
1091 },
1092 );
1093 s.push_event(Event::SessionAttached {
1094 session_id: session_id.clone(),
1095 role,
1096 });
1097 })
1098 .await?;
1099
1100 let _ = self
1101 .inner
1102 .event_tx
1103 .send(Event::SessionAttached { session_id, role });
1104 Ok(token)
1105 }
1106
1107 /// Build an `OperatorInfo` by looking up the session's registered IDs
1108 /// on the `BridgeRegistry`, plus resolving the 4-tier `OperatorKind`
1109 /// cascade for `agent_name` via `crate::core::ctx::collapse_operator_kind`.
1110 /// Used when `dispatch_attempt` injects `Ctx`. An unresolved ID
1111 /// (nothing registered) is silently `None` — the bridge / hook simply
1112 /// does not fire and the default behaviour applies.
1113 async fn resolve_operator_info(
1114 &self,
1115 session: &OperatorSession,
1116 agent_name: &str,
1117 ) -> OperatorInfo {
1118 let senior_bridge = if let Some(id) = &session.bridge_id {
1119 self.inner.senior_bridges.read().await.get(id).cloned()
1120 } else {
1121 None
1122 };
1123 let spawn_hook = if let Some(id) = &session.hook_id {
1124 self.inner.spawn_hooks.read().await.get(id).cloned()
1125 } else {
1126 None
1127 };
1128 let operator = if let Some(id) = &session.operator_backend_id {
1129 self.inner.operators.read().await.get(id).cloned()
1130 } else {
1131 None
1132 };
1133 let runtime_agent = session.runtime_agent_kinds.get(agent_name).copied();
1134 // "Runtime Global" tier: `Some(_)` is always an explicit request
1135 // (see the field doc on `OperatorSession.operator_kind`).
1136 let runtime_global = session.operator_kind;
1137 let bp_agent = session.bp_agent_kinds.get(agent_name).copied();
1138 let bp_global = session.bp_global_kind;
1139 let kind = crate::core::ctx::collapse_operator_kind(
1140 runtime_agent,
1141 runtime_global,
1142 bp_agent,
1143 bp_global,
1144 );
1145 OperatorInfo {
1146 kind,
1147 id: session.operator_id.clone(),
1148 senior_bridge,
1149 spawn_hook,
1150 operator,
1151 }
1152 }
1153
1154 /// Convenience attach that takes an `OperatorInfo` (three
1155 /// `Arc<dyn ...>` fields plus `kind`) **inline**.
1156 ///
1157 /// # Pipeline
1158 ///
1159 /// Each `Arc<dyn ...>` is auto-registered on the engine's registry
1160 /// under a synthetic ID (`br-<hex>` / `hk-<hex>` / `ob-<hex>`), and
1161 /// the session stores that synthetic ID. Subsequent `dispatch_attempt`
1162 /// calls rebuild the `Arc`s from those IDs via
1163 /// `resolve_operator_info`, and the three middlewares fire as usual.
1164 ///
1165 /// # ⚠ Non-persisted sessions only
1166 ///
1167 /// Because this API takes inline `Arc`s, the reattach path after
1168 /// session persistence cannot rebuild them — the synthetic IDs are
1169 /// not present in a freshly started process's registry. If you need
1170 /// persistence, use [`Self::attach_with_ids`] with `register_*` calls
1171 /// beforehand to go through **named IDs** instead.
1172 ///
1173 /// Handy for tests and short-lived in-process sessions. Production
1174 /// WebSocket callbacks and the like should prefer `attach_with_ids`
1175 /// as the canonical path.
1176 pub async fn attach_with(
1177 &self,
1178 operator_id: impl Into<String>,
1179 role: Role,
1180 ttl: Duration,
1181 operator_info: crate::core::ctx::OperatorInfo,
1182 ) -> Result<CapToken, EngineError> {
1183 let operator_id = operator_id.into();
1184 // The caller always hands in a fully-formed `OperatorInfo`
1185 // (including its `kind`), so it is stored as an explicit "Runtime
1186 // Global" tier request (`Some(kind)`) — this path never persists
1187 // BP-level tiers (both stay empty below), so `Some(kind)` resolves
1188 // to the same `kind` at dispatch either way; see
1189 // `OperatorSession.operator_kind` doc.
1190 let kind = operator_info.kind;
1191 // BridgeRegistry auto-register: when the caller hands in an
1192 // `Arc<dyn>` directly, register it under a synthesised ID (the inline
1193 // path aware of persistence). Callers who want to pre-register with a
1194 // named ID should use `register_senior_bridge` / `register_spawn_hook`
1195 // + `attach_with_ids`.
1196 let bridge_id = if let Some(bridge) = operator_info.senior_bridge.clone() {
1197 let id = format!("br-{}", crate::types::uid_hex(8));
1198 self.inner
1199 .senior_bridges
1200 .write()
1201 .await
1202 .insert(id.clone(), bridge);
1203 Some(id)
1204 } else {
1205 None
1206 };
1207 let hook_id = if let Some(hook) = operator_info.spawn_hook.clone() {
1208 let id = format!("hk-{}", crate::types::uid_hex(8));
1209 self.inner
1210 .spawn_hooks
1211 .write()
1212 .await
1213 .insert(id.clone(), hook);
1214 Some(id)
1215 } else {
1216 None
1217 };
1218 let operator_backend_id = if let Some(operator) = operator_info.operator.clone() {
1219 // `ob-` = operator-backend registry id. Renamed from `op-` in the
1220 // issue #11 prefix reconciliation: `op-` used to collide with the
1221 // WS operator sid shape (now unified into `S-<hex>` anyway), and a
1222 // shared prefix across two unrelated registries made log filtering
1223 // by prefix silently ambiguous.
1224 let id = format!("ob-{}", crate::types::uid_hex(8));
1225 self.inner
1226 .operators
1227 .write()
1228 .await
1229 .insert(id.clone(), operator);
1230 Some(id)
1231 } else {
1232 None
1233 };
1234
1235 let token = self
1236 .inner
1237 .signer
1238 .session(operator_id.clone(), role, vec!["*".into()], ttl);
1239 let session_id = SessionId::new();
1240 let fp = token.fingerprint();
1241 let now = now_unix();
1242 let token_for_store = token.clone();
1243
1244 self.with_state("attach_with", |s| {
1245 s.tokens
1246 .insert(fp.clone(), CapTokenRecord::from_token(token_for_store));
1247 s.sessions.insert(
1248 session_id.clone(),
1249 OperatorSession {
1250 id: session_id.clone(),
1251 operator_id,
1252 role,
1253 attached_at: now,
1254 last_seen: now,
1255 attached: true,
1256 owned_task_ids: Vec::new(),
1257 token_fp: fp.clone(),
1258 operator_kind: Some(kind),
1259 runtime_agent_kinds: HashMap::new(),
1260 bp_agent_kinds: HashMap::new(),
1261 bp_global_kind: None,
1262 bridge_id,
1263 hook_id,
1264 operator_backend_id,
1265 },
1266 );
1267 s.push_event(Event::SessionAttached {
1268 session_id: session_id.clone(),
1269 role,
1270 });
1271 })
1272 .await?;
1273
1274 let _ = self
1275 .inner
1276 .event_tx
1277 .send(Event::SessionAttached { session_id, role });
1278 Ok(token)
1279 }
1280
1281 /// Mark the session bound to `token` as detached (`attached = false`).
1282 /// Tasks are left in place — a later `attach`/`attach_with_ids` call
1283 /// carrying the same registered bridge/hook IDs can pick them back up.
1284 pub async fn detach(&self, token: &CapToken) -> Result<(), EngineError> {
1285 self.verify_token(token, Verb::DetachSession).await?;
1286 let fp = token.fingerprint();
1287 self.with_state("detach", move |s| {
1288 let sid = s
1289 .sessions
1290 .iter()
1291 .find(|(_, sess)| sess.token_fp == fp)
1292 .map(|(id, _)| id.clone());
1293 if let Some(sid) = sid {
1294 if let Some(sess) = s.sessions.get_mut(&sid) {
1295 sess.attached = false;
1296 }
1297 s.push_event(Event::SessionDetached {
1298 session_id: sid.clone(),
1299 });
1300 let _ = sid;
1301 }
1302 })
1303 .await?;
1304 Ok(())
1305 }
1306
1307 /// Refresh the session's `last_seen` timestamp and mark it `attached`.
1308 /// Called periodically by an attached client to avoid being flipped to
1309 /// detached by `start_detach_loop`.
1310 pub async fn heartbeat(&self, token: &CapToken) -> Result<(), EngineError> {
1311 self.verify_token(token, Verb::Heartbeat).await?;
1312 let now = now_unix();
1313 let fp = token.fingerprint();
1314 self.with_state("heartbeat", move |s| {
1315 if let Some(sess) = s.sessions.values_mut().find(|sess| sess.token_fp == fp) {
1316 sess.last_seen = now;
1317 sess.attached = true;
1318 }
1319 })
1320 .await?;
1321 Ok(())
1322 }
1323
1324 // ═══════════════════════════════════════════════════════════════════════
1325 // Task lifecycle
1326 // ═══════════════════════════════════════════════════════════════════════
1327
1328 /// Create a new `TaskState` from `spec` and register its initial
1329 /// prompt. When the calling token is a Worker (i.e. this is a
1330 /// recursive spawn), the new task inherits `parent.spawn_depth + 1`
1331 /// and is rejected with `SpawnDepthExceeded` once `max_spawn_depth` is
1332 /// hit; an Operator-issued call starts at depth 0.
1333 pub async fn start_task(
1334 &self,
1335 token: &CapToken,
1336 spec: TaskSpec,
1337 ) -> Result<StepId, EngineError> {
1338 self.verify_token(token, Verb::StartTask).await?;
1339 let task_id = StepId::new();
1340 let initial_directive = spec.initial_directive.clone();
1341 let task_id_clone = task_id.clone();
1342 let fp = token.fingerprint();
1343 let max_depth = self.inner.cfg.max_spawn_depth;
1344 self.with_state("start_task", move |s| {
1345 // Recursive swarm depth gate (recursion guard):
1346 // Worker tokens carry CapTokenRecord.parent_task_id. Give the
1347 // child parent's spawn_depth + 1; if it exceeds `max`, raise an
1348 // error. Operator tokens (parent_task_id=None) start at depth 0.
1349 let parent_depth_opt = s
1350 .tokens
1351 .get(&fp)
1352 .and_then(|rec| rec.task_id.as_ref())
1353 .and_then(|tid| s.tasks.get(tid))
1354 .map(|t| t.spawn_depth);
1355 let depth = match parent_depth_opt {
1356 Some(d) => {
1357 if d + 1 >= max_depth {
1358 return Err(EngineError::SpawnDepthExceeded {
1359 current: d + 1,
1360 max: max_depth,
1361 });
1362 }
1363 d + 1
1364 }
1365 None => 0,
1366 };
1367
1368 let mut task = TaskState::new(task_id_clone.clone(), spec);
1369 task.spawn_depth = depth;
1370 s.tasks.insert(task_id_clone.clone(), task);
1371 s.prompts
1372 .insert((task_id_clone.clone(), 1), initial_directive);
1373 // Link to the owner session (only Operator tokens match; Worker tokens have no session).
1374 if let Some(sess) = s.sessions.values_mut().find(|sess| sess.token_fp == fp) {
1375 sess.owned_task_ids.push(task_id_clone.clone());
1376 }
1377 s.push_event(Event::TaskCreated {
1378 task_id: task_id_clone.clone(),
1379 });
1380 Ok::<(), EngineError>(())
1381 })
1382 .await??;
1383 let _ = self.inner.event_tx.send(Event::TaskCreated {
1384 task_id: task_id.clone(),
1385 });
1386 Ok(task_id)
1387 }
1388
1389 /// Fetch a snapshot of `TaskState` for `task_id`, subject to the
1390 /// task-ownership gate (see `verify_token_for_task`).
1391 pub async fn read_task_state(
1392 &self,
1393 token: &CapToken,
1394 task_id: &StepId,
1395 ) -> Result<TaskState, EngineError> {
1396 self.verify_token_for_task(token, Verb::ReadTaskState, task_id)
1397 .await?;
1398 let task_id = task_id.clone();
1399 self.with_state("read_task_state", move |s| {
1400 s.tasks
1401 .get(&task_id)
1402 .cloned()
1403 .ok_or_else(|| EngineError::TaskNotFound(task_id.to_string()))
1404 })
1405 .await?
1406 }
1407
1408 /// Mark `task_id` as `Cancelled` and wake any caller blocked in
1409 /// `poll_task` for it.
1410 pub async fn cancel_task(&self, token: &CapToken, task_id: &StepId) -> Result<(), EngineError> {
1411 self.verify_token_for_task(token, Verb::CancelTask, task_id)
1412 .await?;
1413 let tid = task_id.clone();
1414 self.with_state("cancel_task", move |s| {
1415 let task = s
1416 .tasks
1417 .get_mut(&tid)
1418 .ok_or_else(|| EngineError::TaskNotFound(tid.to_string()))?;
1419 task.status = TaskStatus::Cancelled;
1420 task.updated_at = now_unix();
1421 s.push_event(Event::TaskCancelled {
1422 task_id: tid.clone(),
1423 });
1424 Ok::<(), EngineError>(())
1425 })
1426 .await??;
1427 self.wake_task(task_id).await?;
1428 Ok(())
1429 }
1430
1431 /// Dispatch a single attempt through the given `spawner`.
1432 ///
1433 /// The lock is only held for snapshot capture; the actual spawn and
1434 /// completion await happen outside the lock (R3 discipline).
1435 ///
1436 /// Sits on the Domain side of the Data / Domain split. The dispatch
1437 /// path itself does not touch big response bodies — those flow through
1438 /// the Data plane (`output_store` module + sink / input_inject
1439 /// `SpawnerLayer`s) around this method.
1440 ///
1441 /// The caller does the compile plus `service::linker::link` and
1442 /// carries the same stack through each dispatch. Because the spawner
1443 /// is passed per-request rather than looked up from engine-global
1444 /// state, parallel requests against a single `Engine` instance
1445 /// (different Blueprints, different spawners) do not race.
1446 ///
1447 /// `run_id`, when `Some` (issue #13 run_id propagation —
1448 /// `EngineDispatcher` threads it in from its `RunContext`), is
1449 /// inserted into `Ctx.meta.runtime["run_id"]` (a plain JSON string)
1450 /// alongside `worker_handle`, so `Operator::execute` implementations
1451 /// (e.g. `WSOperatorSession`) can read it back and surface it to the
1452 /// worker (Spawn directive / prompt). `None` (every pre-existing
1453 /// caller / test) omits the key entirely — unchanged behavior.
1454 pub async fn dispatch_attempt_with(
1455 &self,
1456 token: &CapToken,
1457 task_id: &StepId,
1458 spawner: &Arc<dyn SpawnerAdapter>,
1459 run_id: Option<&RunId>,
1460 ) -> Result<DispatchOutcome, EngineError> {
1461 self.verify_token(token, Verb::DispatchAttempt).await?;
1462 let task_id = task_id.clone();
1463
1464 // 1) Under the lock: increment the attempt number, mark Running, snapshot the
1465 // prompt, and pull `operator_info` from the session so we can inject it into Ctx.
1466 let fp = token.fingerprint();
1467 let tid_for_prep = task_id.clone();
1468 let (attempt, agent, session_snapshot, step_ctx) = self
1469 .with_state("dispatch.prep", move |s| {
1470 let task = s
1471 .tasks
1472 .get_mut(&tid_for_prep)
1473 .ok_or_else(|| EngineError::TaskNotFound(tid_for_prep.to_string()))?;
1474 task.attempt += 1;
1475 task.status = TaskStatus::Running;
1476 task.updated_at = now_unix();
1477 // The spawner pulls the prompt via engine.fetch_prompt. In prep,
1478 // if the prompts table has no entry for this attempt yet,
1479 // fall back and insert `initial_directive` so the subsequent
1480 // fetch_prompt succeeds.
1481 let attempt = task.attempt;
1482 let initial = task.spec.initial_directive.clone();
1483 s.prompts
1484 .entry((tid_for_prep.clone(), attempt))
1485 .or_insert(initial);
1486 let task = s
1487 .tasks
1488 .get(&tid_for_prep)
1489 .ok_or_else(|| EngineError::TaskNotFound(tid_for_prep.to_string()))?;
1490 let agent = task.spec.agent.clone();
1491 // GH #21 Phase 2: re-read `TaskSpec.step_ctx` on EVERY
1492 // attempt (not cached once at start_task) so retries and
1493 // Run-rekicks all carry the Step tier through to Ctx —
1494 // see TaskSpec.step_ctx's doc.
1495 let step_ctx = task.spec.step_ctx.clone();
1496 // Session snapshot (looked up by token nonce). When no session
1497 // exists (worker token invoked directly / test injection), fall
1498 // back to None → default OperatorInfo.
1499 let sess_clone = s
1500 .sessions
1501 .values()
1502 .find(|sess| sess.token_fp == fp)
1503 .cloned();
1504 Ok::<_, EngineError>((attempt, agent, sess_clone, step_ctx))
1505 })
1506 .await??;
1507 // BridgeRegistry lookup + per-agent OperatorKind cascade.
1508 let operator_info = match session_snapshot {
1509 Some(sess) => self.resolve_operator_info(&sess, &agent).await,
1510 None => OperatorInfo::default(),
1511 };
1512
1513 // 2) Outside the lock: worker token mint + spawn.
1514 //
1515 // Session-style mint (max_uses=None). Within one attempt the worker is
1516 // expected to hit `verify_token + fetch_prompt + fetch_data + post_result`
1517 // multiple times in order, so `one_time` would exhaust the token on the
1518 // very first verb. Capability is guarded by (a) the role × verb gate and
1519 // (b) the short TTL (1800s).
1520 let worker_token = self.inner.signer.session(
1521 format!("worker-of-{task_id}"),
1522 Role::Worker,
1523 vec!["*".into()],
1524 Duration::from_secs(1800),
1525 );
1526 let worker_fp = worker_token.fingerprint();
1527 let task_id_for_worker = task_id.clone();
1528 let worker_token_for_store = worker_token.clone();
1529 self.with_state("dispatch.mint_worker", move |s| {
1530 s.tokens.insert(
1531 worker_fp,
1532 CapTokenRecord::from_worker_token(worker_token_for_store, task_id_for_worker),
1533 );
1534 })
1535 .await?;
1536
1537 // Mint a short handle (`wh-XXXXXXXX`) and register it in worker_handles.
1538 // Used by the simplified Bearer path for SubAgents (short-handle form
1539 // avoids base64 copy-paste incidents).
1540 let worker_handle = self.mint_worker_handle(worker_token.fingerprint()).await?;
1541
1542 let mut ctx = Ctx::new(task_id.clone(), attempt, agent.clone());
1543 ctx.operator = operator_info; // activates MainAIMiddleware / Senior bridge
1544 ctx.meta
1545 .runtime
1546 .insert("worker_handle".to_string(), Value::String(worker_handle));
1547 if let Some(rid) = run_id {
1548 ctx.meta
1549 .runtime
1550 .insert(RUN_ID_KEY.to_string(), Value::String(rid.to_string()));
1551 }
1552 // GH #21 Phase 2: the Step tier's resolved context bundle (from
1553 // `TaskSpec.step_ctx`, re-read every attempt above) — consumed by
1554 // `AgentContextMiddleware`, which unpacks its keys ahead of the
1555 // Agent / BP-global tiers.
1556 if let Some(step_ctx) = step_ctx {
1557 ctx.meta.runtime.insert(STEP_CTX_KEY.to_string(), step_ctx);
1558 }
1559
1560 let worker = spawner
1561 .spawn(self, &ctx, task_id.clone(), attempt, worker_token)
1562 .await
1563 .map_err(|e| EngineError::DispatchFailed(e.to_string()))?;
1564
1565 // 3) Outside the lock: await worker.join() (signal-only). WorkerError is
1566 // stringified. The value is fetched via output_tail (sink path).
1567 let signal_result: Result<(), String> = worker.join().await.map_err(|e| e.to_string());
1568
1569 // Pull the last Final from output_tail and use it as the value. GH
1570 // #36 ST1 (named multi-part worker output): also fold every
1571 // `Artifact` the WORKER ITSELF staged on the same tail (via
1572 // `stage_worker_artifact_trusted` / `POST /v1/worker/artifact`)
1573 // into a `"parts"` object keyed by name — event order,
1574 // last-write-wins per name (a name staged twice overwrites,
1575 // mirroring `HashMap`/`Map` insert semantics, not an accumulating
1576 // list). `worker_artifact_names_for` is the allowlist that scopes
1577 // this to the worker's own opt-in parts — an `Artifact` some OTHER
1578 // producer appended to this same tail (e.g.
1579 // `AfterRunAuditMiddleware`'s `"audit:<step_ref>"` sidecar finding)
1580 // is left untouched (see `fold_final_and_parts`'s doc). When at
1581 // least one part was staged, the BP-chain value becomes `{"out":
1582 // <final value>, "parts": {...}}`; zero parts staged (the
1583 // pre-GH-#36 case, and every non-opt-in step) leaves the value
1584 // exactly the plain `Final` value, byte-identical to before this
1585 // change.
1586 let value_ok: Result<(Value, bool), String> = match signal_result {
1587 Ok(()) => {
1588 let tail = self.output_tail(&task_id, attempt).await;
1589 let staged_names = self.worker_artifact_names_for(&task_id, attempt).await;
1590 fold_final_and_parts(&tail, &staged_names)
1591 .ok_or_else(|| "no Final in output_tail".to_string())
1592 }
1593 Err(msg) => Err(msg),
1594 };
1595
1596 // 4) Under the lock: apply (split the borrow scope so push_event and task mut can co-exist).
1597 let outcome = self
1598 .with_state("dispatch.apply", |s| {
1599 if !s.tasks.contains_key(&task_id) {
1600 return Err(EngineError::TaskNotFound(task_id.to_string()));
1601 }
1602 match value_ok {
1603 Ok((value, ok)) => {
1604 // GH #76 Skip tier: a Final with ok=true carrying the
1605 // skip-marker sentinel is a Skip tier completion,
1606 // not an ordinary Pass. TaskStatus stays `Pass`
1607 // (the worker itself completed successfully);
1608 // the Skip signal rides on DispatchOutcome so
1609 // EngineDispatcher::dispatch can route it to
1610 // the flow-continuation-without-binding-write
1611 // sentinel path.
1612 let skip_inner = if ok { unwrap_skip_marker(&value) } else { None };
1613 let pass = ok;
1614 {
1615 let task = s.tasks.get_mut(&task_id).unwrap();
1616 task.last_result = Some(value.clone());
1617 task.updated_at = now_unix();
1618 task.status = if pass {
1619 TaskStatus::Pass
1620 } else {
1621 TaskStatus::Blocked
1622 };
1623 }
1624 s.push_event(Event::TaskAttemptCompleted {
1625 task_id: task_id.clone(),
1626 attempt,
1627 result: value.clone(),
1628 });
1629 if let Some(inner) = skip_inner {
1630 s.push_event(Event::TaskPass {
1631 task_id: task_id.clone(),
1632 result: value.clone(),
1633 });
1634 Ok::<_, EngineError>(DispatchOutcome::Skip(inner))
1635 } else if pass {
1636 s.push_event(Event::TaskPass {
1637 task_id: task_id.clone(),
1638 result: value.clone(),
1639 });
1640 Ok::<_, EngineError>(DispatchOutcome::Pass(value))
1641 } else {
1642 s.push_event(Event::TaskBlocked {
1643 task_id: task_id.clone(),
1644 result: value.clone(),
1645 });
1646 Ok(DispatchOutcome::Blocked(value))
1647 }
1648 }
1649 Err(msg) => {
1650 let task = s.tasks.get_mut(&task_id).unwrap();
1651 task.status = TaskStatus::Blocked;
1652 task.updated_at = now_unix();
1653 Err(EngineError::DispatchFailed(msg))
1654 }
1655 }
1656 })
1657 .await??;
1658
1659 // event broadcast (outside the lock — push_event feeds the in-memory tail; broadcast is a separate path).
1660 let _ = self.inner.event_tx.send(Event::TaskAttemptCompleted {
1661 task_id: task_id.clone(),
1662 attempt,
1663 result: match &outcome {
1664 DispatchOutcome::Pass(v)
1665 | DispatchOutcome::Blocked(v)
1666 | DispatchOutcome::Skip(v) => v.clone(),
1667 _ => Value::Null,
1668 },
1669 });
1670
1671 // Wake any callers waiting in poll_task.
1672 self.wake_task(&task_id).await?;
1673
1674 Ok(outcome)
1675 }
1676
1677 /// Dispatch a single attempt, opt-in to the replay-log Core primitive
1678 /// ([`crate::store::replay`]) via `run_ctx`.
1679 ///
1680 /// This is the [`Self::dispatch_attempt_with`] sibling used by callers
1681 /// that carry a `RunContext` with `replay_store` / `replay_cursor`
1682 /// populated. Behavior versus the plain `dispatch_attempt_with`:
1683 ///
1684 /// - **`run_ctx.replay_cursor` is `Some` AND the cursor has a matching
1685 /// `(step_ref, input_hash, occurrence)` row** — the stored value is
1686 /// returned verbatim as `DispatchOutcome::Pass(v)`; the `Adapter`
1687 /// (spawner + worker) is never touched. The task's `attempt` is
1688 /// still bumped and `TaskStatus` set to `Pass`, so downstream state
1689 /// (`task.last_result`, `TaskAttemptCompleted` / `TaskPass` events,
1690 /// `wake_task`) fires the same way an ordinary Pass would.
1691 /// - **Miss (or `replay_cursor: None`)** — the ordinary spawn path
1692 /// runs. When `run_ctx.replay_store` is `Some` AND the outcome is
1693 /// `Pass`, one `ReplayEntry` is appended carrying the whole `Ctx`
1694 /// snapshot (with `operator` dropped by `#[serde(skip)]`) plus the
1695 /// `step_output` value. `Blocked` / `Err` outcomes are never
1696 /// logged — a partial-failure row would poison the replay path
1697 /// after a subsequent successful retry.
1698 ///
1699 /// `run_ctx: None` collapses to the same behavior as
1700 /// `dispatch_attempt_with(token, task_id, spawner, None)` — no run
1701 /// tracing, no replay.
1702 pub async fn dispatch_attempt_with_run_ctx(
1703 &self,
1704 token: &CapToken,
1705 task_id: &StepId,
1706 spawner: &Arc<dyn SpawnerAdapter>,
1707 run_ctx: Option<&RunContext>,
1708 ) -> Result<DispatchOutcome, EngineError> {
1709 self.verify_token(token, Verb::DispatchAttempt).await?;
1710 let task_id = task_id.clone();
1711
1712 // 1) Under the lock: prep (bump attempt, snapshot agent/directive).
1713 let fp = token.fingerprint();
1714 let tid_for_prep = task_id.clone();
1715 let (attempt, agent, session_snapshot, step_ctx, initial_directive) = self
1716 .with_state("dispatch_run_ctx.prep", move |s| {
1717 let task = s
1718 .tasks
1719 .get_mut(&tid_for_prep)
1720 .ok_or_else(|| EngineError::TaskNotFound(tid_for_prep.to_string()))?;
1721 task.attempt += 1;
1722 task.status = TaskStatus::Running;
1723 task.updated_at = now_unix();
1724 let attempt = task.attempt;
1725 let initial = task.spec.initial_directive.clone();
1726 s.prompts
1727 .entry((tid_for_prep.clone(), attempt))
1728 .or_insert(initial.clone());
1729 let task = s
1730 .tasks
1731 .get(&tid_for_prep)
1732 .ok_or_else(|| EngineError::TaskNotFound(tid_for_prep.to_string()))?;
1733 let agent = task.spec.agent.clone();
1734 let step_ctx = task.spec.step_ctx.clone();
1735 let sess_clone = s
1736 .sessions
1737 .values()
1738 .find(|sess| sess.token_fp == fp)
1739 .cloned();
1740 Ok::<_, EngineError>((attempt, agent, sess_clone, step_ctx, initial))
1741 })
1742 .await??;
1743
1744 let operator_info = match session_snapshot {
1745 Some(sess) => self.resolve_operator_info(&sess, &agent).await,
1746 None => OperatorInfo::default(),
1747 };
1748
1749 // 2) Compute the replay key from step_ref (= agent) + hashed input.
1750 // Occurrence comes from the cursor's per-key counter (bumped
1751 // once per dispatch, so a loop that re-visits the same step
1752 // with the same input gets 0, 1, 2, … distinct rows).
1753 let step_ref = agent.clone();
1754 let input_hash = match run_ctx.and_then(|rc| rc.binding_digests.get(&step_ref)) {
1755 Some(binding_digest) => hash_input_value(&serde_json::json!({
1756 "input": initial_directive,
1757 "binding_digest": binding_digest,
1758 })),
1759 None => hash_input_value(&initial_directive),
1760 };
1761 let (replay_hit_value, occurrence) = if let Some(rc) = run_ctx {
1762 if let Some(cursor) = &rc.replay_cursor {
1763 let mut guard = cursor.lock().expect("replay cursor mutex poisoned");
1764 let occ = guard.next_occurrence(&step_ref, &input_hash);
1765 let hit = guard.find(&step_ref, &input_hash, occ);
1766 (hit, occ)
1767 } else {
1768 (None, 0)
1769 }
1770 } else {
1771 (None, 0)
1772 };
1773
1774 // 3) Build the Ctx that (a) either the spawner will see on a miss,
1775 // or (b) we log alongside the replay row.
1776 let mut ctx = Ctx::new(task_id.clone(), attempt, agent.clone());
1777 ctx.operator = operator_info;
1778 if let Some(rc) = run_ctx {
1779 ctx.meta
1780 .runtime
1781 .insert(RUN_ID_KEY.to_string(), Value::String(rc.run_id.to_string()));
1782 }
1783 if let Some(step_ctx) = step_ctx {
1784 ctx.meta.runtime.insert(STEP_CTX_KEY.to_string(), step_ctx);
1785 }
1786
1787 // 4) Replay-hit shortcut: skip the spawn+join, return stored value.
1788 let was_replay_hit = replay_hit_value.is_some();
1789 let value_ok: Result<(Value, bool), String> = if let Some(stored) = replay_hit_value {
1790 tracing::info!(
1791 task_id = %task_id,
1792 step_ref = %step_ref,
1793 occurrence = occurrence,
1794 "replayed from log; worker dispatch skipped"
1795 );
1796 Ok((stored, true))
1797 } else {
1798 // 5) Ordinary spawn path — mint a worker token+handle, run the
1799 // spawner, join, and pull the last Final from output_tail.
1800 let worker_token = self.inner.signer.session(
1801 format!("worker-of-{task_id}"),
1802 Role::Worker,
1803 vec!["*".into()],
1804 Duration::from_secs(1800),
1805 );
1806 let worker_fp = worker_token.fingerprint();
1807 let task_id_for_worker = task_id.clone();
1808 let worker_token_for_store = worker_token.clone();
1809 self.with_state("dispatch_run_ctx.mint_worker", move |s| {
1810 s.tokens.insert(
1811 worker_fp,
1812 CapTokenRecord::from_worker_token(worker_token_for_store, task_id_for_worker),
1813 );
1814 })
1815 .await?;
1816 let worker_handle = self.mint_worker_handle(worker_token.fingerprint()).await?;
1817 ctx.meta
1818 .runtime
1819 .insert("worker_handle".to_string(), Value::String(worker_handle));
1820
1821 let worker = spawner
1822 .spawn(self, &ctx, task_id.clone(), attempt, worker_token)
1823 .await
1824 .map_err(|e| EngineError::DispatchFailed(e.to_string()))?;
1825 let signal_result: Result<(), String> = worker.join().await.map_err(|e| e.to_string());
1826 match signal_result {
1827 Ok(()) => {
1828 let tail = self.output_tail(&task_id, attempt).await;
1829 let staged_names = self.worker_artifact_names_for(&task_id, attempt).await;
1830 fold_final_and_parts(&tail, &staged_names)
1831 .ok_or_else(|| "no Final in output_tail".to_string())
1832 }
1833 Err(msg) => Err(msg),
1834 }
1835 };
1836
1837 // 6) Apply — mirrors `dispatch_attempt_with`'s apply arm exactly
1838 // (task.last_result / status update + TaskAttemptCompleted /
1839 // TaskPass / TaskBlocked events).
1840 let outcome = self
1841 .with_state("dispatch_run_ctx.apply", |s| {
1842 if !s.tasks.contains_key(&task_id) {
1843 return Err(EngineError::TaskNotFound(task_id.to_string()));
1844 }
1845 match value_ok {
1846 Ok((value, ok)) => {
1847 // GH #76 Skip tier: Skip tier detection — same shape
1848 // as the sibling `dispatch_attempt_with` apply
1849 // arm above. See that arm's comment for the
1850 // TaskStatus / Event / DispatchOutcome contract.
1851 let skip_inner = if ok { unwrap_skip_marker(&value) } else { None };
1852 let pass = ok;
1853 {
1854 let task = s.tasks.get_mut(&task_id).unwrap();
1855 task.last_result = Some(value.clone());
1856 task.updated_at = now_unix();
1857 task.status = if pass {
1858 TaskStatus::Pass
1859 } else {
1860 TaskStatus::Blocked
1861 };
1862 }
1863 s.push_event(Event::TaskAttemptCompleted {
1864 task_id: task_id.clone(),
1865 attempt,
1866 result: value.clone(),
1867 });
1868 if let Some(inner) = skip_inner {
1869 s.push_event(Event::TaskPass {
1870 task_id: task_id.clone(),
1871 result: value.clone(),
1872 });
1873 Ok::<_, EngineError>(DispatchOutcome::Skip(inner))
1874 } else if pass {
1875 s.push_event(Event::TaskPass {
1876 task_id: task_id.clone(),
1877 result: value.clone(),
1878 });
1879 Ok::<_, EngineError>(DispatchOutcome::Pass(value))
1880 } else {
1881 s.push_event(Event::TaskBlocked {
1882 task_id: task_id.clone(),
1883 result: value.clone(),
1884 });
1885 Ok(DispatchOutcome::Blocked(value))
1886 }
1887 }
1888 Err(msg) => {
1889 let task = s.tasks.get_mut(&task_id).unwrap();
1890 task.status = TaskStatus::Blocked;
1891 task.updated_at = now_unix();
1892 Err(EngineError::DispatchFailed(msg))
1893 }
1894 }
1895 })
1896 .await??;
1897
1898 // 7) On MISS + Pass + replay_store present, append a replay row.
1899 // Replay-HIT rows are already logged from the original run and
1900 // must never be double-logged (Core primitive contract). A
1901 // secondary-persistence failure here (`tracing::warn!` +
1902 // swallow) matches the `run_ctx.run_store.append_step_entry`
1903 // convention in `EngineDispatcher::dispatch`: it must not mask
1904 // the primary dispatch outcome the caller already has in hand.
1905 if !was_replay_hit {
1906 if let (Some(rc), DispatchOutcome::Pass(v)) = (run_ctx, &outcome) {
1907 if let Some(store) = &rc.replay_store {
1908 match ReplayEntry::from_completion(
1909 rc.run_id.clone(),
1910 step_ref.clone(),
1911 input_hash.clone(),
1912 occurrence,
1913 &ctx,
1914 v,
1915 ) {
1916 Ok(entry) => {
1917 if let Err(e) = store.append(entry).await {
1918 tracing::warn!(
1919 run_id = %rc.run_id,
1920 step_ref = %step_ref,
1921 occurrence = occurrence,
1922 error = %e,
1923 "dispatch_attempt_with_run_ctx: replay_store.append failed"
1924 );
1925 }
1926 }
1927 Err(e) => {
1928 tracing::warn!(
1929 run_id = %rc.run_id,
1930 step_ref = %step_ref,
1931 occurrence = occurrence,
1932 error = %e,
1933 "dispatch_attempt_with_run_ctx: ReplayEntry encode failed"
1934 );
1935 }
1936 }
1937 }
1938 }
1939 }
1940
1941 let _ = self.inner.event_tx.send(Event::TaskAttemptCompleted {
1942 task_id: task_id.clone(),
1943 attempt,
1944 result: match &outcome {
1945 DispatchOutcome::Pass(v)
1946 | DispatchOutcome::Blocked(v)
1947 | DispatchOutcome::Skip(v) => v.clone(),
1948 _ => Value::Null,
1949 },
1950 });
1951
1952 self.wake_task(&task_id).await?;
1953
1954 Ok(outcome)
1955 }
1956
1957 // ═══════════════════════════════════════════════════════════════════════
1958 // Worker-side API (= prompt / data fetch + result post)
1959 // ═══════════════════════════════════════════════════════════════════════
1960
1961 /// Fetch the directive/prompt `Value` for `task_id`'s current attempt.
1962 /// Falls back to `initial_directive` when no prompt has been recorded
1963 /// yet for that attempt. Returns the `Value` end-to-end (issue #18);
1964 /// the render down to `String` happens only at the two consumer
1965 /// boundaries — the Worker HTTP path (`fetch_worker_payload*` →
1966 /// `WorkerPayload.prompt: String`) and the WS Spawn frame text
1967 /// render (`operator_ws::session`).
1968 pub async fn fetch_prompt(
1969 &self,
1970 token: &CapToken,
1971 task_id: &StepId,
1972 ) -> Result<Value, EngineError> {
1973 self.verify_token_for_task(token, Verb::FetchPrompt, task_id)
1974 .await?;
1975 let task_id = task_id.clone();
1976 self.with_state("fetch_prompt", move |s| {
1977 let task = s
1978 .tasks
1979 .get(&task_id)
1980 .ok_or_else(|| EngineError::TaskNotFound(task_id.to_string()))?;
1981 s.prompts
1982 .get(&(task_id.clone(), task.attempt.max(1)))
1983 .cloned()
1984 .ok_or_else(|| {
1985 EngineError::ResourceNotFound(format!(
1986 "prompt({}, attempt={})",
1987 task_id, task.attempt
1988 ))
1989 })
1990 })
1991 .await?
1992 }
1993
1994 /// Combined fetch for `HTTP /v1/worker/prompt`: returns `prompt` +
1995 /// (optional) `system` + `agent` + `attempt` in a single round trip.
1996 /// The verb gate reuses `FetchPrompt` — same semantics as "the worker
1997 /// pulls its task input".
1998 ///
1999 /// `system` is the value written by `OperatorSpawner::spawn` through
2000 /// `bake_worker_system_prompt` when it ran; otherwise `None` (no
2001 /// profile present, or the bake never happened).
2002 pub async fn fetch_worker_payload(
2003 &self,
2004 token: &CapToken,
2005 task_id: &StepId,
2006 ) -> Result<crate::types::WorkerPayload, EngineError> {
2007 self.verify_token_for_task(token, Verb::FetchPrompt, task_id)
2008 .await?;
2009 let task_id_clone = task_id.clone();
2010 let mut payload = self
2011 .with_state("fetch_worker_payload", move |s| {
2012 let task = s
2013 .tasks
2014 .get(&task_id_clone)
2015 .ok_or_else(|| EngineError::TaskNotFound(task_id_clone.to_string()))?;
2016 let attempt = task.attempt.max(1);
2017 let prompt = s
2018 .prompts
2019 .get(&(task_id_clone.clone(), attempt))
2020 .cloned()
2021 .ok_or_else(|| {
2022 EngineError::ResourceNotFound(format!(
2023 "prompt({}, attempt={})",
2024 task_id_clone, attempt
2025 ))
2026 })?;
2027 let system = s
2028 .systems
2029 .get(&(task_id_clone.clone(), attempt))
2030 .cloned()
2031 .unwrap_or(None);
2032 let agent = task.spec.agent.clone();
2033 let context = s
2034 .agent_ctx
2035 .get(&(task_id_clone.clone(), attempt))
2036 .map(|e| e.view.clone());
2037 Ok::<_, EngineError>(crate::types::WorkerPayload {
2038 task_id: task_id_clone.clone(),
2039 attempt,
2040 agent,
2041 prompt: render_directive_to_string(&prompt),
2042 system,
2043 context,
2044 system_ref: None,
2045 })
2046 })
2047 .await??;
2048 self.apply_system_ref_threshold(&mut payload).await?;
2049 Ok(payload)
2050 }
2051
2052 /// Fetch a worker payload via a short handle. Skips token verification
2053 /// and returns `prompt` + `system` + `agent` + `attempt` in a thin
2054 /// path. The caller is expected to have already resolved `task_id`
2055 /// via `task_id_from_handle` — the handle's presence in
2056 /// `worker_handles` means it was minted server-side and is therefore
2057 /// trusted.
2058 pub async fn fetch_worker_payload_trusted(
2059 &self,
2060 task_id: &StepId,
2061 ) -> Result<crate::types::WorkerPayload, EngineError> {
2062 let task_id_clone = task_id.clone();
2063 let mut payload = self
2064 .with_state("fetch_worker_payload_trusted", move |s| {
2065 let task = s
2066 .tasks
2067 .get(&task_id_clone)
2068 .ok_or_else(|| EngineError::TaskNotFound(task_id_clone.to_string()))?;
2069 let attempt = task.attempt.max(1);
2070 let prompt = s
2071 .prompts
2072 .get(&(task_id_clone.clone(), attempt))
2073 .cloned()
2074 .ok_or_else(|| {
2075 EngineError::ResourceNotFound(format!(
2076 "prompt({}, attempt={})",
2077 task_id_clone, attempt
2078 ))
2079 })?;
2080 let system = s
2081 .systems
2082 .get(&(task_id_clone.clone(), attempt))
2083 .cloned()
2084 .unwrap_or(None);
2085 let agent = task.spec.agent.clone();
2086 let context = s
2087 .agent_ctx
2088 .get(&(task_id_clone.clone(), attempt))
2089 .map(|e| e.view.clone());
2090 Ok::<_, EngineError>(crate::types::WorkerPayload {
2091 task_id: task_id_clone.clone(),
2092 attempt,
2093 agent,
2094 prompt: render_directive_to_string(&prompt),
2095 system,
2096 context,
2097 system_ref: None,
2098 })
2099 })
2100 .await??;
2101 self.apply_system_ref_threshold(&mut payload).await?;
2102 Ok(payload)
2103 }
2104
2105 /// GH #31: shared threshold-branch tail for
2106 /// [`Self::fetch_worker_payload`] / [`Self::fetch_worker_payload_trusted`].
2107 /// Both build a raw `WorkerPayload` inside `with_state` with `system`
2108 /// populated as before and `system_ref: None`; this runs *outside* any
2109 /// lock (R3 — `SystemRefMode::File`'s `tokio::fs` write is a genuine
2110 /// `.await`, which `with_state`'s sync-closure contract forbids inside
2111 /// the lock) and rewrites `payload.system` / `payload.system_ref` in
2112 /// place per `SystemRefConfig.threshold_bytes`: over-threshold clears
2113 /// `system` and populates `system_ref`; at-or-under-threshold leaves
2114 /// `system` as-is and `system_ref` stays `None`. A no-op when
2115 /// `payload.system` is already `None` (no `system_prompt` was baked).
2116 async fn apply_system_ref_threshold(
2117 &self,
2118 payload: &mut crate::types::WorkerPayload,
2119 ) -> Result<(), EngineError> {
2120 let Some(rendered) = payload.system.take() else {
2121 return Ok(());
2122 };
2123 let cfg = self.cfg().system_ref.clone();
2124 if rendered.len() <= cfg.threshold_bytes {
2125 payload.system = Some(rendered);
2126 return Ok(());
2127 }
2128 use sha2::Digest;
2129 let size_bytes = rendered.len() as u64;
2130 let sha256 = hex::encode(sha2::Sha256::digest(rendered.as_bytes()));
2131 let task_id = &payload.task_id;
2132 let attempt = payload.attempt;
2133 let system_ref = match cfg.mode {
2134 crate::types::SystemRefMode::Http => crate::types::SystemRef {
2135 // The engine has no knowledge of scheme/host here — see
2136 // `SystemRefMode::Http`'s doc for who fills that in.
2137 uri: format!("/v1/worker/prompt/system?task_id={task_id}&attempt={attempt}"),
2138 sha256,
2139 size_bytes,
2140 mode: crate::types::SystemRefMode::Http,
2141 },
2142 crate::types::SystemRefMode::File => {
2143 tokio::fs::create_dir_all(&cfg.store_dir).await?;
2144 let path = cfg.store_dir.join(format!("{task_id}-{attempt}.md"));
2145 tokio::fs::write(&path, rendered.as_bytes()).await?;
2146 crate::types::SystemRef {
2147 uri: format!("file://{}", path.display()),
2148 sha256,
2149 size_bytes,
2150 mode: crate::types::SystemRefMode::File,
2151 }
2152 }
2153 };
2154 payload.system = None;
2155 payload.system_ref = Some(system_ref);
2156 Ok(())
2157 }
2158
2159 /// GH #83: unconditionally materialize the baked system prompt for
2160 /// `(task_id, attempt)` to a file and return its path — the value
2161 /// source of the `{system_file}` placeholder in a `SubprocessDef`
2162 /// template. Unlike [`Self::apply_system_ref_threshold`] (whose
2163 /// `SystemRefMode::File` write only fires over
2164 /// `SystemRefConfig.threshold_bytes`, a behavior this helper does NOT
2165 /// touch), a template that names `{system_file}` needs a real path
2166 /// regardless of size, so the write here is unconditional. Reuses the
2167 /// same store dir and `{task_id}-{attempt}.md` naming as the File
2168 /// mode, so both paths converge on one on-disk identity per attempt.
2169 ///
2170 /// `Ok(None)` = no system prompt was baked for this attempt (the
2171 /// caller decides whether that is fail-loud — the Subprocess spawn
2172 /// path treats a `{system_file}` reference without a baked system as
2173 /// a `SpawnError`).
2174 pub async fn materialize_system_file(
2175 &self,
2176 task_id: &StepId,
2177 attempt: u32,
2178 ) -> Result<Option<std::path::PathBuf>, EngineError> {
2179 let key = (task_id.clone(), attempt);
2180 let rendered = self
2181 .with_state("materialize_system_file", move |s| {
2182 s.systems.get(&key).cloned().unwrap_or(None)
2183 })
2184 .await?;
2185 let Some(rendered) = rendered else {
2186 return Ok(None);
2187 };
2188 let cfg = self.cfg().system_ref.clone();
2189 tokio::fs::create_dir_all(&cfg.store_dir).await?;
2190 let path = cfg.store_dir.join(format!("{task_id}-{attempt}.md"));
2191 tokio::fs::write(&path, rendered.as_bytes()).await?;
2192 Ok(Some(path))
2193 }
2194
2195 /// Returns the effective [`mlua_swarm_schema::ContextPolicy`]
2196 /// `AgentContextMiddleware` resolved and snapshotted for `(task_id,
2197 /// attempt)` at spawn time (the same policy already applied to that
2198 /// key's `EngineState.agent_ctx` entry's `.view`, GH #23 fold).
2199 /// Pass-all (`ContextPolicy::default()`) when no entry exists — either
2200 /// a pre-ST5 spawn, or a spawner stack that never layered
2201 /// `AgentContextMiddleware` (fail-open, mirroring [`Self::output_tail`]'s
2202 /// "no entry = empty default" convention).
2203 ///
2204 /// `crates/mlua-swarm-server/src/worker.rs`'s `GET /v1/worker/prompt`
2205 /// handler reads this back to filter `WorkerPayload.context.steps` via
2206 /// `ContextPolicy::allows_step`, without re-deriving the policy from
2207 /// the Blueprint at fetch time (`projection-adapter` ST5).
2208 pub async fn context_policy_for(
2209 &self,
2210 task_id: &StepId,
2211 attempt: u32,
2212 ) -> mlua_swarm_schema::ContextPolicy {
2213 let key = (task_id.clone(), attempt);
2214 self.with_state("context_policy_for", move |s| {
2215 s.agent_ctx
2216 .get(&key)
2217 .map(|e| e.policy.clone())
2218 .unwrap_or_default()
2219 })
2220 .await
2221 .unwrap_or_default()
2222 }
2223
2224 /// GH #23: returns the Blueprint-wide
2225 /// [`crate::core::step_naming::StepNaming`] table snapshotted for
2226 /// `task_id` (the same `Arc` `crate::blueprint::EngineDispatcher::dispatch`
2227 /// stashed into `EngineState.step_namings` at dispatch time —
2228 /// `Self::start_task`'s `StepId`, not the `TaskId` work item). `None`
2229 /// when no entry exists — either the dispatcher was never given a
2230 /// `StepNaming` (`EngineDispatcher::with_step_naming` not called) or
2231 /// the lock could not be acquired; callers are expected to fall back
2232 /// to the pre-GH-#23 runtime union rule in that case (subtask-2/3
2233 /// consumers).
2234 pub async fn step_naming_for(
2235 &self,
2236 task_id: &StepId,
2237 ) -> Option<Arc<crate::core::step_naming::StepNaming>> {
2238 let key = task_id.clone();
2239 self.with_state("step_naming_for", move |s| {
2240 s.step_namings.get(&key).cloned()
2241 })
2242 .await
2243 .ok()
2244 .flatten()
2245 }
2246
2247 /// GH #27 (follow-up to #23): returns the Blueprint-wide
2248 /// [`crate::core::projection_placement::ProjectionPlacement`] resolver
2249 /// snapshotted for `task_id` (the same `Arc`
2250 /// `crate::blueprint::EngineDispatcher::dispatch` stashed into
2251 /// `EngineState.projection_placements` at dispatch time — mirroring
2252 /// [`Self::step_naming_for`]'s contract exactly). `None` when no entry
2253 /// exists — either the dispatcher was never given a
2254 /// `ProjectionPlacement` (`EngineDispatcher::with_projection_placement`
2255 /// not called) or the lock could not be acquired; callers are expected
2256 /// to fall back to `ProjectionPlacement::default()` (byte-compat with
2257 /// the pre-#27 hardcoded layout) in that case.
2258 pub async fn projection_placement_for(
2259 &self,
2260 task_id: &StepId,
2261 ) -> Option<Arc<crate::core::projection_placement::ProjectionPlacement>> {
2262 let key = task_id.clone();
2263 self.with_state("projection_placement_for", move |s| {
2264 s.projection_placements.get(&key).cloned()
2265 })
2266 .await
2267 .ok()
2268 .flatten()
2269 }
2270
2271 /// Record normalized per-attempt worker stats reported by a worker
2272 /// boundary (spawner fold site / result captor / `POST
2273 /// /v1/worker/submit`). Last-write-wins per `(task_id, attempt)`.
2274 /// Best-effort: a state-lock failure is logged and swallowed —
2275 /// stats are observational and must never fail the attempt that
2276 /// produced them. Drained by [`Self::take_worker_stats`] at the
2277 /// dispatcher's outcome fold.
2278 pub async fn record_worker_stats(
2279 &self,
2280 task_id: &StepId,
2281 attempt: u32,
2282 stats: crate::store::trace::WorkerStats,
2283 ) {
2284 if stats.is_empty() {
2285 return;
2286 }
2287 let key = (task_id.clone(), attempt);
2288 if let Err(e) = self
2289 .with_state("record_worker_stats", move |s| {
2290 s.worker_stats.insert(key, stats);
2291 })
2292 .await
2293 {
2294 tracing::warn!(
2295 task_id = %task_id,
2296 attempt,
2297 error = %e,
2298 "record_worker_stats failed (swallowed — stats are observational)"
2299 );
2300 }
2301 }
2302
2303 /// Drain every recorded worker-stats entry for `task_id`, returning
2304 /// the highest-attempt one (the attempt whose outcome the dispatcher
2305 /// is folding). Removing ALL of the task's entries — not just the
2306 /// returned one — keeps retries from leaking earlier attempts into
2307 /// `EngineState` for the process lifetime.
2308 pub async fn take_worker_stats(
2309 &self,
2310 task_id: &StepId,
2311 ) -> Option<(u32, crate::store::trace::WorkerStats)> {
2312 let key_task = task_id.clone();
2313 self.with_state("take_worker_stats", move |s| {
2314 let attempts: Vec<u32> = s
2315 .worker_stats
2316 .keys()
2317 .filter(|(tid, _)| *tid == key_task)
2318 .map(|(_, a)| *a)
2319 .collect();
2320 let mut best: Option<(u32, crate::store::trace::WorkerStats)> = None;
2321 for attempt in attempts {
2322 if let Some(stats) = s.worker_stats.remove(&(key_task.clone(), attempt)) {
2323 if best.as_ref().map(|(a, _)| attempt >= *a).unwrap_or(true) {
2324 best = Some((attempt, stats));
2325 }
2326 }
2327 }
2328 best
2329 })
2330 .await
2331 .ok()
2332 .flatten()
2333 }
2334
2335 /// Returns the [`crate::store::trace::TraceHandle`] the dispatcher
2336 /// registered for `task_id`'s in-flight step, if any — the
2337 /// pervasive-insertion read port middlewares (and any other writer
2338 /// holding an `Engine`) use to append their own trace kinds. `None`
2339 /// = no trace rail for this dispatch (RunContext without a trace
2340 /// handle, or the step already folded).
2341 pub async fn trace_handle(&self, task_id: &StepId) -> Option<crate::store::trace::TraceHandle> {
2342 let key = task_id.clone();
2343 self.with_state("trace_handle", move |s| s.trace_handles.get(&key).cloned())
2344 .await
2345 .ok()
2346 .flatten()
2347 }
2348
2349 /// Register (or clear, with `None`) the per-dispatch trace handle
2350 /// for `task_id`. Called only by `EngineDispatcher::dispatch` —
2351 /// insert before spawn, clear after the outcome fold. Best-effort:
2352 /// registry failures are swallowed (trace is observational).
2353 pub(crate) async fn set_trace_handle(
2354 &self,
2355 task_id: &StepId,
2356 handle: Option<crate::store::trace::TraceHandle>,
2357 ) {
2358 let key = task_id.clone();
2359 let _ = self
2360 .with_state("set_trace_handle", move |s| match handle {
2361 Some(h) => {
2362 s.trace_handles.insert(key, h);
2363 }
2364 None => {
2365 s.trace_handles.remove(&key);
2366 }
2367 })
2368 .await;
2369 }
2370
2371 /// Returns the [`crate::core::agent_context::AgentContextView`]
2372 /// snapshotted for `(task_id, attempt)`, if `AgentContextMiddleware`
2373 /// stashed one — the same lookup [`Self::fetch_worker_payload`] /
2374 /// [`Self::fetch_worker_payload_trusted`] perform inline, exposed
2375 /// standalone for callers that only need the view (not a full
2376 /// `WorkerPayload`) — e.g. the HTTP debug-plane `GET
2377 /// /v1/tasks/:id/runs/:run/steps*` handlers resolving a
2378 /// materialized-file root for a step *other than* the one currently
2379 /// fetching its own prompt (`projection-adapter` ST5).
2380 pub async fn agent_context_for(
2381 &self,
2382 task_id: &StepId,
2383 attempt: u32,
2384 ) -> Option<crate::core::agent_context::AgentContextView> {
2385 let key = (task_id.clone(), attempt);
2386 self.with_state("agent_context_for", move |s| {
2387 s.agent_ctx.get(&key).map(|e| e.view.clone())
2388 })
2389 .await
2390 .ok()
2391 .flatten()
2392 }
2393
2394 /// Read the current attempt number for a task (server-side lookup, no
2395 /// token verification). Used on `HTTP /v1/worker/result` when the
2396 /// worker omits `attempt` and the server has to fill it in.
2397 pub async fn task_attempt(&self, task_id: &StepId) -> Result<u32, EngineError> {
2398 let task_id = task_id.clone();
2399 self.with_state("task_attempt", move |s| {
2400 s.tasks
2401 .get(&task_id)
2402 .map(|t| t.attempt)
2403 .ok_or_else(|| EngineError::TaskNotFound(task_id.to_string()))
2404 })
2405 .await?
2406 }
2407
2408 /// Server-side admin API that lets `OperatorSpawner::spawn` bake the
2409 /// rendered `system_prompt` into engine state. There is no verb gate
2410 /// — the only expected caller is inside the spawner. SubAgents fetch
2411 /// this alongside the prompt on the `/v1/worker/prompt` path.
2412 pub async fn bake_worker_system_prompt(
2413 &self,
2414 task_id: &StepId,
2415 attempt: u32,
2416 system: Option<String>,
2417 ) -> Result<(), EngineError> {
2418 let task_id = task_id.clone();
2419 self.with_state("bake_worker_system_prompt", move |s| {
2420 // GH #31: record this agent's most-recently-baked render size
2421 // before `system` is moved into `s.systems.insert` below. Same
2422 // `s.tasks.get(&task_id)` → `.spec.agent` lookup pattern
2423 // `fetch_worker_payload` uses (see its doc for why this keying
2424 // is load-bearing for a later `bp_doctor` route).
2425 if let Some(rendered) = system.as_ref() {
2426 if let Some(agent) = s.tasks.get(&task_id).map(|t| t.spec.agent.clone()) {
2427 s.agent_render_sizes.insert(agent, rendered.len());
2428 }
2429 }
2430 s.systems.insert((task_id, attempt), system);
2431 })
2432 .await?;
2433 Ok(())
2434 }
2435
2436 /// GH #31: the most-recently-baked `system_prompt` render size (in
2437 /// bytes) observed for `agent_name`, if `bake_worker_system_prompt` has
2438 /// ever recorded one — last-write-wins across every `(task_id,
2439 /// attempt)` dispatch of that agent. `None` when no `system_prompt`
2440 /// has ever been baked for this agent name. Read by the `bp_doctor`
2441 /// route this subtask's follow-up adds.
2442 pub async fn agent_last_rendered_size(&self, agent_name: &str) -> Option<usize> {
2443 let agent_name = agent_name.to_string();
2444 self.with_state("agent_last_rendered_size", move |s| {
2445 s.agent_render_sizes.get(&agent_name).copied()
2446 })
2447 .await
2448 .ok()
2449 .flatten()
2450 }
2451
2452 /// GH #31: plain read-through of the baked `system` string for
2453 /// `(task_id, attempt)` from `EngineState.systems`, with no threshold
2454 /// branching. Backs `GET /v1/worker/prompt/system` (the `Http`-mode
2455 /// fetch target `system_ref.uri` points at) — that route needs the
2456 /// exact raw bytes to serve as the response body for the client's
2457 /// sha256 verification, not a `WorkerPayload`-wrapped value.
2458 ///
2459 /// Distinct from `apply_system_ref_threshold` (private, mutates an
2460 /// already-built `WorkerPayload` in place after full construction):
2461 /// this accessor has no threshold logic and is `pub` so
2462 /// `mlua-swarm-server`'s `worker` module can call it directly.
2463 ///
2464 /// Returns `Ok(None)` if no baked system exists for that `(task_id,
2465 /// attempt)` (either the task/attempt has no entry in `s.systems`, or
2466 /// the entry is present but stores `None`) — the caller maps this to
2467 /// a 404.
2468 pub async fn raw_system_prompt(
2469 &self,
2470 task_id: &StepId,
2471 attempt: u32,
2472 ) -> Result<Option<String>, EngineError> {
2473 let task_id = task_id.clone();
2474 self.with_state("raw_system_prompt", move |s| {
2475 s.systems.get(&(task_id, attempt)).cloned().unwrap_or(None)
2476 })
2477 .await
2478 }
2479
2480 /// Fetch an arbitrary named resource previously stored via
2481 /// `set_resource`. Not task-scoped — any valid token with the
2482 /// `FetchData` verb may read any key.
2483 pub async fn fetch_data(&self, token: &CapToken, key: &str) -> Result<Value, EngineError> {
2484 self.verify_token(token, Verb::FetchData).await?;
2485 let key = key.to_string();
2486 self.with_state("fetch_data", move |s| {
2487 s.resources
2488 .get(&key)
2489 .cloned()
2490 .ok_or(EngineError::ResourceNotFound(key))
2491 })
2492 .await?
2493 }
2494
2495 // ───────────────────────────────────────────────────────────────────────
2496 // Output path.
2497 // ───────────────────────────────────────────────────────────────────────
2498
2499 /// Send one output event from inside a `SpawnerAdapter` or worker.
2500 /// Structuring is assumed to be complete by the time we cross the
2501 /// `SpawnerAdapter` boundary; this API just appends to the
2502 /// `OutputStore`, pushes to the `EventLog`, and (for `Final`) emits
2503 /// the `TaskAttemptCompleted` event.
2504 ///
2505 /// This is Domain-side plumbing: it feeds the engine's verdict flow,
2506 /// not the Data-plane store in the `output_store` module. It also
2507 /// does not wake the dispatch path — that is done through the
2508 /// spawner's completion oneshot when the worker terminates.
2509 ///
2510 /// # Submit-time projection sink (subtask-4 / ST2 rework)
2511 ///
2512 /// A `Final` event additionally fans out to the submit-time projection
2513 /// sink ([`Self::materialize_final_submission`]): (a) when
2514 /// [`Self::set_output_store`] has wired a Data-plane
2515 /// [`crate::store::output::OutputStore`], the event is dual-written
2516 /// there (`producer_agent` = `TaskState.spec.agent`, resolved to its
2517 /// GH #23 canonical projection name — see below), and (b) when this
2518 /// task's spawn ran through `AgentContextMiddleware` (so
2519 /// `EngineState.agent_ctx` has a `.view.work_dir` / `.view.project_root`
2520 /// for it), the value is additionally materialized to the
2521 /// [`crate::core::projection_placement::ProjectionPlacement`]
2522 /// resolver's target (byte-compat default layout
2523 /// `<root>/workspace/tasks/<task_id>/ctx/<canonical_agent>.md`) — see
2524 /// `crate::core::projection`'s module doc.
2525 ///
2526 /// **GH #23 subtask-2 (canonical sink):** both writes above key off the
2527 /// canonical name — `Engine::step_naming_for(task_id)`'s
2528 /// `StepNaming::canonical_of_producer(producer_agent)` when a table was
2529 /// snapshotted for this task (`EngineDispatcher::with_step_naming`),
2530 /// else `producer_agent` unchanged (fail-open, byte-identical to
2531 /// pre-GH-#23 behavior — see [`crate::core::step_naming`]'s module
2532 /// doc).
2533 ///
2534 /// **Invariants** (Subtask 4): (1) this sink is fail-open — an
2535 /// unresolved root, an unconfigured `OutputStore`, or either one
2536 /// erroring, only logs a `tracing::warn!` and never turns this
2537 /// `Ok(())` into an `Err`; (2) the wired `OutputStore` stays the single
2538 /// source of truth for cross-step queries — the materialized file is a
2539 /// projection of it, not a second store; (3) core does not depend on
2540 /// `mlua-swarm-server` — everything this sink touches
2541 /// (`crate::store::output` / `crate::core::projection`) already lives
2542 /// in this crate.
2543 ///
2544 /// # `Artifact` dual-write (GH #34 subtask-3 gap fix)
2545 ///
2546 /// An `Artifact` event ALSO fans out to the Data-plane, via
2547 /// [`Self::materialize_artifact_submission`] — general-form: every
2548 /// `Artifact` submitted through this API dual-writes, no name-prefix
2549 /// gate. Unlike `Final`, the dual-write key is the artifact's own
2550 /// `name` field, verbatim — NOT resolved through the GH #23 canonical
2551 /// `StepNaming` table. An artifact's `name` IS its identity (mirrors
2552 /// [`crate::store::output::OutputStore::get_latest_by_name`]'s doc),
2553 /// so no canonicalization applies. Same fail-open discipline as
2554 /// `Final` (Invariant 1 above), but `Artifact` does NOT drive the
2555 /// file-materialize half (b) — artifact findings (e.g.
2556 /// `AfterRunAuditMiddleware`'s `"audit:<step_ref>"`) are observational
2557 /// sidecar data, not a step's own submission a work_dir/project_root
2558 /// projection needs to track. `Progress` / `Partial` events are
2559 /// unaffected — no behavior change.
2560 pub async fn submit_output(
2561 &self,
2562 token: &crate::types::CapToken,
2563 task_id: &StepId,
2564 attempt: u32,
2565 event: crate::worker::output::OutputEvent,
2566 ) -> Result<(), EngineError> {
2567 self.verify_token_for_task(token, crate::types::Verb::EmitOutput, task_id)
2568 .await?;
2569 // GH #51 — completion-time verdict-contract enforcement, embedded
2570 // choke point 2 of 2 (see `Self::verdict_contract_completion_check`'s
2571 // doc). Guarded to `Final` only — the ONLY `OutputEvent` variant a
2572 // verdict contract's completion can meaningfully address; this
2573 // guard is defensive (this function is empirically called with
2574 // `Final` only today, both from `worker.rs`'s `worker_result` and
2575 // from `operator.rs`'s WS fallback) but costs nothing and protects
2576 // against a future non-`Final` caller. Runs BEFORE the
2577 // `output_tail` write immediately below: on `Err`, this returns
2578 // immediately and the write never happens — a rejected value
2579 // never reaches `output_tail` / the flow ctx.
2580 if let crate::worker::output::OutputEvent::Final { content, ok } = &event {
2581 let comparable_value = content_ref_to_comparable_string(content.clone());
2582 self.verdict_contract_completion_check(task_id, attempt, *ok, &comparable_value)
2583 .await?;
2584 }
2585 let task_id_for_apply = task_id.clone();
2586 let event_clone = event.clone();
2587 self.with_state("submit_output", move |s| {
2588 s.output_store
2589 .entry((task_id_for_apply.clone(), attempt))
2590 .or_default()
2591 .push(event_clone.clone());
2592 s.push_event(crate::core::state::Event::WorkerOutput {
2593 task_id: task_id_for_apply,
2594 attempt,
2595 event: event_clone,
2596 });
2597 })
2598 .await?;
2599 match &event {
2600 crate::worker::output::OutputEvent::Final { content, ok } => {
2601 self.materialize_final_submission(task_id, attempt, content, *ok)
2602 .await?;
2603 }
2604 crate::worker::output::OutputEvent::Artifact { name, content } => {
2605 self.materialize_artifact_submission(task_id, attempt, name, content)
2606 .await?;
2607 }
2608 _ => {}
2609 }
2610 Ok(())
2611 }
2612
2613 /// Submit-time projection sink (subtask-4 / ST2 rework) shared by
2614 /// [`Self::submit_output`] and [`Self::submit_worker_result_trusted`].
2615 /// Best-effort / fail-open throughout (see `submit_output`'s doc
2616 /// Invariants): every failure path only `tracing::warn!`s and returns.
2617 ///
2618 /// Reads `(producer_agent, view)` via one read-only [`Self::with_state`]
2619 /// call — `producer_agent` off `TaskState.spec.agent`, `view` (the
2620 /// full [`crate::core::agent_context::AgentContextView`]) off
2621 /// `EngineState.agent_ctx[(task_id, attempt)]`, the same snapshot
2622 /// `crate::middleware::agent_context::AgentContextMiddleware` writes at
2623 /// spawn time — then does its actual (dual-write / file-write) work
2624 /// *outside* that lock, so a slow disk write or Data-plane store call
2625 /// never holds up unrelated `Engine::with_state` callers. `root` itself
2626 /// is resolved from `view` AFTER the lock via
2627 /// [`crate::core::projection_placement::ProjectionPlacement::resolve_root`]
2628 /// (GH #27, follow-up to #23) — the SAME resolver
2629 /// [`Self::step_naming_for`]'s sibling accessor
2630 /// [`Self::projection_placement_for`] snapshotted at dispatch time, so
2631 /// this sink's root-preference / fallback order is identical to the
2632 /// server read-back and the spawn-time pointer.
2633 async fn materialize_final_submission(
2634 &self,
2635 task_id: &StepId,
2636 attempt: u32,
2637 content: &crate::worker::output::ContentRef,
2638 ok: bool,
2639 ) -> Result<(), EngineError> {
2640 let server_policy = self.cfg().check_policy;
2641 let task_id_for_lookup = task_id.clone();
2642 let lookup = self
2643 .with_state("materialize_final_submission.lookup", move |s| {
2644 let entry = s.tasks.get(&task_id_for_lookup);
2645 let producer_agent = entry.map(|t| t.spec.agent.clone());
2646 let task_policy = entry.and_then(|t| t.spec.check_policy);
2647 let view = s
2648 .agent_ctx
2649 .get(&(task_id_for_lookup.clone(), attempt))
2650 .map(|e| e.view.clone());
2651 (producer_agent, task_policy, view)
2652 })
2653 .await;
2654 // Per-task `TaskSpec.check_policy` (ST1c) wins
2655 // over the server-wide `EngineCfg.check_policy` when set — a
2656 // per-run override forwarded from the launch entry point (see
2657 // `TaskLaunchRequest.check_policy` /
2658 // `TaskLaunchInput.check_policy`). `None` leaves the server
2659 // default in effect (backward compat).
2660 let policy = lookup
2661 .as_ref()
2662 .ok()
2663 .and_then(|(_, tp, _)| *tp)
2664 .unwrap_or(server_policy);
2665 let (producer_agent, view) = match lookup.map(|(pa, _, view)| (pa, view)) {
2666 Ok(pair) => pair,
2667 Err(err) => {
2668 if !matches!(policy, crate::core::config::CheckPolicy::Silent) {
2669 tracing::warn!(
2670 %task_id,
2671 error = %err,
2672 "submit-time projection sink: state lookup failed; skipping (fail-open)"
2673 );
2674 }
2675 apply_check_policy(
2676 policy,
2677 "submit-time projection sink: state lookup",
2678 "state lookup failed; skipping (fail-open)",
2679 )?;
2680 return Ok(());
2681 }
2682 };
2683 let Some(producer_agent) = producer_agent else {
2684 // Defensive only: `task_id` is always a just-looked-up task at
2685 // every real call site. No task, no addressable producer name
2686 // — nothing to project. Not gated by `CheckPolicy` — a missing
2687 // task is an intentional early-exit path, not a fail-open
2688 // condition to surface.
2689 return Ok(());
2690 };
2691 let placement = self
2692 .projection_placement_for(task_id)
2693 .await
2694 .unwrap_or_default();
2695 let root = view.and_then(|v| placement.resolve_root(&v));
2696
2697 // GH #23 subtask-2: resolve `producer_agent` to its canonical
2698 // projection name via the Blueprint-wide `StepNaming` table
2699 // snapshotted at dispatch time (`Engine::step_naming_for`). Both
2700 // write paths below ((a) data-plane, (b) file stem) use the
2701 // *canonical* name — `StepNaming::canonical_of_producer` returns
2702 // `producer_agent` unchanged for undeclared steps (byte-identical
2703 // to pre-GH-#23 behavior), and `None` (no table for this
2704 // `task_id`, e.g. a spawn that never went through
2705 // `EngineDispatcher::with_step_naming`) is a defensive fail-open
2706 // to the raw `producer_agent`, same discipline as the rest of this
2707 // sink.
2708 let canonical_agent = self
2709 .step_naming_for(task_id)
2710 .await
2711 .and_then(|naming| {
2712 naming
2713 .canonical_of_producer(&producer_agent)
2714 .map(str::to_string)
2715 })
2716 .unwrap_or_else(|| producer_agent.clone());
2717
2718 // (a) Data-plane dual-write, when an OutputStore backend is wired.
2719 if let Some(store) = self.output_store_backend() {
2720 if let Err(err) = store
2721 .append(
2722 task_id.as_str(),
2723 attempt,
2724 &canonical_agent,
2725 crate::worker::output::OutputEvent::Final {
2726 content: content.clone(),
2727 ok,
2728 },
2729 Vec::new(),
2730 )
2731 .await
2732 {
2733 if !matches!(policy, crate::core::config::CheckPolicy::Silent) {
2734 tracing::warn!(
2735 %task_id,
2736 agent = %producer_agent,
2737 canonical = %canonical_agent,
2738 error = %err,
2739 "submit-time projection sink: OutputStore dual-write failed (fail-open)"
2740 );
2741 }
2742 apply_check_policy(
2743 policy,
2744 "submit-time projection sink: OutputStore dual-write",
2745 "OutputStore dual-write failed (fail-open)",
2746 )?;
2747 }
2748 }
2749
2750 // (b) File materialize, when a root resolved.
2751 let Some(root) = root else {
2752 if !matches!(policy, crate::core::config::CheckPolicy::Silent) {
2753 tracing::warn!(
2754 %task_id,
2755 agent = %producer_agent,
2756 canonical = %canonical_agent,
2757 "submit-time projection sink: no work_dir/project_root resolved; skipping file materialize (fail-open)"
2758 );
2759 }
2760 apply_check_policy(
2761 policy,
2762 "submit-time projection sink: file materialize",
2763 "no work_dir/project_root resolved; skipping file materialize (fail-open)",
2764 )?;
2765 return Ok(());
2766 };
2767 let value = match content {
2768 crate::worker::output::ContentRef::Inline { value } => value.clone(),
2769 crate::worker::output::ContentRef::FileRef {
2770 path,
2771 mime,
2772 size_hint,
2773 } => serde_json::json!({
2774 "file_ref": path.to_string_lossy(),
2775 "mime": mime,
2776 "size_hint": size_hint,
2777 }),
2778 };
2779 let key = crate::core::projection::ProjectionKey {
2780 task_id: task_id.to_string(),
2781 run_id: None,
2782 step: Some(canonical_agent.clone()),
2783 path: None,
2784 };
2785 let adapter = crate::core::projection::FileProjectionAdapter::with_placement(
2786 root,
2787 (*placement).clone(),
2788 );
2789 if let Err(err) = adapter.materialize_submission(&key, &value, attempt, ok) {
2790 if !matches!(policy, crate::core::config::CheckPolicy::Silent) {
2791 tracing::warn!(
2792 %task_id,
2793 agent = %producer_agent,
2794 canonical = %canonical_agent,
2795 error = %err,
2796 "submit-time projection sink: file materialize failed (fail-open)"
2797 );
2798 }
2799 apply_check_policy(
2800 policy,
2801 "submit-time projection sink: file materialize",
2802 "file materialize failed (fail-open)",
2803 )?;
2804 }
2805 Ok(())
2806 }
2807
2808 /// Submit-time projection sink for `OutputEvent::Artifact` (GH #34
2809 /// subtask-3, later extended to drive the file half too). Two halves, the
2810 /// [`Self::materialize_final_submission`] mirror for staged named parts:
2811 ///
2812 /// - **Data-plane dual-write** — when [`Self::set_output_store`] has
2813 /// wired a [`crate::store::output::OutputStore`], the artifact
2814 /// dual-writes there under its own `name`, verbatim (general form:
2815 /// every `Artifact` staged via [`Self::submit_output`] /
2816 /// [`Self::stage_worker_artifact_trusted`] materializes this way, no
2817 /// name-prefix gate).
2818 /// - **File materialize** — when a `root` resolves off the spawn-time
2819 /// [`crate::core::agent_context::AgentContextView`], the part's
2820 /// content is written raw to `<ctx-dir>/<name>` via
2821 /// [`crate::core::projection::FileProjectionAdapter::materialize_part`].
2822 /// That file is the IN file the *next* Agent step reads: materializing
2823 /// a Step's OUTPUT to disk is the
2824 /// [`crate::core::projection::FileProjectionAdapter`]'s
2825 /// responsibility, and a staged named part is as much an OUTPUT the
2826 /// next step consumes as a `Final` is — so the sink materializes it
2827 /// too, rather than leaving parts Data-plane-only.
2828 ///
2829 /// Unlike the Final sink, no `StepNaming` canonicalization is applied:
2830 /// an artifact's `name` already IS the key both halves address (it
2831 /// names the file directly, extension included — `plan.md` — so
2832 /// `materialize_part` writes it verbatim, not through the `<stem>.md`
2833 /// synthesis the Final sink's canonical-agent path uses).
2834 ///
2835 /// Fail-open throughout, the same `check_policy` cascade as
2836 /// [`Self::materialize_final_submission`]: a per-task lookup error falls
2837 /// back to the server default (and a `None` view ⇒ the file half's
2838 /// unresolved-root path), an unconfigured `OutputStore` skips the
2839 /// dual-write, an unresolved root skips the file half, and a
2840 /// dual-write / file-write / name-guard error only `tracing::warn!`s
2841 /// (`Silent` suppresses even that) before applying [`apply_check_policy`]
2842 /// (`Strict` surfaces an [`EngineError`], `Warn` / `Silent` return
2843 /// `Ok(())`) — a staged part never turns a would-have-succeeded submit
2844 /// into a failure under the default policy.
2845 async fn materialize_artifact_submission(
2846 &self,
2847 task_id: &StepId,
2848 attempt: u32,
2849 name: &str,
2850 content: &crate::worker::output::ContentRef,
2851 ) -> Result<(), EngineError> {
2852 // Per-task `TaskSpec.check_policy` override + the `AgentContextView`
2853 // snapshot, resolved in ONE read-only `with_state` (the same lock
2854 // the policy lookup already needed — no extra `with_state` for the
2855 // view). Silent per-task lookup failure (`with_state` error) falls
2856 // back to the server-wide default and a `None` view (⇒ the file
2857 // half's own unresolved-root fail-open path); this sink never
2858 // surfaces the lookup error itself as a step failure.
2859 let server_policy = self.cfg().check_policy;
2860 let task_id_for_lookup = task_id.clone();
2861 let lookup = self
2862 .with_state("materialize_artifact_submission.lookup", move |s| {
2863 let task_policy = s
2864 .tasks
2865 .get(&task_id_for_lookup)
2866 .and_then(|t| t.spec.check_policy);
2867 let view = s
2868 .agent_ctx
2869 .get(&(task_id_for_lookup.clone(), attempt))
2870 .map(|e| e.view.clone());
2871 (task_policy, view)
2872 })
2873 .await
2874 .ok();
2875 let policy = lookup
2876 .as_ref()
2877 .and_then(|(tp, _)| *tp)
2878 .unwrap_or(server_policy);
2879 let view = lookup.and_then(|(_, view)| view);
2880
2881 // (a) Data-plane dual-write, when an OutputStore backend is wired —
2882 // the artifact's own `name` is its Data-plane key (no
2883 // canonicalization, unlike the Final sink's `StepNaming`
2884 // resolution).
2885 if let Some(store) = self.output_store_backend() {
2886 if let Err(err) = store
2887 .append(
2888 task_id.as_str(),
2889 attempt,
2890 name,
2891 crate::worker::output::OutputEvent::Artifact {
2892 name: name.to_string(),
2893 content: content.clone(),
2894 },
2895 Vec::new(),
2896 )
2897 .await
2898 {
2899 if !matches!(policy, crate::core::config::CheckPolicy::Silent) {
2900 tracing::warn!(
2901 %task_id,
2902 artifact = %name,
2903 error = %err,
2904 "submit-time projection sink: OutputStore dual-write failed for Artifact (fail-open)"
2905 );
2906 }
2907 apply_check_policy(
2908 policy,
2909 "submit-time projection sink: Artifact OutputStore dual-write",
2910 "OutputStore dual-write failed for Artifact (fail-open)",
2911 )?;
2912 }
2913 }
2914
2915 // (b) File materialize, when a root resolved — writes the staged
2916 // part raw to `<ctx-dir>/<name>`, the IN file the next Agent step
2917 // reads (see `FileProjectionAdapter::materialize_part`'s doc for
2918 // why raw / why the name is verbatim). A name-guard violation lands
2919 // on the same fail-open path as any other write error below.
2920 let placement = self
2921 .projection_placement_for(task_id)
2922 .await
2923 .unwrap_or_default();
2924 let Some(root) = view.and_then(|v| placement.resolve_root(&v)) else {
2925 if !matches!(policy, crate::core::config::CheckPolicy::Silent) {
2926 tracing::warn!(
2927 %task_id,
2928 artifact = %name,
2929 "submit-time projection sink: no work_dir/project_root resolved; skipping part file materialize (fail-open)"
2930 );
2931 }
2932 apply_check_policy(
2933 policy,
2934 "submit-time projection sink: part file materialize",
2935 "no work_dir/project_root resolved; skipping part file materialize (fail-open)",
2936 )?;
2937 return Ok(());
2938 };
2939 let value = match content {
2940 crate::worker::output::ContentRef::Inline { value } => value.clone(),
2941 crate::worker::output::ContentRef::FileRef {
2942 path,
2943 mime,
2944 size_hint,
2945 } => serde_json::json!({
2946 "file_ref": path.to_string_lossy(),
2947 "mime": mime,
2948 "size_hint": size_hint,
2949 }),
2950 };
2951 let adapter = crate::core::projection::FileProjectionAdapter::with_placement(
2952 root,
2953 (*placement).clone(),
2954 );
2955 if let Err(err) = adapter.materialize_part(task_id.as_str(), name, &value) {
2956 if !matches!(policy, crate::core::config::CheckPolicy::Silent) {
2957 tracing::warn!(
2958 %task_id,
2959 artifact = %name,
2960 error = %err,
2961 "submit-time projection sink: part file materialize failed (fail-open)"
2962 );
2963 }
2964 apply_check_policy(
2965 policy,
2966 "submit-time projection sink: part file materialize",
2967 "part file materialize failed (fail-open)",
2968 )?;
2969 }
2970 Ok(())
2971 }
2972
2973 /// Snapshot the entire output tail for a given `(task_id, attempt)`.
2974 /// Used by the dispatch path when pulling `Final`, and by observers
2975 /// reading the trace.
2976 pub async fn output_tail(
2977 &self,
2978 task_id: &StepId,
2979 attempt: u32,
2980 ) -> Vec<crate::worker::output::OutputEvent> {
2981 let key = (task_id.clone(), attempt);
2982 self.with_state("output_tail", move |s| {
2983 s.output_store.get(&key).cloned().unwrap_or_default()
2984 })
2985 .await
2986 .unwrap_or_default()
2987 }
2988
2989 /// Record an interim `last_result` for `task_id` without changing its
2990 /// `status`. Distinct from the terminal `Final` output event handled
2991 /// through `submit_output` / `dispatch_attempt_with`.
2992 pub async fn post_result(
2993 &self,
2994 token: &CapToken,
2995 task_id: &StepId,
2996 result: Value,
2997 ) -> Result<(), EngineError> {
2998 self.verify_token_for_task(token, Verb::PostResult, task_id)
2999 .await?;
3000 let task_id = task_id.clone();
3001 let result_clone = result.clone();
3002 self.with_state("post_result", move |s| {
3003 let task = s
3004 .tasks
3005 .get_mut(&task_id)
3006 .ok_or_else(|| EngineError::TaskNotFound(task_id.to_string()))?;
3007 task.last_result = Some(result_clone);
3008 task.updated_at = now_unix();
3009 Ok::<(), EngineError>(())
3010 })
3011 .await??;
3012 Ok(())
3013 }
3014
3015 /// Store a named resource value, retrievable later via `fetch_data`.
3016 /// No token is required — this is a server-side/admin-style setter
3017 /// (mirrors `bake_worker_system_prompt`).
3018 pub async fn set_resource(
3019 &self,
3020 key: impl Into<String>,
3021 value: Value,
3022 ) -> Result<(), EngineError> {
3023 let key = key.into();
3024 self.with_state("set_resource", move |s| {
3025 s.resources.insert(key, value);
3026 })
3027 .await?;
3028 Ok(())
3029 }
3030
3031 // ═══════════════════════════════════════════════════════════════════════
3032 // Senior suspend / resume
3033 // ═══════════════════════════════════════════════════════════════════════
3034
3035 /// Ask a question of the Senior, mark the task `Suspended`, and
3036 /// return a `ResumeKey`. The suspended state persists until another
3037 /// task calls `resume(key, answer)`.
3038 ///
3039 /// Resume-side waiting is `Notify`-based, so a caller (typically
3040 /// MainAI) can detach, reattach from a different process, and still
3041 /// pull the answer out via `await_resume(key, timeout)` — the answer
3042 /// is stored inside `EngineState`.
3043 pub async fn query_senior(
3044 &self,
3045 token: &CapToken,
3046 task_id: &StepId,
3047 question: Value,
3048 ) -> Result<ResumeKey, EngineError> {
3049 self.verify_token(token, Verb::QuerySenior).await?;
3050 let task_id = task_id.clone();
3051 let key = ResumeKey::for_senior(&task_id);
3052 let task_notify = self
3053 .with_state("query_senior.notify_ensure", |s| {
3054 s.ensure_task_notify(&task_id)
3055 })
3056 .await?;
3057
3058 let key_clone = key.clone();
3059 let task_id_inner = task_id.clone();
3060 let question_clone = question.clone();
3061 self.with_state("query_senior.suspend", move |s| {
3062 let task = s
3063 .tasks
3064 .get_mut(&task_id_inner)
3065 .ok_or_else(|| EngineError::TaskNotFound(task_id_inner.to_string()))?;
3066 task.status = TaskStatus::Suspended;
3067 task.suspended_on = Some(key_clone.clone());
3068 task.updated_at = now_unix();
3069 s.pending_resumes
3070 .insert(key_clone.clone(), ResumePending::new());
3071 s.push_event(Event::SeniorQueried {
3072 task_id: task_id_inner.clone(),
3073 question: question_clone.clone(),
3074 });
3075 s.push_event(Event::TaskSuspended {
3076 task_id: task_id_inner.clone(),
3077 key: key_clone.clone(),
3078 });
3079 Ok::<(), EngineError>(())
3080 })
3081 .await??;
3082
3083 // Notify callers waiting for a task status change (Running → Suspended).
3084 task_notify.notify_waiters();
3085
3086 let _ = self
3087 .inner
3088 .event_tx
3089 .send(Event::SeniorQueried { task_id, question });
3090 Ok(key)
3091 }
3092
3093 /// Store the answer for a `ResumeKey` in `EngineState` and wake the
3094 /// waiting caller via `Notify`. Also flips the suspended task's
3095 /// status back to `Running` and fires the per-task notifier.
3096 pub async fn resume(&self, key: ResumeKey, answer: Value) -> Result<(), EngineError> {
3097 let answer_for_state = answer.clone();
3098 let answer_for_event = answer.clone();
3099 let key_clone = key.clone();
3100 let (notify, task_notify, task_id_opt) = self
3101 .with_state("resume.set", move |s| {
3102 let pending = s
3103 .pending_resumes
3104 .get_mut(&key_clone)
3105 .ok_or(EngineError::ResumeKeyNotFound)?;
3106 pending.answer = Some(answer_for_state);
3107 let notify = pending.notify.clone();
3108
3109 let task_id = s
3110 .tasks
3111 .iter()
3112 .find(|(_, t)| t.suspended_on.as_ref() == Some(&key_clone))
3113 .map(|(id, _)| id.clone());
3114
3115 let task_notify = task_id.as_ref().map(|tid| s.ensure_task_notify(tid));
3116
3117 if let Some(tid) = &task_id {
3118 if let Some(task) = s.tasks.get_mut(tid) {
3119 task.suspended_on = None;
3120 task.status = TaskStatus::Running;
3121 task.updated_at = now_unix();
3122 }
3123 s.push_event(Event::TaskResumed {
3124 task_id: tid.clone(),
3125 key: key_clone.clone(),
3126 });
3127 s.push_event(Event::SeniorAnswered {
3128 task_id: tid.clone(),
3129 answer: answer_for_event.clone(),
3130 });
3131 }
3132 Ok::<_, EngineError>((notify, task_notify, task_id))
3133 })
3134 .await??;
3135
3136 // Outside the lock: notify_waiters for both the ResumePending and task-status waits.
3137 notify.notify_waiters();
3138 if let Some(n) = task_notify {
3139 n.notify_waiters();
3140 }
3141
3142 if let Some(tid) = task_id_opt {
3143 let _ = self
3144 .inner
3145 .event_tx
3146 .send(Event::TaskResumed { task_id: tid, key });
3147 }
3148 Ok(())
3149 }
3150
3151 /// Wait for the resume answer. Even if the caller (an Operator)
3152 /// detached and reattached, the answer is available immediately here
3153 /// — if it was already stored, this returns without waiting on the
3154 /// notifier.
3155 ///
3156 /// `timeout = Duration::ZERO` performs an instant check without
3157 /// waiting.
3158 pub async fn await_resume(
3159 &self,
3160 key: ResumeKey,
3161 timeout: Duration,
3162 ) -> Result<Value, EngineError> {
3163 // (1) Under the lock: clone the notify handle and check for an existing answer.
3164 let key_clone = key.clone();
3165 let (notify, existing) = self
3166 .with_state("await_resume.snapshot", move |s| {
3167 let pending = s
3168 .pending_resumes
3169 .get(&key_clone)
3170 .ok_or(EngineError::ResumeKeyNotFound)?;
3171 Ok::<_, EngineError>((pending.notify.clone(), pending.answer.clone()))
3172 })
3173 .await??;
3174
3175 // (2) If an answer has already been stored, return immediately (detach / reattach pattern).
3176 if let Some(v) = existing {
3177 return Ok(v);
3178 }
3179
3180 // (3) Outside the lock: wait on the notify with a timeout.
3181 if timeout.is_zero() {
3182 return Err(EngineError::PollTimeout);
3183 }
3184 let waited = tokio::time::timeout(timeout, notify.notified()).await;
3185 if waited.is_err() {
3186 return Err(EngineError::PollTimeout);
3187 }
3188
3189 // (4) Under the lock: re-read the answer (should be present now that we were notified).
3190 let key_clone = key.clone();
3191 self.with_state("await_resume.read", move |s| {
3192 let pending = s
3193 .pending_resumes
3194 .get(&key_clone)
3195 .ok_or(EngineError::ResumeKeyNotFound)?;
3196 pending
3197 .answer
3198 .clone()
3199 .ok_or_else(|| EngineError::Internal("notified but answer missing".into()))
3200 })
3201 .await?
3202 }
3203
3204 // ═══════════════════════════════════════════════════════════════════════
3205 // poll_task — the "wait" path that waits for task-status changes (works for long-poll and regular wait).
3206 // ═══════════════════════════════════════════════════════════════════════
3207
3208 /// Wait until the task's status **transitions to terminal or
3209 /// `Suspended`**, then return the latest `TaskState`. Returns
3210 /// immediately if the task is already in a terminal state.
3211 /// Exceeding the timeout returns `EngineError::PollTimeout`.
3212 ///
3213 /// A `hold` of `Duration::from_secs(0)` returns a snapshot immediately
3214 /// (no wait). Larger holds — tens of minutes up to days — are fine;
3215 /// the wait state is kept in memory inside the engine and does not
3216 /// degrade.
3217 pub async fn poll_task(
3218 &self,
3219 token: &CapToken,
3220 task_id: &StepId,
3221 hold: Duration,
3222 ) -> Result<TaskState, EngineError> {
3223 self.verify_token_for_task(token, Verb::PollTask, task_id)
3224 .await?;
3225 let task_id_inner = task_id.clone();
3226
3227 // (1) Under the lock: take a snapshot and clone task_notify.
3228 let (state, notify) = self
3229 .with_state("poll_task.snapshot", move |s| {
3230 let task = s
3231 .tasks
3232 .get(&task_id_inner)
3233 .cloned()
3234 .ok_or_else(|| EngineError::TaskNotFound(task_id_inner.to_string()))?;
3235 let notify = s.ensure_task_notify(&task_id_inner);
3236 Ok::<_, EngineError>((task, notify))
3237 })
3238 .await??;
3239
3240 // (2) Immediate-return condition: already terminal / Suspended (nothing left to wait on).
3241 if matches!(
3242 state.status,
3243 TaskStatus::Pass | TaskStatus::Blocked | TaskStatus::Cancelled | TaskStatus::Suspended
3244 ) {
3245 return Ok(state);
3246 }
3247 if hold.is_zero() {
3248 return Ok(state);
3249 }
3250
3251 // (3) Outside the lock: wait on Notify with a timeout.
3252 let waited = tokio::time::timeout(hold, notify.notified()).await;
3253 if waited.is_err() {
3254 return Err(EngineError::PollTimeout);
3255 }
3256
3257 // (4) Under the lock: take a fresh snapshot.
3258 let task_id_inner = task_id.clone();
3259 self.with_state("poll_task.reread", move |s| {
3260 s.tasks
3261 .get(&task_id_inner)
3262 .cloned()
3263 .ok_or_else(|| EngineError::TaskNotFound(task_id_inner.to_string()))
3264 })
3265 .await?
3266 }
3267
3268 // ═══════════════════════════════════════════════════════════════════════
3269 // Background: heartbeat miss → detach loop
3270 // ═══════════════════════════════════════════════════════════════════════
3271
3272 /// Background loop that scans sessions every `heartbeat_interval` and
3273 /// flips `attached = false` on any session whose `last_seen` exceeds
3274 /// `heartbeat_miss_threshold * interval`.
3275 ///
3276 /// The tasks themselves are kept (assuming
3277 /// `keepalive_on_idle = true`), so another client can reattach with
3278 /// the same token and resume immediately. Dropping the returned
3279 /// `JoinHandle` does not stop the loop — the handle exists so callers
3280 /// who want to abort can hold onto it.
3281 pub fn start_detach_loop(&self) -> tokio::task::JoinHandle<()> {
3282 let engine = self.clone();
3283 let cfg = self.inner.cfg.long_hold.clone();
3284 let interval = cfg.heartbeat_interval;
3285 let miss_secs = cfg.heartbeat_interval.as_secs() * cfg.heartbeat_miss_threshold as u64;
3286
3287 tokio::spawn(async move {
3288 let mut ticker = tokio::time::interval(interval);
3289 ticker.tick().await; // first tick is immediate
3290 loop {
3291 ticker.tick().await;
3292 let now = now_unix();
3293 let detached = engine
3294 .with_state("detach_loop.scan", |s| {
3295 let mut detached = Vec::new();
3296 for (sid, sess) in s.sessions.iter_mut() {
3297 if !sess.attached {
3298 continue;
3299 }
3300 if now.saturating_sub(sess.last_seen) >= miss_secs {
3301 sess.attached = false;
3302 detached.push(sid.clone());
3303 }
3304 }
3305 for sid in &detached {
3306 s.push_event(Event::SessionDetached {
3307 session_id: sid.clone(),
3308 });
3309 }
3310 detached
3311 })
3312 .await
3313 .unwrap_or_default();
3314 for sid in detached {
3315 let _ = engine
3316 .inner
3317 .event_tx
3318 .send(Event::SessionDetached { session_id: sid });
3319 }
3320 }
3321 })
3322 }
3323
3324 /// Helper: wake a task whose status has changed. Called from the
3325 /// method body outside the lock.
3326 async fn wake_task(&self, task_id: &StepId) -> Result<(), EngineError> {
3327 let task_id = task_id.clone();
3328 let notify_opt = self
3329 .with_state("wake_task.get_notify", move |s| {
3330 s.task_notifies.get(&task_id).cloned()
3331 })
3332 .await?;
3333 if let Some(n) = notify_opt {
3334 n.notify_waiters();
3335 }
3336 Ok(())
3337 }
3338}
3339
3340/// Decide what a submit-time projection sink should do at a fail-open
3341/// branch given the configured [`crate::core::config::CheckPolicy`].
3342///
3343/// Returns `Ok(())` under [`CheckPolicy::Silent`] and
3344/// [`CheckPolicy::Warn`] — the caller continues with fail-open. Returns
3345/// [`EngineError::CheckPolicyStrict`] under [`CheckPolicy::Strict`],
3346/// carrying the caller-supplied `context` (call-site identifier) and
3347/// `message` (the pre-existing warn-log message literal, preserved
3348/// verbatim for log parse compatibility).
3349///
3350/// This helper deliberately does **not** call `tracing::warn!` itself —
3351/// the caller is responsible for firing the existing warn! (with its
3352/// full structured-field payload — `%task_id`, `agent`, `canonical`,
3353/// `error`, etc.) under `Warn` mode, and for skipping the warn! under
3354/// `Silent` mode. Keeping the warn! at the call site preserves the
3355/// exact structured-field shape every existing log-parse consumer sees;
3356/// forwarding it through the helper would either drop those fields or
3357/// require a macro (deferred, see subtask-1b).
3358///
3359/// Design intent: the fail-open discipline of every submit-time
3360/// projection sink is byte-identical to the pre-`CheckPolicy` behaviour
3361/// under the default [`CheckPolicy::Warn`]. `Silent` is a per-run opt-in
3362/// to suppress noise (e.g., a caller that has already verified upstream
3363/// invariants); `Strict` is a per-run opt-in to fail loudly (e.g., a
3364/// caller that requires all parts to materialize). See
3365/// [`crate::core::config::CheckPolicy`] for the "state dirty on fail"
3366/// semantics of `Strict`.
3367pub(crate) fn apply_check_policy(
3368 policy: crate::core::config::CheckPolicy,
3369 context: &str,
3370 message: &str,
3371) -> Result<(), EngineError> {
3372 match policy {
3373 crate::core::config::CheckPolicy::Silent | crate::core::config::CheckPolicy::Warn => Ok(()),
3374 crate::core::config::CheckPolicy::Strict => Err(EngineError::CheckPolicyStrict {
3375 context: context.to_string(),
3376 message: message.to_string(),
3377 }),
3378 }
3379}
3380
3381#[cfg(test)]
3382mod check_policy_helper_tests {
3383 use super::apply_check_policy;
3384 use crate::core::config::CheckPolicy;
3385 use crate::core::errors::EngineError;
3386
3387 /// `Silent` returns `Ok(())` without producing an error. Log
3388 /// suppression (the "no `tracing::warn!`" half of the semantics) is
3389 /// enforced at the call site, not inside the helper — see the
3390 /// helper's doc comment for why.
3391 #[test]
3392 fn silent_returns_ok() {
3393 let result = apply_check_policy(CheckPolicy::Silent, "call/site", "sink message");
3394 assert!(matches!(result, Ok(())));
3395 }
3396
3397 /// `Warn` (the default) returns `Ok(())` — the caller continues
3398 /// with fail-open, having already fired its own `tracing::warn!`
3399 /// with the full structured-field payload.
3400 #[test]
3401 fn warn_returns_ok() {
3402 let result = apply_check_policy(CheckPolicy::Warn, "call/site", "sink message");
3403 assert!(matches!(result, Ok(())));
3404 }
3405
3406 /// `Strict` returns
3407 /// [`EngineError::CheckPolicyStrict`] with `context` and `message`
3408 /// copied verbatim from the caller — the completion route surfaces
3409 /// this as a step / launch error so a caller that has opted in can
3410 /// fail fast instead of proceeding with a partially-realized
3411 /// submission.
3412 #[test]
3413 fn strict_returns_error_with_context_and_message() {
3414 let result = apply_check_policy(
3415 CheckPolicy::Strict,
3416 "submit-time projection sink: file materialize",
3417 "no work_dir/project_root resolved; skipping file materialize (fail-open)",
3418 );
3419 match result {
3420 Err(EngineError::CheckPolicyStrict { context, message }) => {
3421 assert_eq!(context, "submit-time projection sink: file materialize");
3422 assert_eq!(
3423 message,
3424 "no work_dir/project_root resolved; skipping file materialize (fail-open)"
3425 );
3426 }
3427 other => panic!("expected CheckPolicyStrict, got {:?}", other),
3428 }
3429 }
3430}
3431
3432// ─── UT: R4 max-hold guard — panic in debug, warn + continue in release ─────
3433#[cfg(test)]
3434mod max_hold_guard_tests {
3435 use super::*;
3436
3437 /// Debug builds keep the hard failure: an over-budget closure unwinds
3438 /// with the historical message so an R3 violation is impossible to miss
3439 /// in dev and test.
3440 #[cfg(debug_assertions)]
3441 #[tokio::test]
3442 #[should_panic(expected = "suspected R3 violation")]
3443 async fn with_state_over_max_hold_panics_in_debug() {
3444 let engine = Engine::new(EngineCfg {
3445 max_hold_ms: 0,
3446 ..EngineCfg::default()
3447 });
3448 // `with_state` takes a sync `FnOnce`, so a blocking sleep is the
3449 // only way to overrun the budget from inside the lock.
3450 let _ = engine
3451 .with_state("test.over_max_hold", |_s| {
3452 std::thread::sleep(Duration::from_millis(5));
3453 })
3454 .await;
3455 }
3456
3457 /// Release builds only warn: the call returns `Ok` and the caller's task
3458 /// survives, which is what keeps a run driver future from being unwound
3459 /// (and its RunRecord stranded in `Running`) by a load-dependent overrun.
3460 #[cfg(not(debug_assertions))]
3461 #[tokio::test]
3462 async fn with_state_over_max_hold_warns_and_returns_in_release() {
3463 let engine = Engine::new(EngineCfg {
3464 max_hold_ms: 0,
3465 ..EngineCfg::default()
3466 });
3467 let result = engine
3468 .with_state("test.over_max_hold", |_s| {
3469 std::thread::sleep(Duration::from_millis(5));
3470 42u32
3471 })
3472 .await;
3473 assert_eq!(result.expect("release build must not panic"), 42);
3474 }
3475}
3476
3477// ─── UT: issue #14 — token store keyed by fingerprint, not nonce ────────────
3478#[cfg(test)]
3479mod token_fingerprint_store_tests {
3480 use super::*;
3481
3482 /// A token that was never attached fails verify with a `TokenNotFound`
3483 /// that carries the fingerprint — never the nonce. The error string can
3484 /// surface in HTTP error bodies, so this is the secret-hygiene contract.
3485 #[tokio::test]
3486 async fn verify_unknown_token_reports_fingerprint_not_nonce() {
3487 let engine = Engine::new(EngineCfg::default());
3488 // Signed by the engine's own signer (sig passes) but never inserted
3489 // into the store — verify must fail at step (4), the store lookup.
3490 let token = engine.signer().session(
3491 "ghost",
3492 Role::Operator,
3493 vec!["*".into()],
3494 Duration::from_secs(60),
3495 );
3496 let err = engine
3497 .verify_token(&token, Verb::ReadTaskState)
3498 .await
3499 .expect_err("token is not in the store");
3500 let msg = err.to_string();
3501 assert!(
3502 msg.contains(&token.fingerprint()),
3503 "error must carry the fingerprint: {msg}"
3504 );
3505 assert!(
3506 !msg.contains(&token.nonce),
3507 "error must not leak the nonce: {msg}"
3508 );
3509 }
3510
3511 /// attach → verify → heartbeat → detach all resolve the session /
3512 /// token record through fingerprint keys (mint/verify lifecycle
3513 /// regression guard for the issue #14 key migration).
3514 #[tokio::test]
3515 async fn attach_verify_heartbeat_detach_cycle_with_fp_keying() {
3516 let engine = Engine::new(EngineCfg::default());
3517 let token = engine
3518 .attach("op-1", Role::Operator, Duration::from_secs(60))
3519 .await
3520 .expect("attach");
3521 engine
3522 .verify_token(&token, Verb::ReadTaskState)
3523 .await
3524 .expect("verify consumes via fp key");
3525 engine
3526 .heartbeat(&token)
3527 .await
3528 .expect("heartbeat finds the session by fp");
3529 engine
3530 .detach(&token)
3531 .await
3532 .expect("detach finds the session by fp");
3533 }
3534}
3535
3536// ─── UT: `OperatorKind` "Runtime Global" tier — `Option` semantics ─────────
3537//
3538// Regression coverage for the "explicit Automate is indistinguishable from
3539// unspecified" defect: `OperatorSession.operator_kind` (and the
3540// `attach_with_ids` `kind` parameter it stores) is `Option<OperatorKind>`,
3541// so `Some(Automate)` is an explicit Runtime Global request that must
3542// outrank `bp_global`, while `None` must let `bp_global` decide. Exercises
3543// the real `resolve_operator_info` cascade path (not just
3544// `collapse_operator_kind` in isolation), attaching via `attach_with_ids`
3545// exactly as `TaskLaunchService::launch` does.
3546#[cfg(test)]
3547mod resolve_operator_info_runtime_global_tests {
3548 use super::*;
3549
3550 async fn attach_and_resolve(
3551 runtime_global: Option<OperatorKind>,
3552 bp_global: Option<OperatorKind>,
3553 ) -> OperatorInfo {
3554 let engine = Engine::new(EngineCfg::default());
3555 let token = engine
3556 .attach_with_ids(
3557 "ut-op",
3558 Role::Operator,
3559 Duration::from_secs(30),
3560 runtime_global,
3561 None,
3562 None,
3563 None,
3564 HashMap::new(),
3565 HashMap::new(),
3566 bp_global,
3567 )
3568 .await
3569 .expect("attach_with_ids ok");
3570 let session = engine
3571 .with_state("test.find_session", |s| {
3572 s.sessions
3573 .values()
3574 .find(|sess| sess.token_fp == token.fingerprint())
3575 .cloned()
3576 })
3577 .await
3578 .expect("with_state ok")
3579 .expect("session present after attach_with_ids");
3580 engine.resolve_operator_info(&session, "agent-x").await
3581 }
3582
3583 #[tokio::test]
3584 async fn explicit_some_automate_outranks_bp_global_main_ai() {
3585 // Runtime Global explicitly requests Automate; bp_global is MainAi.
3586 // The explicit `Some(Automate)` must win — this is exactly the case
3587 // the old `== OperatorKind::default()` convention got wrong (it
3588 // could not tell "explicitly Automate" from "unspecified" and would
3589 // have let `bp_global` (MainAi) take over instead).
3590 let info =
3591 attach_and_resolve(Some(OperatorKind::Automate), Some(OperatorKind::MainAi)).await;
3592 assert_eq!(
3593 info.kind,
3594 OperatorKind::Automate,
3595 "explicit Some(Automate) runtime_global must outrank bp_global MainAi"
3596 );
3597 }
3598
3599 #[tokio::test]
3600 async fn none_lets_bp_global_main_ai_win() {
3601 // Runtime Global left unspecified (`None`); bp_global is MainAi.
3602 // With nothing more specific set, `bp_global` must decide.
3603 let info = attach_and_resolve(None, Some(OperatorKind::MainAi)).await;
3604 assert_eq!(
3605 info.kind,
3606 OperatorKind::MainAi,
3607 "None runtime_global must let bp_global MainAi win"
3608 );
3609 }
3610}
3611
3612/// issue #13 run_id propagation: `dispatch_attempt_with`'s `run_id` param
3613/// must land in `Ctx.meta.runtime["run_id"]` (the same slot pattern as the
3614/// pre-existing `worker_handle`), or be omitted entirely when `None`. Same
3615/// `CtxProbe` shape as `middleware::worker_binding`'s test module — an
3616/// inner `SpawnerAdapter` that snapshots the `Ctx` it was called with and
3617/// fails the spawn (only the ctx snapshot matters here).
3618#[cfg(test)]
3619mod dispatch_attempt_with_run_id_tests {
3620 use super::*;
3621 use crate::worker::adapter::{SpawnError, SpawnerAdapter};
3622 use crate::worker::Worker;
3623 use std::sync::Mutex as StdMutex;
3624
3625 struct CtxProbe {
3626 seen: Arc<StdMutex<Option<Ctx>>>,
3627 }
3628
3629 #[async_trait::async_trait]
3630 impl SpawnerAdapter for CtxProbe {
3631 async fn spawn(
3632 &self,
3633 _engine: &Engine,
3634 ctx: &Ctx,
3635 _task_id: StepId,
3636 _attempt: u32,
3637 _token: CapToken,
3638 ) -> Result<Box<dyn Worker>, SpawnError> {
3639 *self.seen.lock().unwrap() = Some(ctx.clone());
3640 Err(SpawnError::Internal("probe stop".into()))
3641 }
3642 }
3643
3644 async fn dispatch_with_probe(run_id: Option<&RunId>) -> Ctx {
3645 let engine = Engine::new(EngineCfg::default());
3646 let token = engine
3647 .attach("ut-op", Role::Operator, Duration::from_secs(30))
3648 .await
3649 .expect("attach");
3650 let tid = engine
3651 .start_task(
3652 &token,
3653 TaskSpec {
3654 agent: "probe".into(),
3655 initial_directive: "hi".into(),
3656 step_ctx: None,
3657 check_policy: None,
3658 },
3659 )
3660 .await
3661 .expect("start_task");
3662 let seen: Arc<StdMutex<Option<Ctx>>> = Arc::new(StdMutex::new(None));
3663 let spawner: Arc<dyn SpawnerAdapter> = Arc::new(CtxProbe { seen: seen.clone() });
3664 // The probe always errors the spawn (`SpawnError::Internal`); we
3665 // only care about the `Ctx` snapshot it captured, so the dispatch
3666 // outcome itself (`Err`) is discarded.
3667 let _ = engine
3668 .dispatch_attempt_with(&token, &tid, &spawner, run_id)
3669 .await;
3670 let captured = seen.lock().unwrap().clone();
3671 captured.expect("inner ctx captured")
3672 }
3673
3674 #[tokio::test]
3675 async fn run_id_lands_in_ctx_meta_runtime_when_some() {
3676 let run_id = RunId::new();
3677 let observed = dispatch_with_probe(Some(&run_id)).await;
3678 assert_eq!(
3679 observed.meta.runtime.get("run_id").and_then(|v| v.as_str()),
3680 Some(run_id.as_str()),
3681 "ctx.meta.runtime[\"run_id\"] must carry the run_id passed to dispatch_attempt_with"
3682 );
3683 }
3684
3685 #[tokio::test]
3686 async fn run_id_key_absent_when_none() {
3687 let observed = dispatch_with_probe(None).await;
3688 assert!(
3689 !observed.meta.runtime.contains_key("run_id"),
3690 "no run_id key must be injected when dispatch_attempt_with is called with None"
3691 );
3692 }
3693}
3694
3695/// GH #21 Phase 2: `TaskSpec.step_ctx` must land in
3696/// `Ctx.meta.runtime[STEP_CTX_KEY]` — re-read from the spec on EVERY
3697/// attempt (the prep closure re-reads `task.spec.step_ctx` every call, not
3698/// caching it once at `start_task`), so a retry (attempt 2) carries it
3699/// too. Same `CtxProbe` shape as `dispatch_attempt_with_run_id_tests`.
3700#[cfg(test)]
3701mod dispatch_attempt_with_step_ctx_tests {
3702 use super::*;
3703 use crate::worker::adapter::{SpawnError, SpawnerAdapter};
3704 use crate::worker::Worker;
3705 use std::sync::Mutex as StdMutex;
3706
3707 struct CtxProbe {
3708 seen: Arc<StdMutex<Option<Ctx>>>,
3709 }
3710
3711 #[async_trait::async_trait]
3712 impl SpawnerAdapter for CtxProbe {
3713 async fn spawn(
3714 &self,
3715 _engine: &Engine,
3716 ctx: &Ctx,
3717 _task_id: StepId,
3718 _attempt: u32,
3719 _token: CapToken,
3720 ) -> Result<Box<dyn Worker>, SpawnError> {
3721 *self.seen.lock().unwrap() = Some(ctx.clone());
3722 Err(SpawnError::Internal("probe stop".into()))
3723 }
3724 }
3725
3726 #[tokio::test]
3727 async fn step_ctx_lands_in_ctx_meta_runtime_on_attempt_1_and_2() {
3728 let engine = Engine::new(EngineCfg::default());
3729 let token = engine
3730 .attach("ut-op", Role::Operator, Duration::from_secs(30))
3731 .await
3732 .expect("attach");
3733 let tid = engine
3734 .start_task(
3735 &token,
3736 TaskSpec {
3737 agent: "probe".into(),
3738 initial_directive: "hi".into(),
3739 step_ctx: Some(serde_json::json!({ "work_dir": "/step" })),
3740 check_policy: None,
3741 },
3742 )
3743 .await
3744 .expect("start_task");
3745 let seen: Arc<StdMutex<Option<Ctx>>> = Arc::new(StdMutex::new(None));
3746 let spawner: Arc<dyn SpawnerAdapter> = Arc::new(CtxProbe { seen: seen.clone() });
3747
3748 // The probe always errors the spawn; only the ctx snapshot matters.
3749 let _ = engine
3750 .dispatch_attempt_with(&token, &tid, &spawner, None)
3751 .await;
3752 let first = seen
3753 .lock()
3754 .unwrap()
3755 .clone()
3756 .expect("attempt 1 ctx captured");
3757 assert_eq!(
3758 first.meta.runtime.get(STEP_CTX_KEY),
3759 Some(&serde_json::json!({ "work_dir": "/step" })),
3760 "attempt 1 must carry TaskSpec.step_ctx in ctx.meta.runtime[STEP_CTX_KEY]"
3761 );
3762
3763 let _ = engine
3764 .dispatch_attempt_with(&token, &tid, &spawner, None)
3765 .await;
3766 let second = seen
3767 .lock()
3768 .unwrap()
3769 .clone()
3770 .expect("attempt 2 ctx captured");
3771 assert_eq!(
3772 second.meta.runtime.get(STEP_CTX_KEY),
3773 Some(&serde_json::json!({ "work_dir": "/step" })),
3774 "attempt 2 (retry) must ALSO carry TaskSpec.step_ctx — prep re-reads the spec every attempt"
3775 );
3776 }
3777
3778 #[tokio::test]
3779 async fn step_ctx_key_absent_when_none() {
3780 let engine = Engine::new(EngineCfg::default());
3781 let token = engine
3782 .attach("ut-op", Role::Operator, Duration::from_secs(30))
3783 .await
3784 .expect("attach");
3785 let tid = engine
3786 .start_task(
3787 &token,
3788 TaskSpec {
3789 agent: "probe".into(),
3790 initial_directive: "hi".into(),
3791 step_ctx: None,
3792 check_policy: None,
3793 },
3794 )
3795 .await
3796 .expect("start_task");
3797 let seen: Arc<StdMutex<Option<Ctx>>> = Arc::new(StdMutex::new(None));
3798 let spawner: Arc<dyn SpawnerAdapter> = Arc::new(CtxProbe { seen: seen.clone() });
3799 let _ = engine
3800 .dispatch_attempt_with(&token, &tid, &spawner, None)
3801 .await;
3802 let observed = seen.lock().unwrap().clone().expect("ctx captured");
3803 assert!(
3804 !observed.meta.runtime.contains_key(STEP_CTX_KEY),
3805 "no step_ctx key must be injected when TaskSpec.step_ctx is None"
3806 );
3807 }
3808}
3809
3810// ─── issue #18: `TaskSpec.initial_directive` `Value` pass-through ──────────
3811#[cfg(test)]
3812mod initial_directive_value_passthrough_tests {
3813 use super::*;
3814
3815 async fn seeded_engine(initial_directive: Value) -> (Engine, CapToken, StepId) {
3816 let engine = Engine::new(EngineCfg::default());
3817 let op_token = engine
3818 .attach("ut-op", Role::Operator, Duration::from_secs(30))
3819 .await
3820 .expect("attach");
3821 let task_id = engine
3822 .start_task(
3823 &op_token,
3824 TaskSpec {
3825 agent: "planner".to_string(),
3826 initial_directive,
3827 step_ctx: None,
3828 check_policy: None,
3829 },
3830 )
3831 .await
3832 .expect("start_task");
3833 (engine, op_token, task_id)
3834 }
3835
3836 /// Mint + register a `Role::Worker` token the same way
3837 /// `dispatch_attempt_with` does — `fetch_prompt` is worker-verb-gated.
3838 async fn mint_worker_token(engine: &Engine, task_id: &StepId) -> CapToken {
3839 let worker_token = engine.signer().session(
3840 format!("worker-of-{task_id}"),
3841 Role::Worker,
3842 vec!["*".into()],
3843 Duration::from_secs(600),
3844 );
3845 let fp = worker_token.fingerprint();
3846 let record = CapTokenRecord::from_worker_token(worker_token.clone(), task_id.clone());
3847 engine
3848 .with_state("test.mint_worker", move |s| {
3849 s.tokens.insert(fp, record);
3850 })
3851 .await
3852 .expect("mint worker token");
3853 worker_token
3854 }
3855
3856 /// `EngineDispatcher::dispatch` no longer stringifies the evaluated
3857 /// `Step.in` value before seeding `TaskSpec.initial_directive` — an
3858 /// Object seed must round-trip through `start_task` /
3859 /// `read_task_state` byte-for-byte as the same `Value::Object`, not a
3860 /// JSON-stringified `Value::String`.
3861 #[tokio::test]
3862 async fn object_seed_passes_through_task_spec_unchanged() {
3863 let seed = serde_json::json!({"key": "value"});
3864 let (engine, token, task_id) = seeded_engine(seed.clone()).await;
3865 let state = engine
3866 .read_task_state(&token, &task_id)
3867 .await
3868 .expect("read_task_state");
3869 assert_eq!(
3870 state.spec.initial_directive, seed,
3871 "TaskSpec.initial_directive must equal the raw Object seed, not a stringified copy"
3872 );
3873 }
3874
3875 /// `Engine::fetch_prompt` returns the `Value` end-to-end (issue #18):
3876 /// an Object seed stays a `Value::Object` and is not stringified in
3877 /// the engine layer. The Worker HTTP boundary
3878 /// (`fetch_worker_payload*`) is what performs the render down to a
3879 /// JSON literal `String` for `WorkerPayload.prompt`.
3880 #[tokio::test]
3881 async fn object_seed_passes_through_fetch_prompt_as_value() {
3882 let seed = serde_json::json!({"key": "value"});
3883 let (engine, _token, task_id) = seeded_engine(seed.clone()).await;
3884 let worker_token = mint_worker_token(&engine, &task_id).await;
3885 let prompt = engine
3886 .fetch_prompt(&worker_token, &task_id)
3887 .await
3888 .expect("fetch_prompt");
3889 assert_eq!(
3890 prompt, seed,
3891 "fetch_prompt must return the raw Object Value, not a stringified copy"
3892 );
3893 }
3894
3895 /// The Worker HTTP boundary is the render point: `fetch_worker_payload*`
3896 /// coerces the stored `Value` down to `WorkerPayload.prompt: String`
3897 /// (JSON-literal shape for non-strings). Verifies the boundary render
3898 /// stays intact for an Object seed.
3899 #[tokio::test]
3900 async fn object_seed_renders_as_json_literal_at_worker_payload_boundary() {
3901 let seed = serde_json::json!({"key": "value"});
3902 let (engine, _token, task_id) = seeded_engine(seed).await;
3903 let worker_token = mint_worker_token(&engine, &task_id).await;
3904 let payload = engine
3905 .fetch_worker_payload(&worker_token, &task_id)
3906 .await
3907 .expect("fetch_worker_payload");
3908 assert_eq!(
3909 payload.prompt, r#"{"key":"value"}"#,
3910 "WorkerPayload.prompt must be the JSON literal String render of the Value seed"
3911 );
3912 }
3913
3914 /// A `String` seed is unaffected — still passes through verbatim, both
3915 /// as the `TaskSpec.initial_directive` `Value` and as the Worker
3916 /// `fetch_prompt` return (issue #18 Invariant 2).
3917 #[tokio::test]
3918 async fn string_seed_passes_through_unchanged() {
3919 let (engine, token, task_id) = seeded_engine(serde_json::json!("do the thing")).await;
3920 let state = engine
3921 .read_task_state(&token, &task_id)
3922 .await
3923 .expect("read_task_state");
3924 assert_eq!(
3925 state.spec.initial_directive,
3926 serde_json::json!("do the thing")
3927 );
3928 let worker_token = mint_worker_token(&engine, &task_id).await;
3929 let prompt = engine
3930 .fetch_prompt(&worker_token, &task_id)
3931 .await
3932 .expect("fetch_prompt");
3933 assert_eq!(prompt, serde_json::json!("do the thing"));
3934 }
3935}
3936
3937/// GH #31: `fetch_worker_payload{,_trusted}`'s size-threshold branch
3938/// between inline (`WorkerPayload.system`) and by-reference
3939/// (`WorkerPayload.system_ref`) delivery, plus the `bake_worker_system_prompt`
3940/// `agent_render_sizes` bookkeeping that feeds `agent_last_rendered_size`.
3941#[cfg(test)]
3942mod system_ref_threshold_tests {
3943 use super::*;
3944
3945 async fn seeded_engine_with_cfg(cfg: EngineCfg) -> (Engine, CapToken, StepId) {
3946 let engine = Engine::new(cfg);
3947 let op_token = engine
3948 .attach("ut-op", Role::Operator, Duration::from_secs(30))
3949 .await
3950 .expect("attach");
3951 let task_id = engine
3952 .start_task(
3953 &op_token,
3954 TaskSpec {
3955 agent: "planner".to_string(),
3956 initial_directive: serde_json::json!("do the thing"),
3957 step_ctx: None,
3958 check_policy: None,
3959 },
3960 )
3961 .await
3962 .expect("start_task");
3963 (engine, op_token, task_id)
3964 }
3965
3966 /// Same worker-token-minting fixture as
3967 /// `initial_directive_value_passthrough_tests::mint_worker_token`
3968 /// (kept local to this module — the two `mod`s do not share private
3969 /// helpers across `cfg(test)` boundaries).
3970 async fn mint_worker_token(engine: &Engine, task_id: &StepId) -> CapToken {
3971 let worker_token = engine.signer().session(
3972 format!("worker-of-{task_id}"),
3973 Role::Worker,
3974 vec!["*".into()],
3975 Duration::from_secs(600),
3976 );
3977 let fp = worker_token.fingerprint();
3978 let record = CapTokenRecord::from_worker_token(worker_token.clone(), task_id.clone());
3979 engine
3980 .with_state("test.mint_worker", move |s| {
3981 s.tokens.insert(fp, record);
3982 })
3983 .await
3984 .expect("mint worker token");
3985 worker_token
3986 }
3987
3988 /// Under-threshold: `system` stays inline, `system_ref` stays `None`.
3989 #[tokio::test]
3990 async fn under_threshold_stays_inline() {
3991 let (engine, _op_token, task_id) = seeded_engine_with_cfg(EngineCfg::default()).await;
3992 let worker_token = mint_worker_token(&engine, &task_id).await;
3993 let rendered = "a short system prompt".to_string();
3994 engine
3995 .bake_worker_system_prompt(&task_id, 1, Some(rendered.clone()))
3996 .await
3997 .expect("bake");
3998 let payload = engine
3999 .fetch_worker_payload(&worker_token, &task_id)
4000 .await
4001 .expect("fetch_worker_payload");
4002 assert_eq!(payload.system, Some(rendered));
4003 assert!(payload.system_ref.is_none());
4004 }
4005
4006 /// Over-threshold: `system` is cleared and `system_ref` is populated
4007 /// with a `sha256` matching the known input string. Exercises
4008 /// `fetch_worker_payload_trusted` (the `_trusted` sibling must be
4009 /// behaviorally identical to `fetch_worker_payload`).
4010 #[tokio::test]
4011 async fn over_threshold_switches_to_system_ref_with_matching_sha256() {
4012 let mut cfg = EngineCfg::default();
4013 cfg.system_ref.threshold_bytes = 16;
4014 cfg.system_ref.mode = crate::types::SystemRefMode::File;
4015 cfg.system_ref.store_dir =
4016 std::env::temp_dir().join(format!("mse-system-ref-test-{}", crate::types::now_unix()));
4017 let (engine, _op_token, task_id) = seeded_engine_with_cfg(cfg).await;
4018 let rendered =
4019 "this system prompt is deliberately longer than the 16 byte threshold".to_string();
4020 engine
4021 .bake_worker_system_prompt(&task_id, 1, Some(rendered.clone()))
4022 .await
4023 .expect("bake");
4024 let payload = engine
4025 .fetch_worker_payload_trusted(&task_id)
4026 .await
4027 .expect("fetch_worker_payload_trusted");
4028 assert!(
4029 payload.system.is_none(),
4030 "over-threshold response must not also inline `system`"
4031 );
4032 let system_ref = payload
4033 .system_ref
4034 .expect("over-threshold response must populate system_ref");
4035 assert_eq!(system_ref.size_bytes, rendered.len() as u64);
4036 assert_eq!(system_ref.mode, crate::types::SystemRefMode::File);
4037 use sha2::Digest;
4038 let expected_sha256 = hex::encode(sha2::Sha256::digest(rendered.as_bytes()));
4039 assert_eq!(system_ref.sha256, expected_sha256);
4040 assert!(system_ref.uri.starts_with("file://"));
4041 let written = tokio::fs::read_to_string(system_ref.uri.trim_start_matches("file://"))
4042 .await
4043 .expect("File mode must have written the referenced path");
4044 assert_eq!(written, rendered);
4045 }
4046
4047 /// `Http` mode never writes a file — `system_ref.uri` is the bare path
4048 /// the engine can construct on its own, scheme/host-free.
4049 #[tokio::test]
4050 async fn over_threshold_http_mode_constructs_path_only_uri() {
4051 let mut cfg = EngineCfg::default();
4052 cfg.system_ref.threshold_bytes = 16;
4053 cfg.system_ref.mode = crate::types::SystemRefMode::Http;
4054 let (engine, _op_token, task_id) = seeded_engine_with_cfg(cfg).await;
4055 let worker_token = mint_worker_token(&engine, &task_id).await;
4056 let rendered =
4057 "this system prompt is deliberately longer than the 16 byte threshold".to_string();
4058 engine
4059 .bake_worker_system_prompt(&task_id, 1, Some(rendered))
4060 .await
4061 .expect("bake");
4062 let payload = engine
4063 .fetch_worker_payload(&worker_token, &task_id)
4064 .await
4065 .expect("fetch_worker_payload");
4066 let system_ref = payload.system_ref.expect("system_ref must be populated");
4067 assert_eq!(system_ref.mode, crate::types::SystemRefMode::Http);
4068 assert_eq!(
4069 system_ref.uri,
4070 format!("/v1/worker/prompt/system?task_id={task_id}&attempt=1")
4071 );
4072 }
4073
4074 /// `bake_worker_system_prompt` records the render size keyed by agent
4075 /// name (last-write-wins), readable via `agent_last_rendered_size`.
4076 #[tokio::test]
4077 async fn bake_records_agent_render_size_last_write_wins() {
4078 let (engine, _op_token, task_id) = seeded_engine_with_cfg(EngineCfg::default()).await;
4079 assert_eq!(engine.agent_last_rendered_size("planner").await, None);
4080 engine
4081 .bake_worker_system_prompt(&task_id, 1, Some("a".repeat(10)))
4082 .await
4083 .expect("bake 1");
4084 assert_eq!(engine.agent_last_rendered_size("planner").await, Some(10));
4085 engine
4086 .bake_worker_system_prompt(&task_id, 2, Some("b".repeat(20)))
4087 .await
4088 .expect("bake 2");
4089 assert_eq!(
4090 engine.agent_last_rendered_size("planner").await,
4091 Some(20),
4092 "most-recently-observed size wins, not the largest"
4093 );
4094 }
4095
4096 /// GH #83: `materialize_system_file` writes the baked system prompt
4097 /// unconditionally — a system well UNDER `threshold_bytes` still
4098 /// lands on disk, because a `{system_file}` template reference needs
4099 /// a real path regardless of size.
4100 #[tokio::test]
4101 async fn materialize_system_file_writes_under_threshold_system() {
4102 let mut cfg = EngineCfg::default();
4103 cfg.system_ref.store_dir =
4104 std::env::temp_dir().join(format!("mse-system-file-test-{}", crate::types::now_unix()));
4105 assert!(cfg.system_ref.threshold_bytes > 64, "fixture premise");
4106 let (engine, _op_token, task_id) = seeded_engine_with_cfg(cfg).await;
4107 let rendered = "a short system prompt".to_string();
4108 engine
4109 .bake_worker_system_prompt(&task_id, 1, Some(rendered.clone()))
4110 .await
4111 .expect("bake");
4112 let path = engine
4113 .materialize_system_file(&task_id, 1)
4114 .await
4115 .expect("materialize_system_file")
4116 .expect("baked system must yield a path");
4117 let written = tokio::fs::read_to_string(&path)
4118 .await
4119 .expect("materialized path must exist");
4120 assert_eq!(written, rendered);
4121 }
4122
4123 /// GH #83: no baked system → `Ok(None)` (the Subprocess spawn path
4124 /// turns this into a fail-loud `SpawnError` when `{system_file}` is
4125 /// actually referenced).
4126 #[tokio::test]
4127 async fn materialize_system_file_none_when_nothing_baked() {
4128 let (engine, _op_token, task_id) = seeded_engine_with_cfg(EngineCfg::default()).await;
4129 let path = engine
4130 .materialize_system_file(&task_id, 1)
4131 .await
4132 .expect("materialize_system_file");
4133 assert!(path.is_none());
4134 }
4135}
4136
4137/// subtask-4 / ST2 rework: `submit_output` / `submit_worker_result_trusted`'s
4138/// submit-time projection sink (`Engine::materialize_final_submission`) —
4139/// the Data-plane `OutputStore` dual-write plus the
4140/// `FileProjectionAdapter`-backed file materialize, both fail-open. See
4141/// the subtask-4 Tests this module covers inline on each test.
4142#[cfg(test)]
4143mod submit_time_projection_sink_tests {
4144 use super::*;
4145 use crate::core::agent_context::AgentContextView;
4146 use crate::store::output::{ContentRef, InMemoryOutputStore, OutputEvent};
4147
4148 /// Starts a task under `agent`, returning `(engine, op_token, task_id,
4149 /// worker_token)` — same helper shape as the sibling test modules
4150 /// above (`initial_directive_value_passthrough_tests::seeded_engine` /
4151 /// `mint_worker_token`), duplicated locally per this file's
4152 /// established per-module convention.
4153 async fn seeded_task(agent: &str) -> (Engine, CapToken, StepId, CapToken) {
4154 let engine = Engine::new(EngineCfg::default());
4155 let op_token = engine
4156 .attach("ut-op", Role::Operator, Duration::from_secs(30))
4157 .await
4158 .expect("attach");
4159 let task_id = engine
4160 .start_task(
4161 &op_token,
4162 TaskSpec {
4163 agent: agent.to_string(),
4164 initial_directive: Value::String("go".into()),
4165 step_ctx: None,
4166 check_policy: None,
4167 },
4168 )
4169 .await
4170 .expect("start_task");
4171 let worker_token = engine.signer().session(
4172 format!("worker-of-{task_id}"),
4173 Role::Worker,
4174 vec!["*".into()],
4175 Duration::from_secs(600),
4176 );
4177 let fp = worker_token.fingerprint();
4178 let record = CapTokenRecord::from_worker_token(worker_token.clone(), task_id.clone());
4179 engine
4180 .with_state("test.mint_worker", move |s| {
4181 s.tokens.insert(fp, record);
4182 })
4183 .await
4184 .expect("mint worker token");
4185 (engine, op_token, task_id, worker_token)
4186 }
4187
4188 /// Sibling of [`seeded_task`] that lets a caller pin the engine's
4189 /// `EngineCfg.check_policy` before the engine is constructed — used
4190 /// by the `check_policy_*` regression tests below to exercise the
4191 /// three [`crate::core::config::CheckPolicy`] modes without touching
4192 /// the shared `seeded_task` helper (which every unrelated sink test
4193 /// depends on).
4194 async fn seeded_task_with_policy(
4195 agent: &str,
4196 policy: crate::core::config::CheckPolicy,
4197 ) -> (Engine, CapToken, StepId, CapToken) {
4198 let cfg = EngineCfg {
4199 check_policy: policy,
4200 ..EngineCfg::default()
4201 };
4202 let engine = Engine::new(cfg);
4203 let op_token = engine
4204 .attach("ut-op", Role::Operator, Duration::from_secs(30))
4205 .await
4206 .expect("attach");
4207 let task_id = engine
4208 .start_task(
4209 &op_token,
4210 TaskSpec {
4211 agent: agent.to_string(),
4212 initial_directive: Value::String("go".into()),
4213 step_ctx: None,
4214 check_policy: None,
4215 },
4216 )
4217 .await
4218 .expect("start_task");
4219 let worker_token = engine.signer().session(
4220 format!("worker-of-{task_id}"),
4221 Role::Worker,
4222 vec!["*".into()],
4223 Duration::from_secs(600),
4224 );
4225 let fp = worker_token.fingerprint();
4226 let record = CapTokenRecord::from_worker_token(worker_token.clone(), task_id.clone());
4227 engine
4228 .with_state("test.mint_worker", move |s| {
4229 s.tokens.insert(fp, record);
4230 })
4231 .await
4232 .expect("mint worker token");
4233 (engine, op_token, task_id, worker_token)
4234 }
4235
4236 /// Seeds `EngineState.agent_ctx[(task_id, attempt)].view` directly —
4237 /// the same snapshot `AgentContextMiddleware` writes at spawn time
4238 /// (see its module doc), stood up here without the full spawner
4239 /// stack so these tests can exercise `submit_output` in isolation.
4240 async fn seed_agent_context(engine: &Engine, task_id: &StepId, attempt: u32, work_dir: &str) {
4241 let task_id = task_id.clone();
4242 let work_dir = work_dir.to_string();
4243 engine
4244 .with_state("test.seed_agent_context", move |s| {
4245 s.agent_ctx.insert(
4246 (task_id, attempt),
4247 crate::core::state::AgentCtxEntry {
4248 view: AgentContextView {
4249 work_dir: Some(work_dir),
4250 ..Default::default()
4251 },
4252 policy: Default::default(),
4253 },
4254 );
4255 })
4256 .await
4257 .expect("seed agent_ctx");
4258 }
4259
4260 /// GH #27 (follow-up to #23): seeds `EngineState.agent_ctx` with an
4261 /// arbitrary `work_dir` / `project_root` pair (either may be `None`),
4262 /// unlike [`seed_agent_context`] (which only ever sets `work_dir`) —
4263 /// lets these tests exercise `ProjectionPlacement::resolve_root`'s
4264 /// fallback in both directions.
4265 async fn seed_agent_context_roots(
4266 engine: &Engine,
4267 task_id: &StepId,
4268 attempt: u32,
4269 work_dir: Option<&str>,
4270 project_root: Option<&str>,
4271 ) {
4272 let task_id = task_id.clone();
4273 let work_dir = work_dir.map(str::to_string);
4274 let project_root = project_root.map(str::to_string);
4275 engine
4276 .with_state("test.seed_agent_context_roots", move |s| {
4277 s.agent_ctx.insert(
4278 (task_id, attempt),
4279 crate::core::state::AgentCtxEntry {
4280 view: AgentContextView {
4281 work_dir,
4282 project_root,
4283 ..Default::default()
4284 },
4285 policy: Default::default(),
4286 },
4287 );
4288 })
4289 .await
4290 .expect("seed agent_ctx");
4291 }
4292
4293 /// GH #27 (follow-up to #23): seeds `EngineState.projection_placements`
4294 /// directly — the same snapshot `EngineDispatcher::dispatch` stashes
4295 /// at dispatch time (mirroring [`seed_step_naming`]'s contract) — so
4296 /// these tests can exercise a declared `ProjectionPlacement` without
4297 /// driving a real `Compiler::compile`.
4298 async fn seed_projection_placement(
4299 engine: &Engine,
4300 task_id: &StepId,
4301 placement: crate::core::projection_placement::ProjectionPlacement,
4302 ) {
4303 let task_id = task_id.clone();
4304 let placement = Arc::new(placement);
4305 engine
4306 .with_state("test.seed_projection_placement", move |s| {
4307 s.projection_placements.insert(task_id, placement);
4308 })
4309 .await
4310 .expect("seed projection_placements");
4311 }
4312
4313 /// GH #23 subtask-2: builds a fixture
4314 /// [`crate::core::step_naming::StepNaming`] table declaring `producer`
4315 /// → `canonical` (`AgentMeta.projection_name`), then seeds it into
4316 /// `EngineState.step_namings` for `task_id` — the same snapshot
4317 /// `EngineDispatcher::dispatch` stashes at dispatch time
4318 /// (`blueprint.rs`'s "construct once, read many" contract), stood up
4319 /// here without the full Blueprint-compile path so these tests can
4320 /// exercise the canonical-sink resolution in isolation.
4321 async fn seed_step_naming(engine: &Engine, task_id: &StepId, producer: &str, canonical: &str) {
4322 use crate::blueprint::{
4323 current_schema_version, AgentDef, AgentKind, AgentMeta, Blueprint, BlueprintMetadata,
4324 CompilerHints, CompilerStrategy,
4325 };
4326 use crate::core::step_naming::StepNaming;
4327 use mlua_flow_ir::{Expr, Node};
4328
4329 let flow = Node::Step {
4330 ref_: producer.to_string(),
4331 in_: Expr::Path {
4332 at: "$.in".parse().expect("literal test path: $.in"),
4333 },
4334 out: Expr::Path {
4335 at: format!("$.{producer}_out")
4336 .parse()
4337 .expect("literal test path"),
4338 },
4339 };
4340 let bp = Blueprint {
4341 schema_version: current_schema_version(),
4342 id: "sink-canonical-ut".into(),
4343 flow,
4344 agents: vec![AgentDef {
4345 name: producer.to_string(),
4346 kind: AgentKind::RustFn,
4347 spec: serde_json::json!({ "fn_id": producer }),
4348 profile: None,
4349 meta: Some(AgentMeta {
4350 projection_name: Some(canonical.to_string()),
4351 ..Default::default()
4352 }),
4353 runner: None,
4354 runner_ref: None,
4355 verdict: None,
4356 }],
4357 operators: vec![],
4358 metas: vec![],
4359 hints: CompilerHints::default(),
4360 strategy: CompilerStrategy::default(),
4361 metadata: BlueprintMetadata::default(),
4362 spawner_hints: Default::default(),
4363 default_agent_kind: AgentKind::Operator,
4364 default_operator_kind: None,
4365 default_init_ctx: None,
4366 default_agent_ctx: None,
4367 default_context_policy: None,
4368 projection_placement: None,
4369 audits: vec![],
4370 degradation_policy: None,
4371 runners: vec![],
4372 default_runner: None,
4373 subprocesses: vec![],
4374 check_policy: None,
4375 blueprint_ref_includes: Vec::new(),
4376 };
4377 let (naming, warnings) = StepNaming::from_blueprint(&bp).expect("no collision");
4378 assert!(warnings.is_empty(), "single-step fixture has no collisions");
4379 let naming = Arc::new(naming);
4380 let task_id = task_id.clone();
4381 engine
4382 .with_state("test.seed_step_naming", move |s| {
4383 s.step_namings.insert(task_id, naming);
4384 })
4385 .await
4386 .expect("seed step_namings");
4387 }
4388
4389 fn final_event(value: Value, ok: bool) -> crate::worker::output::OutputEvent {
4390 crate::worker::output::OutputEvent::Final {
4391 content: crate::worker::output::ContentRef::Inline { value },
4392 ok,
4393 }
4394 }
4395
4396 /// Subtask 4 Test #2: `submit_output`'s `Final` writes
4397 /// `<root>/workspace/tasks/<task_id>/ctx/<agent>.md`, content matching
4398 /// the submitted value.
4399 #[tokio::test]
4400 async fn submit_output_final_materializes_file_when_work_dir_resolved() {
4401 let dir = tempfile::TempDir::new().unwrap();
4402 let (engine, _op, task_id, worker_token) = seeded_task("planner").await;
4403 seed_agent_context(&engine, &task_id, 1, &dir.path().to_string_lossy()).await;
4404
4405 engine
4406 .submit_output(
4407 &worker_token,
4408 &task_id,
4409 1,
4410 final_event(serde_json::json!({"plan": "do it"}), true),
4411 )
4412 .await
4413 .expect("submit_output");
4414
4415 let expected_file = dir
4416 .path()
4417 .join("workspace/tasks")
4418 .join(task_id.as_str())
4419 .join("ctx/planner.md");
4420 assert!(
4421 expected_file.exists(),
4422 "materialized submission file missing at {expected_file:?}"
4423 );
4424 let body = std::fs::read_to_string(expected_file).unwrap();
4425 assert!(body.contains(r#""plan": "do it""#), "body: {body}");
4426 }
4427
4428 /// Subtask 4 Test #3: `work_dir` unresolved (no `agent_ctx`
4429 /// snapshot for this `(task_id, attempt)`) — submit still succeeds,
4430 /// fail-open, no file.
4431 #[tokio::test]
4432 async fn submit_output_final_skips_file_when_root_unresolved() {
4433 let (engine, _op, task_id, worker_token) = seeded_task("planner").await;
4434 // No seed_agent_context call — root is unresolved.
4435
4436 let result = engine
4437 .submit_output(
4438 &worker_token,
4439 &task_id,
4440 1,
4441 final_event(serde_json::json!("hi"), true),
4442 )
4443 .await;
4444 assert!(
4445 result.is_ok(),
4446 "submit must succeed even with no resolvable root (fail-open, Invariant 1)"
4447 );
4448 }
4449
4450 /// Regression for the check_policy cascade: the default
4451 /// [`crate::core::config::CheckPolicy::Warn`] preserves the
4452 /// pre-`CheckPolicy` fail-open semantics — a submit whose root is
4453 /// unresolved still succeeds. Byte-compat with
4454 /// `submit_output_final_skips_file_when_root_unresolved`; this test
4455 /// pins the mode explicitly so a future default change to
4456 /// `Strict` (silent breakage) is caught here.
4457 #[tokio::test]
4458 async fn submit_output_final_check_policy_warn_preserves_fail_open() {
4459 let (engine, _op, task_id, worker_token) =
4460 seeded_task_with_policy("planner", crate::core::config::CheckPolicy::Warn).await;
4461
4462 let result = engine
4463 .submit_output(
4464 &worker_token,
4465 &task_id,
4466 1,
4467 final_event(serde_json::json!("hi"), true),
4468 )
4469 .await;
4470 assert!(
4471 result.is_ok(),
4472 "Warn mode preserves fail-open: submit must succeed when root unresolved"
4473 );
4474 }
4475
4476 /// Regression for the check_policy cascade:
4477 /// [`crate::core::config::CheckPolicy::Strict`] surfaces the "no
4478 /// work_dir/project_root resolved" fail-open condition as an
4479 /// [`EngineError::CheckPolicyStrict`], letting a caller who has
4480 /// opted in fail fast instead of proceeding with a partially-
4481 /// realized submission. The error's `context` identifies the call
4482 /// site (`"file materialize"`), and `message` preserves the
4483 /// pre-`CheckPolicy` warn literal verbatim (log-parse compat).
4484 #[tokio::test]
4485 async fn submit_output_final_check_policy_strict_surfaces_error_when_root_unresolved() {
4486 let (engine, _op, task_id, worker_token) =
4487 seeded_task_with_policy("planner", crate::core::config::CheckPolicy::Strict).await;
4488
4489 let err = engine
4490 .submit_output(
4491 &worker_token,
4492 &task_id,
4493 1,
4494 final_event(serde_json::json!("hi"), true),
4495 )
4496 .await
4497 .expect_err("Strict mode must return an error when root unresolved");
4498 match err {
4499 EngineError::CheckPolicyStrict { context, message } => {
4500 assert!(
4501 context.contains("file materialize"),
4502 "context must identify the call site: {context}"
4503 );
4504 assert!(
4505 message.contains("no work_dir/project_root resolved"),
4506 "message must preserve the warn-log literal for log-parse compat: {message}"
4507 );
4508 }
4509 other => panic!(
4510 "expected EngineError::CheckPolicyStrict, got a different variant: {other:?}"
4511 ),
4512 }
4513 }
4514
4515 /// Regression for the check_policy cascade:
4516 /// [`crate::core::config::CheckPolicy::Silent`] returns `Ok(())` (
4517 /// like `Warn`) without surfacing an error. The log-suppression side
4518 /// of `Silent` (no `tracing::warn!`) is enforced at the call site
4519 /// via the `if !matches!(policy, Silent) { warn!(...) }` guard —
4520 /// verifying tracing output shape here would couple the test to a
4521 /// subscriber setup, so the assertion is limited to the error-
4522 /// return semantics (matches the helper unit tests in
4523 /// `check_policy_helper_tests`).
4524 #[tokio::test]
4525 async fn submit_output_final_check_policy_silent_returns_ok_when_root_unresolved() {
4526 let (engine, _op, task_id, worker_token) =
4527 seeded_task_with_policy("planner", crate::core::config::CheckPolicy::Silent).await;
4528
4529 let result = engine
4530 .submit_output(
4531 &worker_token,
4532 &task_id,
4533 1,
4534 final_event(serde_json::json!("hi"), true),
4535 )
4536 .await;
4537 assert!(
4538 result.is_ok(),
4539 "Silent mode returns Ok(()) at the error surface: submit must succeed"
4540 );
4541 }
4542
4543 /// Subtask 4 Test #4 (file half): re-submitting under the same
4544 /// `(task_id, agent)` overwrites the materialized file with the
4545 /// latest value.
4546 #[tokio::test]
4547 async fn resubmit_overwrites_materialized_file_with_latest() {
4548 let dir = tempfile::TempDir::new().unwrap();
4549 let (engine, _op, task_id, worker_token) = seeded_task("planner").await;
4550 seed_agent_context(&engine, &task_id, 1, &dir.path().to_string_lossy()).await;
4551
4552 engine
4553 .submit_output(
4554 &worker_token,
4555 &task_id,
4556 1,
4557 final_event(serde_json::json!("first"), true),
4558 )
4559 .await
4560 .expect("first submit");
4561 engine
4562 .submit_output(
4563 &worker_token,
4564 &task_id,
4565 1,
4566 final_event(serde_json::json!("second"), true),
4567 )
4568 .await
4569 .expect("second submit");
4570
4571 let expected_file = dir
4572 .path()
4573 .join("workspace/tasks")
4574 .join(task_id.as_str())
4575 .join("ctx/planner.md");
4576 let body = std::fs::read_to_string(expected_file).unwrap();
4577 assert!(body.contains("second"), "body must reflect latest: {body}");
4578 assert!(
4579 !body.contains("first"),
4580 "body must not carry the stale value: {body}"
4581 );
4582 }
4583
4584 /// GH #27 (follow-up to #23): the byte-compat default
4585 /// `ProjectionPlacement` (`root_preference = WorkDir`) falls back to
4586 /// `project_root` when `work_dir` is absent — the same fallback
4587 /// [`crate::core::projection_placement::ProjectionPlacement::resolve_root`]
4588 /// now performs for every one of the "3 path" call sites, this one
4589 /// exercised at the submit-sink layer.
4590 #[tokio::test]
4591 async fn submit_output_final_falls_back_to_project_root_when_work_dir_absent() {
4592 let dir = tempfile::TempDir::new().unwrap();
4593 let (engine, _op, task_id, worker_token) = seeded_task("planner").await;
4594 seed_agent_context_roots(
4595 &engine,
4596 &task_id,
4597 1,
4598 None,
4599 Some(&dir.path().to_string_lossy()),
4600 )
4601 .await;
4602
4603 engine
4604 .submit_output(
4605 &worker_token,
4606 &task_id,
4607 1,
4608 final_event(serde_json::json!({"plan": "via project_root"}), true),
4609 )
4610 .await
4611 .expect("submit_output");
4612
4613 let expected_file = dir
4614 .path()
4615 .join("workspace/tasks")
4616 .join(task_id.as_str())
4617 .join("ctx/planner.md");
4618 assert!(
4619 expected_file.exists(),
4620 "materialized submission file missing at {expected_file:?} \
4621 (work_dir absent must fall back to project_root)"
4622 );
4623 }
4624
4625 /// GH #27 (follow-up to #23): a declared `ProjectionPlacement`
4626 /// (`root_preference = ProjectRoot`, custom `dir_template`) changes
4627 /// BOTH which root is preferred (project_root wins even though
4628 /// work_dir is also present) AND the target directory layout — proof
4629 /// the submit sink consults the snapshotted resolver rather than a
4630 /// hardcoded layout.
4631 #[tokio::test]
4632 async fn submit_output_final_uses_declared_projection_placement() {
4633 let work_dir = tempfile::TempDir::new().unwrap();
4634 let project_root = tempfile::TempDir::new().unwrap();
4635 let (engine, _op, task_id, worker_token) = seeded_task("planner").await;
4636 seed_agent_context_roots(
4637 &engine,
4638 &task_id,
4639 1,
4640 Some(&work_dir.path().to_string_lossy()),
4641 Some(&project_root.path().to_string_lossy()),
4642 )
4643 .await;
4644 seed_projection_placement(
4645 &engine,
4646 &task_id,
4647 crate::core::projection_placement::ProjectionPlacement {
4648 root_preference: crate::core::projection_placement::RootPreference::ProjectRoot,
4649 dir_template: "custom/{task_id}/out".to_string(),
4650 },
4651 )
4652 .await;
4653
4654 engine
4655 .submit_output(
4656 &worker_token,
4657 &task_id,
4658 1,
4659 final_event(serde_json::json!({"plan": "via custom placement"}), true),
4660 )
4661 .await
4662 .expect("submit_output");
4663
4664 let expected_file = project_root
4665 .path()
4666 .join("custom")
4667 .join(task_id.as_str())
4668 .join("out/planner.md");
4669 assert!(
4670 expected_file.exists(),
4671 "materialized submission file missing at custom placement target {expected_file:?}"
4672 );
4673 let unexpected_file = work_dir
4674 .path()
4675 .join("workspace/tasks")
4676 .join(task_id.as_str())
4677 .join("ctx/planner.md");
4678 assert!(
4679 !unexpected_file.exists(),
4680 "declared root_preference=ProjectRoot must not fall back to work_dir: {unexpected_file:?}"
4681 );
4682 }
4683
4684 /// Subtask 4 Invariant 3 / crux requirement #3: when
4685 /// [`Engine::set_output_store`] wires a Data-plane [`crate::store::output::OutputStore`],
4686 /// `submit_output`'s `Final` dual-writes into it under
4687 /// `producer_agent = TaskState.spec.agent` — the store becomes
4688 /// queryable via `get_latest_by_name`, independent of whether a root
4689 /// resolved for the file half.
4690 #[tokio::test]
4691 async fn submit_output_final_dual_writes_into_configured_output_store() {
4692 let (engine, _op, task_id, worker_token) = seeded_task("reviewer").await;
4693 let data_store: Arc<dyn crate::store::output::OutputStore> =
4694 Arc::new(InMemoryOutputStore::new());
4695 engine.set_output_store(data_store.clone());
4696
4697 engine
4698 .submit_output(
4699 &worker_token,
4700 &task_id,
4701 1,
4702 final_event(serde_json::json!({"verdict": "pass"}), true),
4703 )
4704 .await
4705 .expect("submit_output");
4706
4707 let record = data_store
4708 .get_latest_by_name("reviewer")
4709 .await
4710 .expect("dual-written record");
4711 match record.event {
4712 OutputEvent::Final { content, ok } => {
4713 assert!(ok);
4714 match content {
4715 ContentRef::Inline { value } => {
4716 assert_eq!(value, serde_json::json!({"verdict": "pass"}));
4717 }
4718 other => panic!("expected Inline content, got {other:?}"),
4719 }
4720 }
4721 other => panic!("expected Final event, got {other:?}"),
4722 }
4723 }
4724
4725 /// GH #34 subtask-3 gap fix: an `Artifact` event submitted via
4726 /// `submit_output` dual-writes into a wired Data-plane `OutputStore`
4727 /// under its OWN `name`, verbatim — mirrors
4728 /// `submit_output_final_dual_writes_into_configured_output_store`
4729 /// above, but for the `Artifact` variant.
4730 #[tokio::test]
4731 async fn submit_output_artifact_dual_writes_into_configured_output_store() {
4732 let (engine, _op, task_id, worker_token) = seeded_task("echo").await;
4733 let data_store: Arc<dyn crate::store::output::OutputStore> =
4734 Arc::new(InMemoryOutputStore::new());
4735 engine.set_output_store(data_store.clone());
4736
4737 engine
4738 .submit_output(
4739 &worker_token,
4740 &task_id,
4741 1,
4742 OutputEvent::Artifact {
4743 name: "audit:echo".to_string(),
4744 content: ContentRef::Inline {
4745 value: serde_json::json!({"finding": "clean"}),
4746 },
4747 },
4748 )
4749 .await
4750 .expect("submit_output");
4751
4752 let record = data_store
4753 .get_latest_by_name("audit:echo")
4754 .await
4755 .expect("dual-written artifact record");
4756 match record.event {
4757 OutputEvent::Artifact { name, content } => {
4758 assert_eq!(name, "audit:echo");
4759 match content {
4760 ContentRef::Inline { value } => {
4761 assert_eq!(value, serde_json::json!({"finding": "clean"}));
4762 }
4763 other => panic!("expected Inline content, got {other:?}"),
4764 }
4765 }
4766 other => panic!("expected Artifact event, got {other:?}"),
4767 }
4768 // The `Artifact` dual-write must never collide with / overwrite
4769 // the producing step's own `Final` name — `submit_output` never
4770 // materialized a `Final` here, so `"echo"` must stay unresolved.
4771 assert!(
4772 data_store.get_latest_by_name("echo").await.is_err(),
4773 "artifact write must not fabricate a record under the raw producer_agent name"
4774 );
4775 }
4776
4777 /// Invariant 1 (fail-open) for `Artifact`, mirroring
4778 /// `submit_output_final_skips_file_when_root_unresolved`'s Final-side
4779 /// coverage: no `OutputStore` wired at all — submit still succeeds.
4780 #[tokio::test]
4781 async fn submit_output_artifact_is_fail_open_when_no_output_store_configured() {
4782 let (engine, _op, task_id, worker_token) = seeded_task("echo").await;
4783
4784 let result = engine
4785 .submit_output(
4786 &worker_token,
4787 &task_id,
4788 1,
4789 OutputEvent::Artifact {
4790 name: "audit:echo".to_string(),
4791 content: ContentRef::Inline {
4792 value: serde_json::json!("finding"),
4793 },
4794 },
4795 )
4796 .await;
4797 assert!(
4798 result.is_ok(),
4799 "submit must succeed even with no OutputStore wired (fail-open, Invariant 1)"
4800 );
4801 }
4802
4803 /// `submit_worker_result_trusted` (the `/v1/worker/submit` short-handle
4804 /// path) triggers the exact same sink as `submit_output` — parity
4805 /// across both worker-submit entry points.
4806 #[tokio::test]
4807 async fn submit_worker_result_trusted_also_triggers_projection_sink() {
4808 let dir = tempfile::TempDir::new().unwrap();
4809 let (engine, _op, task_id, _worker_token) = seeded_task("planner").await;
4810 seed_agent_context(&engine, &task_id, 1, &dir.path().to_string_lossy()).await;
4811 let data_store: Arc<dyn crate::store::output::OutputStore> =
4812 Arc::new(InMemoryOutputStore::new());
4813 engine.set_output_store(data_store.clone());
4814
4815 engine
4816 .submit_worker_result_trusted(
4817 &task_id,
4818 1,
4819 serde_json::json!("trusted-value"),
4820 SubmitOutcome::Pass,
4821 )
4822 .await
4823 .expect("submit_worker_result_trusted");
4824
4825 let expected_file = dir
4826 .path()
4827 .join("workspace/tasks")
4828 .join(task_id.as_str())
4829 .join("ctx/planner.md");
4830 assert!(expected_file.exists());
4831 let record = data_store
4832 .get_latest_by_name("planner")
4833 .await
4834 .expect("dual-written record");
4835 assert!(matches!(record.event, OutputEvent::Final { ok: true, .. }));
4836 }
4837
4838 /// GH #23 subtask-2 (canonical sink): a declared `projection_name`
4839 /// (`AgentMeta.projection_name`, surfaced via `StepNaming`) redirects
4840 /// `submit_output`'s Final canonical sink — both the Data-plane
4841 /// dual-write name and the materialized file stem resolve to the
4842 /// canonical name, not the raw `producer_agent`.
4843 #[tokio::test]
4844 async fn submit_output_final_uses_canonical_name_when_step_naming_declares_one() {
4845 let dir = tempfile::TempDir::new().unwrap();
4846 let (engine, _op, task_id, worker_token) = seeded_task("reviewer").await;
4847 seed_agent_context(&engine, &task_id, 1, &dir.path().to_string_lossy()).await;
4848 seed_step_naming(&engine, &task_id, "reviewer", "verdict-final").await;
4849 let data_store: Arc<dyn crate::store::output::OutputStore> =
4850 Arc::new(InMemoryOutputStore::new());
4851 engine.set_output_store(data_store.clone());
4852
4853 engine
4854 .submit_output(
4855 &worker_token,
4856 &task_id,
4857 1,
4858 final_event(serde_json::json!({"verdict": "pass"}), true),
4859 )
4860 .await
4861 .expect("submit_output");
4862
4863 let record = data_store
4864 .get_latest_by_name("verdict-final")
4865 .await
4866 .expect("dual-written record under canonical name");
4867 assert!(matches!(record.event, OutputEvent::Final { ok: true, .. }));
4868 assert!(
4869 data_store.get_latest_by_name("reviewer").await.is_err(),
4870 "raw producer_agent name must not be written once canonical resolves"
4871 );
4872
4873 let expected_file = dir
4874 .path()
4875 .join("workspace/tasks")
4876 .join(task_id.as_str())
4877 .join("ctx/verdict-final.md");
4878 assert!(
4879 expected_file.exists(),
4880 "materialized file stem must be canonical at {expected_file:?}"
4881 );
4882 }
4883
4884 /// GH #23 subtask-2: no `StepNaming` table snapshotted for this
4885 /// `task_id` (the pre-GH-#23 / no-`with_step_naming` path) is a
4886 /// defensive fail-open — the canonical sink falls back to the raw
4887 /// `producer_agent`, byte-identical to
4888 /// `submit_output_final_dual_writes_into_configured_output_store`
4889 /// above (which never calls `seed_step_naming`).
4890 #[tokio::test]
4891 async fn submit_output_final_falls_back_to_producer_agent_when_no_step_naming_table() {
4892 let (engine, _op, task_id, worker_token) = seeded_task("reviewer").await;
4893 let data_store: Arc<dyn crate::store::output::OutputStore> =
4894 Arc::new(InMemoryOutputStore::new());
4895 engine.set_output_store(data_store.clone());
4896
4897 engine
4898 .submit_output(
4899 &worker_token,
4900 &task_id,
4901 1,
4902 final_event(serde_json::json!({"verdict": "pass"}), true),
4903 )
4904 .await
4905 .expect("submit_output");
4906
4907 let record = data_store
4908 .get_latest_by_name("reviewer")
4909 .await
4910 .expect("fail-open dual-write under raw producer_agent name");
4911 assert!(matches!(record.event, OutputEvent::Final { ok: true, .. }));
4912 }
4913
4914 /// GH #23 subtask-2 (Layer 2): `OutputStore::get_latest_by_name_in_run`
4915 /// resolves the value `submit_output` dual-wrote for this exact
4916 /// `(task_id, attempt)` run, independent of `get_latest_by_name`'s
4917 /// cross-Run race (two Runs sharing a producer name never bleed into
4918 /// each other through the Run-scoped accessor).
4919 #[tokio::test]
4920 async fn submit_output_final_is_resolvable_via_run_scoped_lookup() {
4921 let (engine, _op, task_id, worker_token) = seeded_task("reviewer").await;
4922 let data_store: Arc<dyn crate::store::output::OutputStore> =
4923 Arc::new(InMemoryOutputStore::new());
4924 engine.set_output_store(data_store.clone());
4925
4926 engine
4927 .submit_output(
4928 &worker_token,
4929 &task_id,
4930 1,
4931 final_event(serde_json::json!({"verdict": "pass"}), true),
4932 )
4933 .await
4934 .expect("submit_output");
4935
4936 let record = data_store
4937 .get_latest_by_name_in_run(task_id.as_str(), 1, "reviewer")
4938 .await
4939 .expect("run-scoped lookup resolves the dual-written record");
4940 assert!(matches!(record.event, OutputEvent::Final { ok: true, .. }));
4941
4942 // A different attempt of the same task must not resolve — the
4943 // Run-scoped lookup does not fall back across attempts.
4944 assert!(
4945 data_store
4946 .get_latest_by_name_in_run(task_id.as_str(), 2, "reviewer")
4947 .await
4948 .is_err(),
4949 "a different attempt must not resolve the same-named record"
4950 );
4951 }
4952
4953 // ─── staged part file materialize ───
4954
4955 /// Staging a part with a resolved `work_dir` writes
4956 /// `<work_dir>/workspace/tasks/<task_id>/ctx/<name>` with the part's
4957 /// content RAW (no front matter / fenced wrapper).
4958 #[tokio::test]
4959 async fn stage_artifact_materializes_part_file_when_work_dir_resolved() {
4960 let dir = tempfile::TempDir::new().unwrap();
4961 let (engine, _op, task_id, _worker_token) = seeded_task("planner").await;
4962 seed_agent_context(&engine, &task_id, 1, &dir.path().to_string_lossy()).await;
4963
4964 engine
4965 .stage_worker_artifact_trusted(
4966 &task_id,
4967 1,
4968 "plan.md".to_string(),
4969 serde_json::json!("# Plan\n\nstep one\n"),
4970 )
4971 .await
4972 .expect("stage artifact");
4973
4974 let expected_file = dir
4975 .path()
4976 .join("workspace/tasks")
4977 .join(task_id.as_str())
4978 .join("ctx/plan.md");
4979 assert!(
4980 expected_file.exists(),
4981 "materialized part file missing at {expected_file:?}"
4982 );
4983 let body = std::fs::read_to_string(expected_file).unwrap();
4984 // Raw — no YAML front matter / fenced-json wrapper.
4985 assert_eq!(body, "# Plan\n\nstep one\n");
4986 }
4987
4988 /// No resolvable root + `Warn` — staging still
4989 /// succeeds (fail-open), and no part file is written.
4990 #[tokio::test]
4991 async fn stage_artifact_check_policy_warn_skips_part_file_when_root_unresolved() {
4992 let dir = tempfile::TempDir::new().unwrap();
4993 let (engine, _op, task_id, _worker_token) =
4994 seeded_task_with_policy("planner", crate::core::config::CheckPolicy::Warn).await;
4995 // No seed_agent_context — root unresolved.
4996
4997 let result = engine
4998 .stage_worker_artifact_trusted(
4999 &task_id,
5000 1,
5001 "plan.md".to_string(),
5002 serde_json::json!("x"),
5003 )
5004 .await;
5005 assert!(
5006 result.is_ok(),
5007 "Warn mode preserves fail-open: stage must succeed when root unresolved"
5008 );
5009 assert!(
5010 !dir.path().join("workspace").exists(),
5011 "no part file may be materialized when root is unresolved"
5012 );
5013 }
5014
5015 /// No resolvable root + `Strict` — staging surfaces
5016 /// the fail-open condition as an [`EngineError::CheckPolicyStrict`],
5017 /// its message identifying the "part file materialize" call site.
5018 #[tokio::test]
5019 async fn stage_artifact_check_policy_strict_surfaces_error_when_root_unresolved() {
5020 let (engine, _op, task_id, _worker_token) =
5021 seeded_task_with_policy("planner", crate::core::config::CheckPolicy::Strict).await;
5022
5023 let err = engine
5024 .stage_worker_artifact_trusted(
5025 &task_id,
5026 1,
5027 "plan.md".to_string(),
5028 serde_json::json!("x"),
5029 )
5030 .await
5031 .expect_err("Strict mode must return an error when root unresolved");
5032 match err {
5033 EngineError::CheckPolicyStrict { context, message } => {
5034 assert!(
5035 context.contains("part file materialize"),
5036 "context must identify the call site: {context}"
5037 );
5038 assert!(
5039 message.contains("part file materialize"),
5040 "message must identify the part-file sink: {message}"
5041 );
5042 assert!(
5043 message.contains("no work_dir/project_root resolved"),
5044 "message must preserve the warn-log literal: {message}"
5045 );
5046 }
5047 other => panic!(
5048 "expected EngineError::CheckPolicyStrict, got a different variant: {other:?}"
5049 ),
5050 }
5051 }
5052
5053 /// A path-traversal `name` (`../evil.md`) with a
5054 /// resolved root — the name guard fails the write, but fail-open keeps
5055 /// the stage succeeding, and nothing is written outside the ctx dir.
5056 #[tokio::test]
5057 async fn stage_artifact_traversal_name_is_fail_open_and_writes_nothing() {
5058 let dir = tempfile::TempDir::new().unwrap();
5059 let (engine, _op, task_id, _worker_token) = seeded_task("planner").await;
5060 seed_agent_context(&engine, &task_id, 1, &dir.path().to_string_lossy()).await;
5061
5062 let result = engine
5063 .stage_worker_artifact_trusted(
5064 &task_id,
5065 1,
5066 "../evil.md".to_string(),
5067 serde_json::json!("pwned"),
5068 )
5069 .await;
5070 assert!(
5071 result.is_ok(),
5072 "default (Warn) policy is fail-open even on a rejected part name"
5073 );
5074 // The escaped target (ctx dir's parent) must not have been written.
5075 let escaped = dir
5076 .path()
5077 .join("workspace/tasks")
5078 .join(task_id.as_str())
5079 .join("evil.md");
5080 assert!(
5081 !escaped.exists(),
5082 "a traversal name must never write outside the ctx dir: {escaped:?}"
5083 );
5084 }
5085}
5086
5087/// GH #36 ST1: named multi-part worker output. Covers (a) the pure
5088/// `fold_final_and_parts` assembly `dispatch_attempt_with`'s Final-pull
5089/// delegates to, (b) `stage_worker_artifact_trusted`'s per-attempt
5090/// isolation on `EngineState.output_store` / `.worker_artifact_names` (the
5091/// same `HashMap<(StepId, u32), _>` key shape `submit_worker_result_trusted`
5092/// uses — a fresh attempt is a fresh key, so nothing to explicitly "clean
5093/// up"), and (c) the allowlist behavior that keeps a non-opt-in `Artifact`
5094/// producer (e.g. `AfterRunAuditMiddleware`) from being folded in.
5095#[cfg(test)]
5096mod named_multi_part_worker_output_tests {
5097 use super::*;
5098 use crate::worker::output::{ContentRef, OutputEvent};
5099
5100 fn artifact(name: &str, value: Value) -> OutputEvent {
5101 OutputEvent::Artifact {
5102 name: name.to_string(),
5103 content: ContentRef::Inline { value },
5104 }
5105 }
5106
5107 fn final_ev(value: Value, ok: bool) -> OutputEvent {
5108 OutputEvent::Final {
5109 content: ContentRef::Inline { value },
5110 ok,
5111 }
5112 }
5113
5114 fn names(list: &[&str]) -> Vec<String> {
5115 list.iter().map(|s| s.to_string()).collect()
5116 }
5117
5118 /// Two staged parts (both in `staged_names`) + a `Final` fold into
5119 /// `{"out", "parts"}`, each value carried through verbatim.
5120 #[test]
5121 fn fold_final_and_parts_assembles_out_and_parts_shape() {
5122 let tail = vec![
5123 artifact("summary", serde_json::json!("the summary")),
5124 artifact("diff", serde_json::json!({"lines": 3})),
5125 final_ev(serde_json::json!("final text"), true),
5126 ];
5127 let staged = names(&["summary", "diff"]);
5128 let (value, ok) = fold_final_and_parts(&tail, &staged).expect("Final present");
5129 assert!(ok);
5130 assert_eq!(
5131 value,
5132 serde_json::json!({
5133 "out": "final text",
5134 "parts": {
5135 "summary": "the summary",
5136 "diff": {"lines": 3},
5137 }
5138 })
5139 );
5140 }
5141
5142 /// Zero staged parts: the value is exactly the plain `Final` value — no
5143 /// `{"out", "parts"}` wrapping. This is the back-compat guarantee (GH
5144 /// #36 must not change the shape for a worker that never POSTs to
5145 /// `/v1/worker/artifact`).
5146 #[test]
5147 fn fold_final_and_parts_with_no_parts_returns_plain_final_value() {
5148 let tail = vec![final_ev(serde_json::json!("plain value"), true)];
5149 let (value, ok) = fold_final_and_parts(&tail, &[]).expect("Final present");
5150 assert!(ok);
5151 assert_eq!(value, serde_json::json!("plain value"));
5152 }
5153
5154 /// The same staged part `name` appearing twice in one attempt: the
5155 /// LATER (tail-order) value wins — `parts` is a `Map`, not an
5156 /// accumulating list.
5157 #[test]
5158 fn fold_final_and_parts_same_name_twice_last_write_wins() {
5159 let tail = vec![
5160 artifact("a", serde_json::json!("first")),
5161 artifact("a", serde_json::json!("second")),
5162 final_ev(serde_json::json!("f"), true),
5163 ];
5164 let staged = names(&["a"]);
5165 let (value, _ok) = fold_final_and_parts(&tail, &staged).expect("Final present");
5166 assert_eq!(
5167 value,
5168 serde_json::json!({"out": "f", "parts": {"a": "second"}})
5169 );
5170 }
5171
5172 /// No `Final` anywhere in the tail (only staged parts, e.g. the worker
5173 /// crashed before submitting) — `None`, the caller's pre-existing "no
5174 /// Final in output_tail" error path.
5175 #[test]
5176 fn fold_final_and_parts_returns_none_when_no_final_present() {
5177 let tail = vec![artifact("a", serde_json::json!("v"))];
5178 let staged = names(&["a"]);
5179 assert!(fold_final_and_parts(&tail, &staged).is_none());
5180 }
5181
5182 /// An `Artifact` on the tail whose name is NOT in `staged_names` (e.g.
5183 /// `AfterRunAuditMiddleware`'s `"audit:<step_ref>"` sidecar finding on
5184 /// an audited step's own tail) must NOT be folded into `"parts"` — the
5185 /// value stays the plain `Final` value, exactly the pre-GH-#36
5186 /// behavior for every producer that isn't the worker's own
5187 /// `/v1/worker/artifact` staging. This is the regression this fold was
5188 /// almost shipped without (see `dispatch_attempt_with`'s doc).
5189 #[test]
5190 fn fold_final_and_parts_ignores_artifacts_outside_the_staged_allowlist() {
5191 let tail = vec![
5192 final_ev(serde_json::json!({"echoed": "hi"}), true),
5193 artifact("audit:echo", serde_json::json!({"finding": "clean"})),
5194 ];
5195 // `staged_names` empty: the worker itself never staged anything —
5196 // the audit sidecar Artifact must be ignored.
5197 let (value, ok) = fold_final_and_parts(&tail, &[]).expect("Final present");
5198 assert!(ok);
5199 assert_eq!(value, serde_json::json!({"echoed": "hi"}));
5200 }
5201
5202 /// Mixed tail: one staged (allowlisted) part and one non-staged
5203 /// (audit-style) `Artifact` — only the staged one is folded in.
5204 #[test]
5205 fn fold_final_and_parts_folds_only_the_staged_subset_of_a_mixed_tail() {
5206 let tail = vec![
5207 artifact("summary", serde_json::json!("s")),
5208 artifact("audit:echo", serde_json::json!({"finding": "clean"})),
5209 final_ev(serde_json::json!("f"), true),
5210 ];
5211 let staged = names(&["summary"]);
5212 let (value, _ok) = fold_final_and_parts(&tail, &staged).expect("Final present");
5213 assert_eq!(
5214 value,
5215 serde_json::json!({"out": "f", "parts": {"summary": "s"}})
5216 );
5217 }
5218
5219 /// `stage_worker_artifact_trusted` writes onto the `(task_id, attempt)`
5220 /// key exactly like `submit_worker_result_trusted` does — a part staged
5221 /// under attempt N is invisible to an `output_tail` / allowlist read of
5222 /// attempt N+1 (a fresh attempt starts empty; nothing carries over).
5223 #[tokio::test]
5224 async fn stage_worker_artifact_trusted_is_isolated_per_attempt() {
5225 let engine = Engine::new(EngineCfg::default());
5226 let task_id = StepId::new();
5227
5228 engine
5229 .stage_worker_artifact_trusted(&task_id, 1, "a".to_string(), serde_json::json!("v1"))
5230 .await
5231 .expect("stage attempt 1");
5232
5233 let attempt_1_tail = engine.output_tail(&task_id, 1).await;
5234 assert_eq!(attempt_1_tail.len(), 1);
5235 assert!(matches!(
5236 &attempt_1_tail[0],
5237 OutputEvent::Artifact { name, .. } if name == "a"
5238 ));
5239 assert_eq!(
5240 engine.worker_artifact_names_for(&task_id, 1).await,
5241 vec!["a".to_string()]
5242 );
5243
5244 let attempt_2_tail = engine.output_tail(&task_id, 2).await;
5245 assert!(
5246 attempt_2_tail.is_empty(),
5247 "attempt 2 must not see attempt 1's staged part"
5248 );
5249 assert!(
5250 engine
5251 .worker_artifact_names_for(&task_id, 2)
5252 .await
5253 .is_empty(),
5254 "attempt 2's allowlist must not see attempt 1's staged name"
5255 );
5256 }
5257}
5258
5259// ─── GH #50 (Subtask 2): `Engine::register_verdict_contracts` /
5260// `Engine::verdict_contract_for_task` ────────────────────────────────────
5261#[cfg(test)]
5262mod verdict_contract_registry_tests {
5263 use super::*;
5264
5265 async fn seeded_engine(agent: &str) -> (Engine, StepId) {
5266 let engine = Engine::new(EngineCfg::default());
5267 let op_token = engine
5268 .attach("ut-op", Role::Operator, Duration::from_secs(30))
5269 .await
5270 .expect("attach");
5271 let task_id = engine
5272 .start_task(
5273 &op_token,
5274 TaskSpec {
5275 agent: agent.to_string(),
5276 initial_directive: serde_json::json!("x"),
5277 step_ctx: None,
5278 check_policy: None,
5279 },
5280 )
5281 .await
5282 .expect("start_task");
5283 (engine, task_id)
5284 }
5285
5286 /// An agent with no registered contract at all → `None` (the opt-in
5287 /// default; every pre-GH-#50 `Engine`).
5288 #[tokio::test]
5289 async fn returns_none_when_no_contract_registered_for_the_agent() {
5290 let (engine, task_id) = seeded_engine("gate").await;
5291 assert_eq!(engine.verdict_contract_for_task(&task_id).await, None);
5292 }
5293
5294 /// A registered contract for the running task's agent is returned
5295 /// verbatim.
5296 #[tokio::test]
5297 async fn returns_the_registered_contract_for_the_running_agent() {
5298 let (engine, task_id) = seeded_engine("gate").await;
5299 let contract = mlua_swarm_schema::VerdictContract {
5300 channel: mlua_swarm_schema::VerdictChannel::Body,
5301 values: vec!["PASS".to_string(), "BLOCKED".to_string()],
5302 };
5303 engine.register_verdict_contracts(HashMap::from([("gate".to_string(), contract.clone())]));
5304 assert_eq!(
5305 engine.verdict_contract_for_task(&task_id).await,
5306 Some(contract)
5307 );
5308 }
5309
5310 /// A registered contract for a DIFFERENT agent name never leaks onto
5311 /// an unrelated task.
5312 #[tokio::test]
5313 async fn does_not_leak_a_contract_registered_for_a_different_agent() {
5314 let (engine, task_id) = seeded_engine("gate").await;
5315 engine.register_verdict_contracts(HashMap::from([(
5316 "other-agent".to_string(),
5317 mlua_swarm_schema::VerdictContract {
5318 channel: mlua_swarm_schema::VerdictChannel::Body,
5319 values: vec!["PASS".to_string()],
5320 },
5321 )]));
5322 assert_eq!(engine.verdict_contract_for_task(&task_id).await, None);
5323 }
5324
5325 /// An unknown `task_id` → `None`, not a panic / error.
5326 #[tokio::test]
5327 async fn returns_none_for_an_unknown_task_id() {
5328 let engine = Engine::new(EngineCfg::default());
5329 let unknown = StepId::new();
5330 assert_eq!(engine.verdict_contract_for_task(&unknown).await, None);
5331 }
5332
5333 /// `register_verdict_contracts` is additive (`HashMap::extend`): a
5334 /// second call registering a DIFFERENT agent does not clobber the
5335 /// first call's entry.
5336 #[tokio::test]
5337 async fn register_verdict_contracts_is_additive_across_calls() {
5338 let (engine, task_id) = seeded_engine("gate").await;
5339 let contract = mlua_swarm_schema::VerdictContract {
5340 channel: mlua_swarm_schema::VerdictChannel::Part,
5341 values: vec!["ALLOW".to_string()],
5342 };
5343 engine.register_verdict_contracts(HashMap::from([("gate".to_string(), contract.clone())]));
5344 engine.register_verdict_contracts(HashMap::from([(
5345 "unrelated-agent".to_string(),
5346 mlua_swarm_schema::VerdictContract {
5347 channel: mlua_swarm_schema::VerdictChannel::Body,
5348 values: vec!["X".to_string()],
5349 },
5350 )]));
5351 assert_eq!(
5352 engine.verdict_contract_for_task(&task_id).await,
5353 Some(contract)
5354 );
5355 }
5356}
5357
5358// ─── GH #51: completion-time verdict-contract enforcement — the shared
5359// `Engine::verdict_contract_completion_check` choke point embedded inside
5360// `submit_worker_result_trusted` / `submit_output`, exercised here at the
5361// `submit_output` level (the WS Operator fallback route's own unit-test
5362// coverage — see `crates/mlua-swarm-server/tests/verdict_contract.rs` for
5363// the HTTP-round-trip coverage of the other 2 routes) ───────────────────
5364#[cfg(test)]
5365mod verdict_contract_completion_tests {
5366 use super::*;
5367
5368 /// Seeds a `Pending` task bound to `agent` and mints a bound
5369 /// `Role::Worker` token for it — the same mint-and-register pattern
5370 /// `initial_directive_value_passthrough_tests::mint_worker_token`
5371 /// uses (duplicated here: that helper is private to its own sibling
5372 /// `#[cfg(test)]` module, not reachable via `super::*` from this one).
5373 async fn seeded_task_with_worker_token(agent: &str) -> (Engine, CapToken, StepId) {
5374 let engine = Engine::new(EngineCfg::default());
5375 let op_token = engine
5376 .attach("ut-op", Role::Operator, Duration::from_secs(30))
5377 .await
5378 .expect("attach");
5379 let task_id = engine
5380 .start_task(
5381 &op_token,
5382 TaskSpec {
5383 agent: agent.to_string(),
5384 initial_directive: serde_json::json!("x"),
5385 step_ctx: None,
5386 check_policy: None,
5387 },
5388 )
5389 .await
5390 .expect("start_task");
5391 let worker_token = engine.signer().session(
5392 format!("worker-of-{task_id}"),
5393 Role::Worker,
5394 vec!["*".into()],
5395 Duration::from_secs(600),
5396 );
5397 let fp = worker_token.fingerprint();
5398 let record = CapTokenRecord::from_worker_token(worker_token.clone(), task_id.clone());
5399 engine
5400 .with_state("test.mint_worker", move |s| {
5401 s.tokens.insert(fp, record);
5402 })
5403 .await
5404 .expect("mint worker token");
5405 (engine, worker_token, task_id)
5406 }
5407
5408 fn body_contract(values: &[&str]) -> mlua_swarm_schema::VerdictContract {
5409 mlua_swarm_schema::VerdictContract {
5410 channel: mlua_swarm_schema::VerdictChannel::Body,
5411 values: values.iter().map(|v| v.to_string()).collect(),
5412 }
5413 }
5414
5415 fn part_contract(values: &[&str]) -> mlua_swarm_schema::VerdictContract {
5416 mlua_swarm_schema::VerdictContract {
5417 channel: mlua_swarm_schema::VerdictChannel::Part,
5418 values: values.iter().map(|v| v.to_string()).collect(),
5419 }
5420 }
5421
5422 fn final_event(value: Value, ok: bool) -> crate::worker::output::OutputEvent {
5423 crate::worker::output::OutputEvent::Final {
5424 content: crate::worker::output::ContentRef::Inline { value },
5425 ok,
5426 }
5427 }
5428
5429 /// Route 3 (WS Operator fallback, `submit_output` level) — a
5430 /// `channel: "part"` contract's attempt completes via a plain
5431 /// `Final` without ever staging a `"verdict"` artifact: rejected
5432 /// with `EngineError::VerdictPartMissing`, and nothing lands on
5433 /// `output_tail` — the rejected value never reaches the flow ctx.
5434 #[tokio::test]
5435 async fn submit_output_rejects_missing_verdict_part() {
5436 let (engine, token, task_id) = seeded_task_with_worker_token("gate").await;
5437 engine.register_verdict_contracts(HashMap::from([(
5438 "gate".to_string(),
5439 part_contract(&["PASS", "BLOCKED"]),
5440 )]));
5441
5442 let err = engine
5443 .submit_output(
5444 &token,
5445 &task_id,
5446 1,
5447 final_event(serde_json::json!("anything"), true),
5448 )
5449 .await
5450 .expect_err("missing staged verdict part must be rejected");
5451 assert!(
5452 matches!(err, EngineError::VerdictPartMissing { .. }),
5453 "unexpected error variant: {err:?}"
5454 );
5455
5456 let tail = engine.output_tail(&task_id, 1).await;
5457 assert!(
5458 !tail
5459 .iter()
5460 .any(|ev| matches!(ev, crate::worker::output::OutputEvent::Final { .. })),
5461 "a rejected completion must not write a Final onto output_tail"
5462 );
5463 }
5464
5465 /// Route 3 — a `channel: "part"` contract completes normally when the
5466 /// worker DID stage a matching `"verdict"` artifact first (defense in
5467 /// depth: presence AND membership both hold).
5468 #[tokio::test]
5469 async fn submit_output_accepts_when_verdict_part_is_staged_and_a_member() {
5470 let (engine, token, task_id) = seeded_task_with_worker_token("gate").await;
5471 engine.register_verdict_contracts(HashMap::from([(
5472 "gate".to_string(),
5473 part_contract(&["PASS", "BLOCKED"]),
5474 )]));
5475 engine
5476 .stage_worker_artifact_trusted(
5477 &task_id,
5478 1,
5479 "verdict".to_string(),
5480 serde_json::json!("PASS"),
5481 )
5482 .await
5483 .expect("stage verdict part");
5484
5485 engine
5486 .submit_output(
5487 &token,
5488 &task_id,
5489 1,
5490 final_event(serde_json::json!("full report"), true),
5491 )
5492 .await
5493 .expect("staged + member verdict part must be accepted");
5494
5495 let tail = engine.output_tail(&task_id, 1).await;
5496 assert!(
5497 tail.iter()
5498 .any(|ev| matches!(ev, crate::worker::output::OutputEvent::Final { .. })),
5499 "an accepted completion must write its Final onto output_tail"
5500 );
5501 }
5502
5503 /// Route 3 — a `channel: "body"` contract's completing value is NOT a
5504 /// member of `values`: rejected with
5505 /// `EngineError::VerdictValueRejected`, no `Final` written.
5506 #[tokio::test]
5507 async fn submit_output_rejects_body_value_outside_contract() {
5508 let (engine, token, task_id) = seeded_task_with_worker_token("gate").await;
5509 engine.register_verdict_contracts(HashMap::from([(
5510 "gate".to_string(),
5511 body_contract(&["PASS", "BLOCKED"]),
5512 )]));
5513
5514 let err = engine
5515 .submit_output(
5516 &token,
5517 &task_id,
5518 1,
5519 final_event(serde_json::json!("UNKNOWN"), true),
5520 )
5521 .await
5522 .expect_err("out-of-contract body value must be rejected");
5523 match err {
5524 EngineError::VerdictValueRejected { value, allowed } => {
5525 assert_eq!(value, "UNKNOWN");
5526 assert_eq!(allowed, vec!["PASS".to_string(), "BLOCKED".to_string()]);
5527 }
5528 other => panic!("unexpected error variant: {other:?}"),
5529 }
5530
5531 let tail = engine.output_tail(&task_id, 1).await;
5532 assert!(
5533 !tail
5534 .iter()
5535 .any(|ev| matches!(ev, crate::worker::output::OutputEvent::Final { .. })),
5536 "a rejected completion must not write a Final onto output_tail"
5537 );
5538 }
5539
5540 /// `ok=false` bypasses the completion-time check entirely, regardless
5541 /// of channel or membership — the exemption acceptance criterion,
5542 /// exercised at the `submit_output` choke point.
5543 #[tokio::test]
5544 async fn submit_output_ok_false_bypasses_the_check() {
5545 let (engine, token, task_id) = seeded_task_with_worker_token("gate").await;
5546 engine.register_verdict_contracts(HashMap::from([(
5547 "gate".to_string(),
5548 body_contract(&["PASS", "BLOCKED"]),
5549 )]));
5550
5551 engine
5552 .submit_output(
5553 &token,
5554 &task_id,
5555 1,
5556 final_event(serde_json::json!("UNKNOWN"), false),
5557 )
5558 .await
5559 .expect("ok=false must bypass the verdict contract check entirely");
5560
5561 let tail = engine.output_tail(&task_id, 1).await;
5562 assert!(
5563 tail.iter()
5564 .any(|ev| matches!(ev, crate::worker::output::OutputEvent::Final { .. })),
5565 "an ok=false completion is exempt, not rejected — its Final must still land"
5566 );
5567 }
5568
5569 /// `staged_verdict_value_for` mirrors `fold_final_and_parts`'s
5570 /// last-write-wins semantics: staging `"verdict"` twice within the
5571 /// same attempt returns the LAST value, not the first.
5572 #[tokio::test]
5573 async fn staged_verdict_value_for_is_last_write_wins() {
5574 let (engine, _token, task_id) = seeded_task_with_worker_token("gate").await;
5575 engine
5576 .stage_worker_artifact_trusted(
5577 &task_id,
5578 1,
5579 "verdict".to_string(),
5580 serde_json::json!("PASS"),
5581 )
5582 .await
5583 .expect("stage first verdict part");
5584 engine
5585 .stage_worker_artifact_trusted(
5586 &task_id,
5587 1,
5588 "verdict".to_string(),
5589 serde_json::json!("BLOCKED"),
5590 )
5591 .await
5592 .expect("stage second verdict part");
5593
5594 assert_eq!(
5595 engine.staged_verdict_value_for(&task_id, 1).await,
5596 Some("BLOCKED".to_string())
5597 );
5598 }
5599
5600 /// `staged_verdict_value_for` ignores artifacts staged under any name
5601 /// OTHER than the literal `"verdict"` — mirrors `channel: "part"`
5602 /// contracts only ever addressing that one part.
5603 #[tokio::test]
5604 async fn staged_verdict_value_for_ignores_other_artifact_names() {
5605 let (engine, _token, task_id) = seeded_task_with_worker_token("gate").await;
5606 engine
5607 .stage_worker_artifact_trusted(
5608 &task_id,
5609 1,
5610 "notes".to_string(),
5611 serde_json::json!("irrelevant"),
5612 )
5613 .await
5614 .expect("stage unrelated part");
5615
5616 assert_eq!(engine.staged_verdict_value_for(&task_id, 1).await, None);
5617 }
5618
5619 /// `staged_verdict_value_for` → `None` when nothing was ever staged —
5620 /// the normal case the completion check turns into
5621 /// `EngineError::VerdictPartMissing`.
5622 #[tokio::test]
5623 async fn staged_verdict_value_for_returns_none_when_nothing_staged() {
5624 let (engine, _token, task_id) = seeded_task_with_worker_token("gate").await;
5625 assert_eq!(engine.staged_verdict_value_for(&task_id, 1).await, None);
5626 }
5627}
5628
5629// ─── GH #76 Skip tier: DispatchOutcome::Skip tier + SubmitOutcome API ────────────
5630#[cfg(test)]
5631mod skip_tier_tests {
5632 use super::*;
5633 use crate::blueprint::compiler::{RustFnInProcessSpawnerFactory, SpawnerFactory};
5634 use crate::blueprint::EngineDispatcher;
5635 use crate::core::state::{
5636 is_skip_marker, unwrap_skip_marker, wrap_skip_marker, SubmitOutcome, SKIP_MARKER_KEY,
5637 };
5638 use crate::store::run::{InMemoryRunStore, RunContext, RunRecord, RunStatus, RunStore};
5639 use crate::types::{RunId, TaskId};
5640 use crate::worker::adapter::WorkerResult;
5641 use mlua_flow_ir::AsyncDispatcher;
5642 use mlua_swarm_schema::{AgentDef, AgentKind};
5643 use serde_json::json;
5644
5645 /// `DispatchOutcome::Skip(v)` roundtrips through serde JSON without
5646 /// loss — the enum is serialized with the default externally-tagged
5647 /// form (same as `Pass`/`Blocked`), so no `#[serde(...)]` tuning is
5648 /// needed for the new variant.
5649 #[test]
5650 fn dispatch_outcome_skip_variant_serializes_roundtrip() {
5651 let outcome = DispatchOutcome::Skip(json!({ "verdict": "SKIP", "reason": "n/a" }));
5652 let serialized = serde_json::to_string(&outcome).expect("serialize");
5653 let round: DispatchOutcome = serde_json::from_str(&serialized).expect("deserialize");
5654 match round {
5655 DispatchOutcome::Skip(v) => {
5656 assert_eq!(v, json!({ "verdict": "SKIP", "reason": "n/a" }));
5657 }
5658 other => panic!("expected Skip after roundtrip, got {other:?}"),
5659 }
5660 }
5661
5662 /// The `is_skip_marker` / `unwrap_skip_marker` / `wrap_skip_marker`
5663 /// helper triangle round-trips consistently and rejects plain
5664 /// payloads. Pinning the reserved-key contract in a unit test guards
5665 /// against a future edit accidentally renaming the sentinel key
5666 /// (which would silently break every downstream reader).
5667 #[test]
5668 fn skip_marker_helpers_wrap_detect_and_unwrap() {
5669 assert!(!is_skip_marker(&json!("plain string")));
5670 assert!(!is_skip_marker(&json!({ "verdict": "PASS" })));
5671 assert!(!is_skip_marker(&json!(null)));
5672
5673 let inner = json!({ "reason": "not applicable" });
5674 let wrapped = wrap_skip_marker(inner.clone());
5675 assert!(is_skip_marker(&wrapped));
5676 assert_eq!(wrapped[SKIP_MARKER_KEY], json!(true));
5677 assert_eq!(unwrap_skip_marker(&wrapped), Some(inner));
5678
5679 // A malformed sentinel (marker key present but `value` absent) is
5680 // still a Skip signal, defaulting the carried payload to Null so
5681 // downstream match arms never observe `None` on a marker match.
5682 let malformed = json!({ SKIP_MARKER_KEY: true });
5683 assert!(is_skip_marker(&malformed));
5684 assert_eq!(unwrap_skip_marker(&malformed), Some(Value::Null));
5685
5686 // Plain payloads → `unwrap_skip_marker` returns `None` (the
5687 // caller falls back to the ordinary Pass/Blocked path).
5688 assert_eq!(unwrap_skip_marker(&json!("plain")), None);
5689 }
5690
5691 /// The `SubmitOutcome::Skip` mapping wraps the payload in the
5692 /// skip-marker sentinel AND records `Final.ok = true` — matching the
5693 /// invariant in the outcome mapping table in
5694 /// `submit_worker_result_trusted`'s doc. This is the wire shape
5695 /// `dispatch_attempt_with*` reads back to route into
5696 /// `DispatchOutcome::Skip`.
5697 #[tokio::test]
5698 async fn submit_worker_result_trusted_skip_outcome_records_final_ok_true_with_sentinel() {
5699 use crate::worker::output::OutputEvent;
5700 let engine = Engine::new(EngineCfg::default());
5701 let op_token = engine
5702 .attach("ut-op", Role::Operator, Duration::from_secs(30))
5703 .await
5704 .expect("attach");
5705 let task_id = engine
5706 .start_task(
5707 &op_token,
5708 TaskSpec {
5709 agent: "analyst".into(),
5710 initial_directive: json!("go"),
5711 step_ctx: None,
5712 check_policy: None,
5713 },
5714 )
5715 .await
5716 .expect("start_task");
5717
5718 let inner_verdict = json!({ "verdict": "SKIP", "reason": "migration=no" });
5719 engine
5720 .submit_worker_result_trusted(&task_id, 1, inner_verdict.clone(), SubmitOutcome::Skip)
5721 .await
5722 .expect("submit with Skip outcome");
5723
5724 let tail = engine.output_tail(&task_id, 1).await;
5725 let final_ev = tail
5726 .iter()
5727 .rev()
5728 .find_map(|ev| match ev {
5729 OutputEvent::Final { content, ok } => Some((content.clone(), *ok)),
5730 _ => None,
5731 })
5732 .expect("Final present after Skip submit");
5733 assert!(
5734 final_ev.1,
5735 "Skip records Final.ok = true (flow-continuation)"
5736 );
5737 let stored_value = super::content_ref_to_value(final_ev.0);
5738 assert!(
5739 is_skip_marker(&stored_value),
5740 "Skip wraps the payload in the sentinel: got {stored_value}"
5741 );
5742 assert_eq!(unwrap_skip_marker(&stored_value), Some(inner_verdict));
5743 }
5744
5745 /// The new `SubmitOutcome::Pass` / `SubmitOutcome::Blocked` arms
5746 /// preserve byte-for-byte the pre-#76 wire shape (Final.ok mirrors
5747 /// the tier; the value is not wrapped). Regression against a future
5748 /// edit that accidentally routes Pass/Blocked through the Skip
5749 /// wrapper.
5750 #[tokio::test]
5751 async fn submit_worker_result_trusted_pass_and_blocked_wire_unchanged() {
5752 use crate::worker::output::OutputEvent;
5753 let engine = Engine::new(EngineCfg::default());
5754 let op_token = engine
5755 .attach("ut-op", Role::Operator, Duration::from_secs(30))
5756 .await
5757 .expect("attach");
5758
5759 // Pass path.
5760 let pass_task = engine
5761 .start_task(
5762 &op_token,
5763 TaskSpec {
5764 agent: "worker".into(),
5765 initial_directive: json!("go"),
5766 step_ctx: None,
5767 check_policy: None,
5768 },
5769 )
5770 .await
5771 .expect("start_task pass");
5772 engine
5773 .submit_worker_result_trusted(&pass_task, 1, json!("pass-value"), SubmitOutcome::Pass)
5774 .await
5775 .expect("submit Pass");
5776 let pass_tail = engine.output_tail(&pass_task, 1).await;
5777 let (pass_content, pass_ok) = pass_tail
5778 .iter()
5779 .rev()
5780 .find_map(|ev| match ev {
5781 OutputEvent::Final { content, ok } => Some((content.clone(), *ok)),
5782 _ => None,
5783 })
5784 .expect("Final present");
5785 assert!(pass_ok);
5786 assert_eq!(
5787 super::content_ref_to_value(pass_content),
5788 json!("pass-value"),
5789 "Pass value must not be wrapped"
5790 );
5791
5792 // Blocked path.
5793 let blocked_task = engine
5794 .start_task(
5795 &op_token,
5796 TaskSpec {
5797 agent: "worker".into(),
5798 initial_directive: json!("go"),
5799 step_ctx: None,
5800 check_policy: None,
5801 },
5802 )
5803 .await
5804 .expect("start_task blocked");
5805 engine
5806 .submit_worker_result_trusted(
5807 &blocked_task,
5808 1,
5809 json!("blocked-value"),
5810 SubmitOutcome::Blocked,
5811 )
5812 .await
5813 .expect("submit Blocked");
5814 let blocked_tail = engine.output_tail(&blocked_task, 1).await;
5815 let (blocked_content, blocked_ok) = blocked_tail
5816 .iter()
5817 .rev()
5818 .find_map(|ev| match ev {
5819 OutputEvent::Final { content, ok } => Some((content.clone(), *ok)),
5820 _ => None,
5821 })
5822 .expect("Final present");
5823 assert!(!blocked_ok);
5824 assert_eq!(
5825 super::content_ref_to_value(blocked_content),
5826 json!("blocked-value"),
5827 "Blocked value must not be wrapped"
5828 );
5829 }
5830
5831 /// End-to-end (engine layer): a worker that returns a skip-marker
5832 /// sentinel value via `WorkerResult { value: wrap_skip_marker(inner),
5833 /// ok: true }` — which is what a Skip-aware caller of
5834 /// `submit_worker_result_trusted(..., SubmitOutcome::Skip)` places on
5835 /// the wire — is folded by `dispatch_attempt_with_run_ctx` into
5836 /// `DispatchOutcome::Skip(inner)`. Proves the sentinel → outcome
5837 /// routing that the flow-ir binding boundary depends on.
5838 #[tokio::test]
5839 async fn dispatcher_folds_skip_sentinel_into_skip_outcome() {
5840 let inner_verdict = json!({ "verdict": "SKIP", "reason": "not applicable" });
5841 let inner_for_worker = inner_verdict.clone();
5842 let factory = RustFnInProcessSpawnerFactory::new().register_fn("analyst", move |_inv| {
5843 let value = wrap_skip_marker(inner_for_worker.clone());
5844 async move {
5845 Ok(WorkerResult {
5846 value,
5847 ok: true,
5848 stats: None,
5849 })
5850 }
5851 });
5852 let def = AgentDef {
5853 name: "analyst".into(),
5854 kind: AgentKind::RustFn,
5855 spec: json!({ "fn_id": "analyst" }),
5856 profile: None,
5857 meta: None,
5858 runner: None,
5859 runner_ref: None,
5860 verdict: None,
5861 };
5862 let spawner = factory.build(&def, None).expect("build");
5863
5864 let engine = Engine::new(EngineCfg::default());
5865 let op_token = engine
5866 .attach("ut-op", Role::Operator, Duration::from_secs(30))
5867 .await
5868 .expect("attach");
5869 let task_id = engine
5870 .start_task(
5871 &op_token,
5872 TaskSpec {
5873 agent: "analyst".into(),
5874 initial_directive: json!("go"),
5875 step_ctx: None,
5876 check_policy: None,
5877 },
5878 )
5879 .await
5880 .expect("start_task");
5881
5882 let outcome = engine
5883 .dispatch_attempt_with_run_ctx(&op_token, &task_id, &spawner, None)
5884 .await
5885 .expect("dispatch ok");
5886
5887 match outcome {
5888 DispatchOutcome::Skip(v) => {
5889 assert_eq!(v, inner_verdict, "Skip carries the unwrapped inner verdict");
5890 }
5891 other => panic!("expected DispatchOutcome::Skip, got {other:?}"),
5892 }
5893 }
5894
5895 /// `EngineDispatcher::dispatch` (the `AsyncDispatcher` impl flow-ir
5896 /// invokes) maps `DispatchOutcome::Skip(v)` to `Ok(wrap_skip_marker(v))`
5897 /// — a successful return whose Value carries the sentinel across the
5898 /// flow-ir boundary. Pinning this mapping in a test guards the arm
5899 /// order (a wildcard `Ok(other) =>` arm accidentally placed BEFORE the
5900 /// Skip arm would route Skip to `EvalError::DispatcherError` and
5901 /// abort the flow — the exact failure mode this tier prevents).
5902 #[tokio::test]
5903 async fn engine_dispatcher_maps_skip_outcome_to_ok_sentinel_value() {
5904 let inner_verdict = json!({ "verdict": "SKIP", "reason": "not applicable" });
5905 let inner_for_worker = inner_verdict.clone();
5906 let factory = RustFnInProcessSpawnerFactory::new().register_fn("analyst", move |_inv| {
5907 let value = wrap_skip_marker(inner_for_worker.clone());
5908 async move {
5909 Ok(WorkerResult {
5910 value,
5911 ok: true,
5912 stats: None,
5913 })
5914 }
5915 });
5916 let def = AgentDef {
5917 name: "analyst".into(),
5918 kind: AgentKind::RustFn,
5919 spec: json!({ "fn_id": "analyst" }),
5920 profile: None,
5921 meta: None,
5922 runner: None,
5923 runner_ref: None,
5924 verdict: None,
5925 };
5926 let spawner = factory.build(&def, None).expect("build");
5927
5928 let engine = Engine::new(EngineCfg::default());
5929 let op_token = engine
5930 .attach("ut-op", Role::Operator, Duration::from_secs(30))
5931 .await
5932 .expect("attach");
5933 let dispatcher = EngineDispatcher::with_spawner(engine.clone(), op_token, spawner);
5934
5935 let out = dispatcher
5936 .dispatch("analyst", json!("go"))
5937 .await
5938 .expect("dispatch returns Ok for Skip tier (not EvalError::DispatcherError)");
5939
5940 assert!(
5941 is_skip_marker(&out),
5942 "returned value must carry the skip-marker sentinel across the flow-ir boundary: got {out}"
5943 );
5944 assert_eq!(unwrap_skip_marker(&out), Some(inner_verdict));
5945 }
5946
5947 /// `EngineDispatcher::dispatch`'s `RunContext` step-entry log records
5948 /// `status = "skipped"` for a Skip completion (distinct from
5949 /// `"passed"` / `"blocked"`), so post-run inspection of
5950 /// `RunRecord.step_entries` can distinguish flow-continuation-with-
5951 /// binding-write from flow-continuation-without-binding-write.
5952 #[tokio::test]
5953 async fn engine_dispatcher_step_entry_status_is_skipped_for_skip_outcome() {
5954 let inner_verdict = json!({ "verdict": "SKIP" });
5955 let inner_for_worker = inner_verdict.clone();
5956 let factory = RustFnInProcessSpawnerFactory::new().register_fn("analyst", move |_inv| {
5957 let value = wrap_skip_marker(inner_for_worker.clone());
5958 async move {
5959 Ok(WorkerResult {
5960 value,
5961 ok: true,
5962 stats: None,
5963 })
5964 }
5965 });
5966 let def = AgentDef {
5967 name: "analyst".into(),
5968 kind: AgentKind::RustFn,
5969 spec: json!({ "fn_id": "analyst" }),
5970 profile: None,
5971 meta: None,
5972 runner: None,
5973 runner_ref: None,
5974 verdict: None,
5975 };
5976 let spawner = factory.build(&def, None).expect("build");
5977
5978 let engine = Engine::new(EngineCfg::default());
5979 let op_token = engine
5980 .attach("ut-op", Role::Operator, Duration::from_secs(30))
5981 .await
5982 .expect("attach");
5983
5984 // Seed a RunContext with an InMemoryRunStore so the dispatcher
5985 // appends a step_entry we can then read back.
5986 let run_id = RunId::new();
5987 let run_store: Arc<dyn RunStore> = Arc::new(InMemoryRunStore::new());
5988 run_store
5989 .create(RunRecord {
5990 id: run_id.clone(),
5991 task_id: TaskId::new(),
5992 status: RunStatus::Running,
5993 step_entries: Vec::new(),
5994 degradations: Vec::new(),
5995 operator_sid: None,
5996 result_ref: None,
5997 input_json: None,
5998 created_at: 0,
5999 updated_at: 0,
6000 })
6001 .await
6002 .expect("create run record");
6003 let run_ctx = RunContext::new(run_id.clone(), run_store.clone());
6004
6005 let dispatcher =
6006 EngineDispatcher::with_spawner(engine.clone(), op_token, spawner).with_run(run_ctx);
6007
6008 let out = dispatcher
6009 .dispatch("analyst", json!("go"))
6010 .await
6011 .expect("dispatch ok");
6012 assert!(is_skip_marker(&out));
6013
6014 let record = run_store.get(&run_id).await.expect("run record present");
6015 let step = record
6016 .step_entries
6017 .first()
6018 .expect("at least one step_entry appended for the dispatched step");
6019 assert_eq!(
6020 step.status.as_deref(),
6021 Some("skipped"),
6022 "Skip outcome must record StepEntry.status = \"skipped\""
6023 );
6024 assert_eq!(step.step_ref.as_deref(), Some("analyst"));
6025 }
6026}