mlua_swarm/core/engine.rs
1//! `Engine` — the long-running stateful runtime plus the `with_state`
2//! helper (R1-R4 discipline).
3//!
4//! The engine owns the Domain side of the Data / Domain split:
5//! flow control (dispatch / verdict), state (`EngineState`), and the
6//! `submit_output` / `output_tail` surface that feeds it. Data-plane
7//! traffic (Big Response bodies) is delegated to the `output_store` module
8//! plus its paired `SpawnerLayer`s and passes through here without the
9//! engine core needing to grow.
10
11use crate::core::agent_context::{RUN_ID_KEY, STEP_CTX_KEY};
12use crate::core::config::EngineCfg;
13use crate::core::ctx::{Ctx, OperatorInfo, OperatorKind, SeniorBridge, SpawnHook};
14use crate::core::errors::EngineError;
15use crate::core::state::{
16 CapTokenRecord, DispatchOutcome, EngineState, Event, EventStream, OperatorSession, ResumeKey,
17 ResumePending, TaskSpec, TaskState, TaskStatus,
18};
19use crate::store::replay::{hash_input_value, ReplayEntry};
20use crate::store::run::RunContext;
21use crate::types::{
22 default_role_verb_table, now_unix, CapToken, Role, RoleVerbGate, RunId, SessionId, StepId,
23 TokenSigner, Verb,
24};
25use crate::worker::adapter::SpawnerAdapter;
26use serde_json::Value;
27use std::collections::HashMap;
28use std::sync::Arc;
29use std::time::{Duration, Instant};
30use tokio::sync::{broadcast, Mutex};
31
32/// Process-wide long-running runtime. Cheap to `clone()` — an `Arc`
33/// lives inside.
34#[derive(Clone)]
35pub struct Engine {
36 inner: Arc<EngineInner>,
37}
38
39struct EngineInner {
40 state: Mutex<EngineState>,
41 cfg: EngineCfg,
42 signer: TokenSigner,
43 gate: RoleVerbGate,
44 event_tx: broadcast::Sender<Event>,
45 /// ID-keyed bridge registry (register-by-ID design). `SeniorBridge`
46 /// and `SpawnHook` are registered by ID; sessions bind to those IDs
47 /// only. Persistence stores just the ID, and on reattach the caller
48 /// re-registers under the same ID to restore presence.
49 senior_bridges: tokio::sync::RwLock<HashMap<String, Arc<dyn SeniorBridge>>>,
50 spawn_hooks: tokio::sync::RwLock<HashMap<String, Arc<dyn SpawnHook>>>,
51 /// ID registry for full-spawn Operator backends (backends that take the
52 /// entire spawn via `execute`). Sibling to `senior_bridges` /
53 /// `spawn_hooks`. `OperatorDelegateMiddleware` looks these up via
54 /// `ctx` and, when `kind = MainAi` / `Composite`, bypasses
55 /// `inner.spawn` and calls `operator.execute` instead.
56 operators: tokio::sync::RwLock<HashMap<String, Arc<dyn crate::operator::Operator>>>,
57 /// Base and hint layer factories for the `SpawnerStack`. At
58 /// `service::linker::link` time, `compiled.router` is wrapped with
59 /// the base factories plus the hint factories resolved from
60 /// `blueprint.spawner_hints.layers`. This is the engine-side
61 /// counterpart to the discipline "Flow / Blueprint doesn't spell out
62 /// middleware implementations — it declares the capabilities it needs
63 /// as hint keys".
64 layer_registry: crate::middleware::LayerRegistry,
65 /// Optional Data-plane `OutputStore` backend (subtask-4 / ST2 rework —
66 /// see `submit_output`'s doc). `None` (the default) preserves
67 /// pre-subtask-4 behavior exactly: `submit_output` /
68 /// `submit_worker_result_trusted` only touch the Domain-plane
69 /// `EngineState.output_store` HashMap, same as before this was added.
70 /// `Some` additionally dual-writes every `Final` event into this store
71 /// via [`crate::store::output::OutputStore::append`], making it
72 /// queryable (e.g. by `mlua-swarm-server`'s `GET /v1/tasks/:id/ctx`)
73 /// even for an in-flight run. A plain `std::sync::RwLock` (not
74 /// `tokio::sync::RwLock`) — set once at boot via [`Engine::set_output_store`]
75 /// from a synchronous call site (`mlua-swarm-server`'s router builder),
76 /// then only ever briefly read (clone the `Option<Arc<..>>`, never held
77 /// across an `.await`) from the async submit path.
78 data_store: std::sync::RwLock<Option<Arc<dyn crate::store::output::OutputStore>>>,
79 /// GH #50 (Subtask 2 — runtime plumbing): agent name → declared
80 /// [`mlua_swarm_schema::VerdictContract`], the Engine-side registry
81 /// [`Self::verdict_contract_for_task`] resolves against. Populated via
82 /// [`Self::register_verdict_contracts`] — same sync-`RwLock`,
83 /// set-outside-the-lock idiom as `data_store` above. Empty by default
84 /// (every pre-GH-#50 `Engine`), which is exactly the opt-in "no
85 /// contract declared" state `verdict_contract_for_task` treats as
86 /// `None`. Populated from a live `Compiler::compile`'s
87 /// `CompiledAgentTable.verdict_contracts` output by
88 /// `TaskLaunchService::launch`, immediately after `compiler.compile`
89 /// succeeds — see [`Self::register_verdict_contracts`]'s doc for the
90 /// overwrite semantics of that merge.
91 verdict_contracts: std::sync::RwLock<HashMap<String, mlua_swarm_schema::VerdictContract>>,
92}
93
94/// Renders a `TaskSpec.initial_directive` / `EngineState.prompts`
95/// `Value` down to the `String` shape that string-consuming boundaries
96/// require (issue #18). Strings pass through verbatim; anything else
97/// (Object / Array / Number / Bool / Null) is serde-stringified. This
98/// is the single canonical rendering — the coercion that used to sit
99/// inside `EngineDispatcher::dispatch` moved here and is invoked only
100/// at consumer boundaries: `WorkerPayload.prompt` (HTTP
101/// `/v1/worker/prompt`), `WorkerInvocation.prompt` (in-process
102/// spawners), the subprocess spawner's directive arg/stdin, and the
103/// WS Spawn frame text render (`operator_ws::session`). Everything
104/// upstream (Blueprint dispatch → engine state → `fetch_prompt` →
105/// `Operator::execute`) keeps the `Value` end-to-end.
106pub(crate) fn render_directive_to_string(v: &Value) -> String {
107 match v {
108 Value::String(s) => s.clone(),
109 other => other.to_string(),
110 }
111}
112
113/// Renders a [`crate::worker::output::ContentRef`] down to the `Value` shape
114/// the BP-chain / `DispatchOutcome` consume. `Inline` passes its `value`
115/// through verbatim; `FileRef` is stringified into the same
116/// `{"file_ref", "mime", "size_hint"}` shape `materialize_final_submission`
117/// uses for its own file-materialize projection — one canonical
118/// stringification, not two independently-maintained copies (GH #36 ST1:
119/// shared by both the `Final`-pull and the `Artifact`-parts fold in
120/// [`Engine::dispatch_attempt_with`]'s doc).
121fn content_ref_to_value(content: crate::worker::output::ContentRef) -> Value {
122 match content {
123 crate::worker::output::ContentRef::Inline { value } => value,
124 crate::worker::output::ContentRef::FileRef {
125 path,
126 mime,
127 size_hint,
128 } => serde_json::json!({
129 "file_ref": path.to_string_lossy(),
130 "mime": mime,
131 "size_hint": size_hint,
132 }),
133 }
134}
135
136/// GH #51 — reduces a [`content_ref_to_value`] result down to the `String`
137/// shape the completion-time verdict-contract check compares against a
138/// declared `VerdictContract.values` token set. A `Value::String` unwraps
139/// to its raw contents (no surrounding JSON quotes) — this mirrors the
140/// pre-GH-#51 `check_verdict_contract` (`mlua-swarm-server`'s
141/// `worker.rs`), which always compared the raw submitted body string
142/// directly, never a JSON-stringified copy. Any OTHER `Value` shape
143/// (`Number` / `Object` / `Array` / `Bool` / `Null` — i.e. a `channel:
144/// "body"` contract whose completing value is not a string at all, or a
145/// `FileRef` content whose `content_ref_to_value` projection is an
146/// object) falls back to `Value::to_string()`'s JSON-encoded form: it can
147/// never collide with a plain declared token like `"PASS"`, so it
148/// naturally fails membership — consistent with the "non-string values
149/// under a body contract are violations" rule (issue #51's Proposal).
150fn content_ref_to_comparable_string(content: crate::worker::output::ContentRef) -> String {
151 let value = content_ref_to_value(content);
152 match value {
153 Value::String(s) => s,
154 other => other.to_string(),
155 }
156}
157
158/// [`Engine::dispatch_attempt_with`]'s Final-pull assembly (GH #36 ST1:
159/// named multi-part worker output), factored out as a pure function of the
160/// output-event tail so it is unit-testable without a live `Engine` /
161/// spawner.
162///
163/// Finds the LAST `Final` event in `tail` (mirrors the pre-GH-#36 pull:
164/// "last Final wins" if more than one was ever appended) and folds every
165/// `Artifact` event in the SAME tail WHOSE NAME APPEARS IN `staged_names`
166/// into a `"parts"` object keyed by `Artifact.name` — walked in tail (=
167/// event-append) order, so a name staged more than once within the attempt
168/// is last-write-wins (`Map` insert semantics, not an accumulating list;
169/// `Engine::stage_worker_artifact_trusted`'s doc). `staged_names` is the
170/// WORKER's own opt-in allowlist (`EngineState.worker_artifact_names`'s
171/// doc) — an `Artifact` on the tail whose name is NOT in `staged_names`
172/// (e.g. `AfterRunAuditMiddleware`'s `"audit:<step_ref>"` sidecar finding)
173/// is left alone, exactly as before GH #36; this is what keeps an audited
174/// step's BP-chain value byte-identical when the worker itself never
175/// staged a part.
176///
177/// At least one matching part: the returned value is `{"out": <final
178/// value>, "parts": {<name>: <value>, ...}}`. Zero matching parts: the
179/// returned value is the plain final value, unchanged from the pre-GH-#36
180/// shape — this is the back-compat guarantee, not an incidental default.
181///
182/// `None` when `tail` carries no `Final` at all (the caller's pre-existing
183/// "no Final in output_tail" error path).
184fn fold_final_and_parts(
185 tail: &[crate::worker::output::OutputEvent],
186 staged_names: &[String],
187) -> Option<(Value, bool)> {
188 let (final_content, ok) = tail.iter().rev().find_map(|ev| match ev {
189 crate::worker::output::OutputEvent::Final { content, ok } => Some((content.clone(), *ok)),
190 _ => None,
191 })?;
192 let final_value = content_ref_to_value(final_content);
193
194 let mut parts = serde_json::Map::new();
195 for ev in tail {
196 if let crate::worker::output::OutputEvent::Artifact { name, content } = ev {
197 if staged_names.iter().any(|staged| staged == name) {
198 parts.insert(name.clone(), content_ref_to_value(content.clone()));
199 }
200 }
201 }
202
203 let value = if parts.is_empty() {
204 final_value
205 } else {
206 serde_json::json!({ "out": final_value, "parts": Value::Object(parts) })
207 };
208 Some((value, ok))
209}
210
211impl Engine {
212 /// Backwards-compatible constructor that starts the engine without a
213 /// layer registry, preserving the signature already used by ~88
214 /// existing call sites. Use this when automatic middleware wrapping
215 /// at bind time is not needed. Callers such as `mlua-swarm-server` go through
216 /// `new_with_layers(cfg, registry)` to enable the hint-resolution path.
217 pub fn new(cfg: EngineCfg) -> Self {
218 Self::new_with_layers(cfg, crate::middleware::LayerRegistry::new())
219 }
220
221 /// Construct an `Engine` with an explicit `LayerRegistry`, enabling
222 /// hint-resolution: `spawner_hints.layers` declared on a `Blueprint`
223 /// are resolved against this registry when the spawner stack is bound
224 /// at `service::linker::link` time.
225 pub fn new_with_layers(
226 cfg: EngineCfg,
227 layer_registry: crate::middleware::LayerRegistry,
228 ) -> Self {
229 let (event_tx, _) = broadcast::channel(256);
230 let signer = TokenSigner::new(&cfg.token_secret);
231 Self {
232 inner: Arc::new(EngineInner {
233 state: Mutex::new(EngineState::new()),
234 cfg,
235 signer,
236 gate: default_role_verb_table(),
237 event_tx,
238 senior_bridges: tokio::sync::RwLock::new(HashMap::new()),
239 spawn_hooks: tokio::sync::RwLock::new(HashMap::new()),
240 operators: tokio::sync::RwLock::new(HashMap::new()),
241 layer_registry,
242 data_store: std::sync::RwLock::new(None),
243 verdict_contracts: std::sync::RwLock::new(HashMap::new()),
244 }),
245 }
246 }
247
248 /// Rebuild this `Engine` with a different `RoleVerbGate`. The gate is
249 /// treated as fixed-at-build-time, so this constructs a fresh
250 /// `EngineInner` (fresh empty `EngineState`) rather than mutating in
251 /// place — mainly a testing convenience for swapping gate rules.
252 pub fn with_gate(self, gate: RoleVerbGate) -> Self {
253 // The gate is fixed at build time — the intent is to build a fresh
254 // instance rather than mutating in place. As a testing convenience we
255 // do allow swapping the inner Arc. Simpler form: just rebuild
256 // Arc<EngineInner>.
257 let inner = Arc::new(EngineInner {
258 state: Mutex::new(EngineState::new()),
259 cfg: self.inner.cfg.clone(),
260 signer: self.inner.signer.clone(),
261 gate,
262 event_tx: self.inner.event_tx.clone(),
263 senior_bridges: tokio::sync::RwLock::new(HashMap::new()),
264 spawn_hooks: tokio::sync::RwLock::new(HashMap::new()),
265 operators: tokio::sync::RwLock::new(HashMap::new()),
266 layer_registry: self.inner.layer_registry.clone(),
267 data_store: std::sync::RwLock::new(None),
268 verdict_contracts: std::sync::RwLock::new(HashMap::new()),
269 });
270 Self { inner }
271 }
272
273 // ═══════════════════════════════════════════════════════════════════════
274 // Accessors. Production code drives execution through compile +
275 // `service::linker::link` + `dispatch_attempt_with(spawner)` inside
276 // `TaskLaunchService`; `Engine` itself is a pure execution surface — it
277 // does not own a BlueprintStore / EnhanceAdapter / Compiler, nor a
278 // global spawner (the spawner is carried per-request, never stashed on
279 // the engine).
280 // ═══════════════════════════════════════════════════════════════════════
281
282 /// Access the `EngineCfg` this engine was built with.
283 pub fn cfg(&self) -> &EngineCfg {
284 &self.inner.cfg
285 }
286
287 /// Expose the internal `LayerRegistry` — used when deriving a
288 /// sub-engine that needs the same registry re-injected. The
289 /// per-request sub-engine in `mlua-swarm-server` reads the parent engine's
290 /// registry through this accessor and passes it to
291 /// `Engine::new_with_layers(cfg, parent.layer_registry().clone())`.
292 pub fn layer_registry(&self) -> &crate::middleware::LayerRegistry {
293 &self.inner.layer_registry
294 }
295
296 /// Access the `TokenSigner` used to mint/verify `CapToken`s.
297 pub fn signer(&self) -> &TokenSigner {
298 &self.inner.signer
299 }
300
301 /// Clone a handle to the process-wide `Event` broadcast sender. Prefer
302 /// `subscribe` for a ready-to-use receiver.
303 pub fn event_tx(&self) -> broadcast::Sender<Event> {
304 self.inner.event_tx.clone()
305 }
306
307 /// Subscribe to the engine's `Event` broadcast stream.
308 pub fn subscribe(&self) -> EventStream {
309 self.inner.event_tx.subscribe()
310 }
311
312 /// Wires the Data-plane [`crate::store::output::OutputStore`] backend
313 /// used by `submit_output` / `submit_worker_result_trusted`'s
314 /// submit-time projection sink (subtask-4 / ST2 rework — see
315 /// `submit_output`'s doc). Synchronous (a plain `std::sync::RwLock`
316 /// write) so a caller can wire it up at boot from a non-`async`
317 /// context (`mlua-swarm-server`'s router builder passes the same
318 /// `Arc` it hands to its `AppState.data_store`, so `POST
319 /// /v1/data/emit` and every worker's ordinary `/v1/worker/submit` land
320 /// in the one store). Calling this more than once replaces the
321 /// previous backend; not calling it at all (the default) preserves
322 /// pre-subtask-4 behavior exactly — `submit_output` only touches the
323 /// Domain-plane `EngineState.output_store` HashMap.
324 pub fn set_output_store(&self, store: Arc<dyn crate::store::output::OutputStore>) {
325 let mut guard = self
326 .inner
327 .data_store
328 .write()
329 .unwrap_or_else(|poisoned| poisoned.into_inner());
330 *guard = Some(store);
331 }
332
333 /// Clones the currently-wired Data-plane store handle, if any. Kept
334 /// private and side-effect-free (no lock held past this call) —
335 /// callers (`materialize_final_submission`) do their actual `.append`
336 /// work outside of any lock.
337 fn output_store_backend(&self) -> Option<Arc<dyn crate::store::output::OutputStore>> {
338 self.inner
339 .data_store
340 .read()
341 .unwrap_or_else(|poisoned| poisoned.into_inner())
342 .clone()
343 }
344
345 /// GH #50 (Subtask 2): merges `contracts` (agent name → declared
346 /// [`mlua_swarm_schema::VerdictContract`]) into the engine's runtime
347 /// verdict-contract registry, later resolved per-task by
348 /// [`Self::verdict_contract_for_task`]. Same sync-write idiom as
349 /// [`Self::set_output_store`] — a plain `std::sync::RwLock` write, so
350 /// this can be called from a non-`async` context. Production call
351 /// site: `TaskLaunchService::launch`, immediately after a successful
352 /// `Compiler::compile`, passing `compiled.router.verdict_contracts.clone()`.
353 ///
354 /// # Overwrite semantics (explicit — read before adding a second call site)
355 ///
356 /// The registry is a single flat `HashMap` **keyed by agent name only**
357 /// (`String`), with process-wide (not per-task, not per-Blueprint,
358 /// not per-launch) scope. Registration is additive via
359 /// `HashMap::extend`: an entry for an agent name NOT already present is
360 /// added; an entry for an agent name ALREADY present is REPLACED
361 /// (last write wins) by the incoming one. Concretely: launching a
362 /// second Blueprint that also declares a `verdict` contract for an
363 /// agent named `"gate"` OVERWRITES whatever contract a first, still
364 /// in-flight, launch registered for an agent of that same name — even
365 /// if the two Blueprints intend it as two semantically different
366 /// agents that merely share a name, and even while the first launch's
367 /// tasks are still running. This is a **known limitation** of the v1
368 /// design; a per-task (or per-`RunId` / per-Blueprint) scoped registry
369 /// is a possible follow-up if two concurrently in-flight Blueprints
370 /// declaring conflicting contracts under the same agent name turns out
371 /// to matter in practice. Calling this with an empty map (or not at
372 /// all — the default) is a no-op, preserving pre-GH-#50 behavior
373 /// exactly (opt-in).
374 pub fn register_verdict_contracts(
375 &self,
376 contracts: HashMap<String, mlua_swarm_schema::VerdictContract>,
377 ) {
378 let mut guard = self
379 .inner
380 .verdict_contracts
381 .write()
382 .unwrap_or_else(|poisoned| poisoned.into_inner());
383 guard.extend(contracts);
384 }
385
386 /// GH #50 (Subtask 2): the declared
387 /// [`mlua_swarm_schema::VerdictContract`] for the agent currently
388 /// running `task_id`, if any. Resolves `task_id` → `TaskState.spec.agent`
389 /// (via `EngineState.tasks`, the same lookup [`Self::task_attempt`]
390 /// performs) and looks that agent name up in the registry
391 /// [`Self::register_verdict_contracts`] populates.
392 ///
393 /// `None` in both of these cases — deliberately collapsed to the same
394 /// value, mirroring [`Self::agent_context_for`]'s `Result`-into-`Option`
395 /// pattern (`.ok().flatten()`; a lookup failure here is never itself an
396 /// error worth surfacing to a caller):
397 /// - `task_id` is unknown (no `TaskState` for it).
398 /// - `task_id` resolves to a known agent, but that agent declared no
399 /// `verdict` contract (the opt-in default).
400 ///
401 /// Callers (`mlua-swarm-server`'s `worker_submit` / `worker_artifact`)
402 /// treat every `None` identically: skip the submit-time verdict gate
403 /// entirely, preserving pre-GH-#50 behavior byte-for-byte.
404 pub async fn verdict_contract_for_task(
405 &self,
406 task_id: &StepId,
407 ) -> Option<mlua_swarm_schema::VerdictContract> {
408 let tid = task_id.clone();
409 let agent = self
410 .with_state("verdict_contract_for_task", move |s| {
411 s.tasks.get(&tid).map(|t| t.spec.agent.clone())
412 })
413 .await
414 .ok()
415 .flatten()?;
416 self.inner
417 .verdict_contracts
418 .read()
419 .unwrap_or_else(|poisoned| poisoned.into_inner())
420 .get(&agent)
421 .cloned()
422 }
423
424 /// GH #51 — the value of the LAST staged `"verdict"` `Artifact` for
425 /// `(task_id, attempt)`, if any. Mirrors [`fold_final_and_parts`]'s
426 /// reverse-scan-of-`output_tail` pattern (last-write-wins per name,
427 /// same as that fold and [`Self::stage_worker_artifact_trusted`]'s
428 /// doc), narrowed to the single literal artifact name
429 /// `channel: "part"` contracts address (Pattern B — see
430 /// `blueprint-authoring.md`'s "Returning verdicts to drive BP flow").
431 ///
432 /// Infallible accessor: `None` is the normal "nothing staged yet"
433 /// case, not an error — the caller
434 /// ([`Self::verdict_contract_completion_check`]) is what converts
435 /// `None` into `Err(EngineError::VerdictPartMissing)`.
436 pub(crate) async fn staged_verdict_value_for(
437 &self,
438 task_id: &StepId,
439 attempt: u32,
440 ) -> Option<String> {
441 let tail = self.output_tail(task_id, attempt).await;
442 tail.iter().rev().find_map(|ev| match ev {
443 crate::worker::output::OutputEvent::Artifact { name, content } if name == "verdict" => {
444 Some(content_ref_to_comparable_string(content.clone()))
445 }
446 _ => None,
447 })
448 }
449
450 /// GH #51 — the single completion-time verdict-contract choke point,
451 /// embedded inside BOTH [`Self::submit_worker_result_trusted`] and
452 /// [`Self::submit_output`] (the two engine-side writes every HTTP/WS
453 /// completion route ultimately passes through). Not duplicated per
454 /// route handler — a future 4th completion route is gated for free
455 /// as long as it funnels through one of those two functions.
456 ///
457 /// `ok=false` is exempt on every route (this single early-return IS
458 /// the exemption, reused identically by both embedding sites — see
459 /// issue #51's "ok=false completions are exempt" acceptance
460 /// criterion). An agent with no declared contract, or a contract for
461 /// the OTHER channel, is untouched (`Ok(())`) — same opt-in,
462 /// byte-for-byte-preserving posture as
463 /// [`Self::verdict_contract_for_task`]'s doc.
464 ///
465 /// - `channel: "body"` — `value` (the completing `Final`'s content,
466 /// already reduced to a comparable string by the caller via
467 /// [`content_ref_to_comparable_string`]) must be a member of
468 /// `contract.values`.
469 /// - `channel: "part"` — [`Self::staged_verdict_value_for`] must find
470 /// a staged `"verdict"` artifact for this attempt (presence,
471 /// defense in depth over the staging-time membership check) AND its
472 /// value must be a member of `contract.values`.
473 async fn verdict_contract_completion_check(
474 &self,
475 task_id: &StepId,
476 attempt: u32,
477 ok: bool,
478 value: &str,
479 ) -> Result<(), EngineError> {
480 if !ok {
481 return Ok(());
482 }
483 let Some(contract) = self.verdict_contract_for_task(task_id).await else {
484 return Ok(());
485 };
486 match contract.channel {
487 mlua_swarm_schema::VerdictChannel::Body => {
488 if contract.values.iter().any(|v| v == value) {
489 Ok(())
490 } else {
491 Err(EngineError::VerdictValueRejected {
492 value: value.to_string(),
493 allowed: contract.values.clone(),
494 })
495 }
496 }
497 mlua_swarm_schema::VerdictChannel::Part => {
498 match self.staged_verdict_value_for(task_id, attempt).await {
499 None => Err(EngineError::VerdictPartMissing {
500 allowed: contract.values.clone(),
501 }),
502 Some(staged) if contract.values.iter().any(|v| v == &staged) => Ok(()),
503 Some(staged) => Err(EngineError::VerdictValueRejected {
504 value: staged,
505 allowed: contract.values.clone(),
506 }),
507 }
508 }
509 }
510 }
511
512 // ═══════════════════════════════════════════════════════════════════════
513 // §7 with_state — single Mutex + R1-R4 (try_lock + bounded retry + max-hold panic)
514 // ═══════════════════════════════════════════════════════════════════════
515
516 /// The closure is a **sync** `FnOnce` — you cannot pass an async
517 /// closure, which enforces R3 at the type level. Exceeding `max_hold`
518 /// panics so that R4 violations surface immediately.
519 pub async fn with_state<F, R>(&self, op: &'static str, f: F) -> Result<R, EngineError>
520 where
521 F: FnOnce(&mut EngineState) -> R,
522 {
523 let cfg = &self.inner.cfg;
524
525 // R2: try_lock + bounded retry
526 let mut guard_opt = None;
527 for attempt in 0..=cfg.max_retry {
528 match self.inner.state.try_lock() {
529 Ok(g) => {
530 guard_opt = Some(g);
531 break;
532 }
533 Err(_) if cfg.try_only => return Err(EngineError::LockBusy(op)),
534 Err(_) => {
535 let backoff = cfg.backoff_ms_step * (attempt as u64 + 1);
536 tokio::time::sleep(Duration::from_millis(backoff)).await;
537 }
538 }
539 }
540 let mut guard = guard_opt.ok_or(EngineError::LockBusyAfterRetry(op))?;
541
542 // R4: max_hold guard
543 let start = Instant::now();
544 let result = f(&mut guard);
545 let elapsed_ms = start.elapsed().as_millis();
546 drop(guard);
547
548 if elapsed_ms > cfg.max_hold_ms {
549 panic!(
550 "Engine.with_state('{op}') held {elapsed_ms}ms > max {}ms — suspected R3 violation (long op inside lock)",
551 cfg.max_hold_ms
552 );
553 }
554 Ok(result)
555 }
556
557 // ═══════════════════════════════════════════════════════════════════════
558 // Token verify (= sig + expire + gate + uses_left)
559 // ═══════════════════════════════════════════════════════════════════════
560
561 /// Four steps: (1) signature verify, (2) expiry check, (3) role × verb
562 /// gate, (4) `uses_left` consume.
563 pub async fn verify_token(&self, token: &CapToken, verb: Verb) -> Result<(), EngineError> {
564 // (1) sig
565 if !self.inner.signer.verify_sig(token) {
566 return Err(EngineError::BadSignature);
567 }
568 // (2) expire
569 if token.is_expired(now_unix()) {
570 return Err(EngineError::TokenExpired);
571 }
572 // (3) role × verb gate
573 if !self.inner.gate.is_allowed(token.role, verb) {
574 return Err(EngineError::RoleViolation {
575 role: token.role,
576 verb,
577 });
578 }
579 // (4) server-side uses_left consume
580 let fp = token.fingerprint();
581 self.with_state("token.consume", move |s| {
582 let rec = s
583 .tokens
584 .get_mut(&fp)
585 .ok_or_else(|| EngineError::TokenNotFound(fp.clone()))?;
586 rec.consume()
587 .map_err(|_: crate::core::state::CapTokenConsumeError| {
588 EngineError::TokenUsesExhausted
589 })?;
590 Ok::<(), EngineError>(())
591 })
592 .await??;
593 Ok(())
594 }
595
596 /// `verify_token` plus the **task-ownership gate**.
597 ///
598 /// When a Worker-role token calls a state-touch verb (`fetch_prompt` /
599 /// `post_result` / `read_task_state` / `cancel_task` / `poll_task`),
600 /// the gate checks that `CapTokenRecord.task_id` matches the argument
601 /// `task_id`; a mismatch returns `EngineError::TokenTaskMismatch`.
602 /// Operator / Senior / Observer tokens are outside the ownership gate
603 /// and may touch any task.
604 ///
605 /// **Verbs exempt from the gate.** `start_task` and `dispatch_attempt`
606 /// stay outside so recursive swarming keeps working; depth is capped
607 /// by `max_spawn_depth`.
608 pub async fn verify_token_for_task(
609 &self,
610 token: &CapToken,
611 verb: Verb,
612 task_id: &StepId,
613 ) -> Result<(), EngineError> {
614 self.verify_token(token, verb).await?;
615 if token.role != Role::Worker {
616 return Ok(());
617 }
618 let fp = token.fingerprint();
619 let arg_tid = task_id.clone();
620 self.with_state("token.ownership_gate", move |s| {
621 let bound = s.tokens.get(&fp).and_then(|r| r.task_id.as_ref()).cloned();
622 match bound {
623 Some(t) if t == arg_tid => Ok(()),
624 Some(t) => Err(EngineError::TokenTaskMismatch {
625 bound: t.into_string(),
626 arg: arg_tid.into_string(),
627 }),
628 None => Err(EngineError::TokenNotFound(fp.clone())),
629 }
630 })
631 .await??;
632 Ok(())
633 }
634
635 /// Resolve the bound `task_id` from a Worker-role token. Used on the
636 /// simple `/v1/worker/submit` endpoint, where the worker POSTs with a
637 /// token but no `task_id`. Returns `Err` if the token role is not
638 /// Worker, or if no bound task is set.
639 pub async fn task_id_from_token(&self, token: &CapToken) -> Result<StepId, EngineError> {
640 if token.role != Role::Worker {
641 return Err(EngineError::RoleViolation {
642 role: token.role,
643 verb: Verb::PostResult,
644 });
645 }
646 let fp = token.fingerprint();
647 self.with_state("task_id_from_token", move |s| {
648 s.tokens
649 .get(&fp)
650 .and_then(|r| r.task_id.as_ref())
651 .cloned()
652 .ok_or_else(|| EngineError::TokenNotFound(fp.clone()))
653 })
654 .await?
655 }
656
657 /// Resolve a short worker handle (`wh-XXXXXXXX`) to the bound
658 /// `task_id`. Used on `/v1/worker/submit` when the Bearer is a short
659 /// handle string rather than a full `CapToken` JSON. A missing entry
660 /// returns `TokenNotFound`, i.e. "the handle is not in the store".
661 pub async fn task_id_from_handle(&self, handle: &str) -> Result<StepId, EngineError> {
662 let h = handle.to_string();
663 self.with_state("task_id_from_handle", move |s| {
664 let fp = s
665 .worker_handles
666 .get(&h)
667 .cloned()
668 .ok_or_else(|| EngineError::TokenNotFound(format!("handle={h}")))?;
669 s.tokens
670 .get(&fp)
671 .and_then(|r| r.task_id.as_ref())
672 .cloned()
673 .ok_or_else(|| EngineError::TokenNotFound(format!("fp={fp}")))
674 })
675 .await?
676 }
677
678 /// Submit a worker result via a short handle. Skips token verification
679 /// and updates `output_tail` `Final` + `task.last_result` directly in
680 /// a thin path. The caller is expected to have already resolved
681 /// `task_id` via `task_id_from_handle` — the handle's presence in
682 /// `worker_handles` means it was minted server-side and is therefore
683 /// trusted.
684 pub async fn submit_worker_result_trusted(
685 &self,
686 task_id: &StepId,
687 attempt: u32,
688 value: Value,
689 ok: bool,
690 ) -> Result<(), EngineError> {
691 // GH #51 — completion-time verdict-contract enforcement, embedded
692 // choke point 1 of 2 (see `Self::verdict_contract_completion_check`'s
693 // doc). This path always submits a `Final` by construction (there
694 // is no other event kind on `/v1/worker/submit`), so the check
695 // always applies — unlike `submit_output` below, no `if let
696 // OutputEvent::Final { .. }` guard is needed here since there is
697 // no other `OutputEvent` variant this function could be asked to
698 // write. Runs BEFORE the `output_tail` write immediately below:
699 // on `Err`, this returns immediately and neither `with_state` call
700 // in this function executes.
701 let comparable_value =
702 content_ref_to_comparable_string(crate::worker::output::ContentRef::Inline {
703 value: value.clone(),
704 });
705 self.verdict_contract_completion_check(task_id, attempt, ok, &comparable_value)
706 .await?;
707 let task_id_for_apply = task_id.clone();
708 let value_for_event = value.clone();
709 self.with_state("submit_worker_result_trusted.output", move |s| {
710 let ev = crate::worker::output::OutputEvent::Final {
711 content: crate::worker::output::ContentRef::Inline {
712 value: value_for_event,
713 },
714 ok,
715 };
716 s.output_store
717 .entry((task_id_for_apply.clone(), attempt))
718 .or_default()
719 .push(ev.clone());
720 s.push_event(crate::core::state::Event::WorkerOutput {
721 task_id: task_id_for_apply,
722 attempt,
723 event: ev,
724 });
725 })
726 .await?;
727 let task_id_for_result = task_id.clone();
728 let value_for_result = value.clone();
729 self.with_state("submit_worker_result_trusted.last_result", move |s| {
730 if let Some(t) = s.tasks.get_mut(&task_id_for_result) {
731 t.last_result = Some(value_for_result);
732 t.updated_at = now_unix();
733 }
734 })
735 .await?;
736 // subtask-4 / ST2 rework: this path always submits a `Final` (there
737 // is no other event kind on `/v1/worker/submit`), so the
738 // submit-time projection sink always fires — see
739 // `materialize_final_submission`'s doc and `submit_output`'s
740 // Invariants (fail-open, never turns a would-have-succeeded submit
741 // into a failure).
742 let content = crate::worker::output::ContentRef::Inline { value };
743 self.materialize_final_submission(task_id, attempt, &content, ok)
744 .await?;
745 Ok(())
746 }
747
748 /// Stage a named `Artifact` from a worker via a short handle (GH #36
749 /// ST1: named multi-part worker output). Trusted analog of
750 /// [`Self::submit_worker_result_trusted`] for `OutputEvent::Artifact`:
751 /// skips token verification for the same reason (the caller already
752 /// resolved `task_id` via `task_id_from_handle`, so the handle's
753 /// presence in `worker_handles` is itself the trust boundary).
754 ///
755 /// Appends to the same per-`(task_id, attempt)` `output_store` tail
756 /// [`Self::dispatch_attempt_with`]'s Final-pull later folds into
757 /// `{"out": <final>, "parts": {<name>: <value>, ...}}` (see that
758 /// method's doc for the fold semantics — event order, last-write-wins
759 /// per name), AND records `name` in `EngineState.worker_artifact_names`
760 /// — the fold's allowlist of the WORKER's own staged parts, as opposed
761 /// to every `Artifact` that happens to land on the shared tail (e.g. an
762 /// audit sidecar finding; see that field's doc). Also dual-writes to
763 /// the Data-plane `OutputStore` the same way [`Self::submit_output`]'s
764 /// `Artifact` arm does, via [`Self::materialize_artifact_submission`]
765 /// (the artifact's own `name` is its Data-plane key, no
766 /// canonicalization — see that method's doc).
767 pub async fn stage_worker_artifact_trusted(
768 &self,
769 task_id: &StepId,
770 attempt: u32,
771 name: String,
772 value: Value,
773 ) -> Result<(), EngineError> {
774 let content = crate::worker::output::ContentRef::Inline { value };
775 let task_id_for_apply = task_id.clone();
776 let name_for_apply = name.clone();
777 let content_for_apply = content.clone();
778 self.with_state("stage_worker_artifact_trusted.output", move |s| {
779 let ev = crate::worker::output::OutputEvent::Artifact {
780 name: name_for_apply.clone(),
781 content: content_for_apply,
782 };
783 s.output_store
784 .entry((task_id_for_apply.clone(), attempt))
785 .or_default()
786 .push(ev.clone());
787 s.worker_artifact_names
788 .entry((task_id_for_apply.clone(), attempt))
789 .or_default()
790 .push(name_for_apply);
791 s.push_event(crate::core::state::Event::WorkerOutput {
792 task_id: task_id_for_apply,
793 attempt,
794 event: ev,
795 });
796 })
797 .await?;
798 self.materialize_artifact_submission(task_id, attempt, &name, &content)
799 .await?;
800 Ok(())
801 }
802
803 /// GH #36 ST1: the set of `Artifact` names staged for `(task_id,
804 /// attempt)` via [`Self::stage_worker_artifact_trusted`] — see
805 /// `EngineState.worker_artifact_names`'s doc. Used by
806 /// [`Self::dispatch_attempt_with`]'s Final-pull to distinguish a
807 /// worker's own named parts from any other `Artifact` producer on the
808 /// same tail.
809 async fn worker_artifact_names_for(&self, task_id: &StepId, attempt: u32) -> Vec<String> {
810 let key = (task_id.clone(), attempt);
811 self.with_state("worker_artifact_names_for", move |s| {
812 s.worker_artifact_names
813 .get(&key)
814 .cloned()
815 .unwrap_or_default()
816 })
817 .await
818 .unwrap_or_default()
819 }
820
821 /// Mint a short handle and register it in the `worker_handles` map.
822 /// Called immediately after the worker-token mint inside
823 /// `dispatch_attempt_with`, and issues a handle bound to the same
824 /// token fingerprint. Format is `wh-<8 hex chars>` (11 chars total),
825 /// designed to remove the base64 copy-paste failure mode.
826 async fn mint_worker_handle(&self, worker_fp: String) -> Result<String, EngineError> {
827 // The handle is a sole bearer secret on the `/v1/worker/submit`
828 // short-handle path (`submit_worker_result_trusted` skips token
829 // verification), so it must be unguessable — OS RNG, not the
830 // predictable uid counter. 8 hex chars (~4B entropy) keeps the
831 // documented `wh-<8 hex>` wire shape; collision between live
832 // handles is negligible at in-process handle counts.
833 let short = crate::types::secure_hex(4);
834 let handle = format!("wh-{short}");
835 let h = handle.clone();
836 self.with_state("mint_worker_handle", move |s| {
837 s.worker_handles.insert(h, worker_fp);
838 })
839 .await?;
840 Ok(handle)
841 }
842
843 // ═══════════════════════════════════════════════════════════════════════
844 // Session API
845 // ═══════════════════════════════════════════════════════════════════════
846
847 /// Attach a new session with default `OperatorInfo` (`Automate`, no
848 /// bridges/hooks). Shorthand for `attach_with(.., OperatorInfo::default())`.
849 pub async fn attach(
850 &self,
851 operator_id: impl Into<String>,
852 role: Role,
853 ttl: Duration,
854 ) -> Result<CapToken, EngineError> {
855 self.attach_with(
856 operator_id,
857 role,
858 ttl,
859 crate::core::ctx::OperatorInfo::default(),
860 )
861 .await
862 }
863
864 // ═══════════════════════════════════════════════════════════════════════
865 // BridgeRegistry API.
866 // ═══════════════════════════════════════════════════════════════════════
867
868 /// Register a `SeniorBridge` under a name. An existing entry with the
869 /// same name is overwritten. On the persisted-session reattach path,
870 /// the caller re-registers under the same ID beforehand and the
871 /// bridge becomes effective again.
872 pub async fn register_senior_bridge(
873 &self,
874 id: impl Into<String>,
875 bridge: Arc<dyn SeniorBridge>,
876 ) {
877 self.inner
878 .senior_bridges
879 .write()
880 .await
881 .insert(id.into(), bridge);
882 }
883
884 /// Register a `SpawnHook` under a name. An existing entry with the
885 /// same name is overwritten.
886 pub async fn register_spawn_hook(&self, id: impl Into<String>, hook: Arc<dyn SpawnHook>) {
887 self.inner.spawn_hooks.write().await.insert(id.into(), hook);
888 }
889
890 /// Register an `Operator` (a spawn-body backend) under a name. An
891 /// existing entry with the same name is overwritten.
892 /// `OperatorDelegateMiddleware` looks this up via `ctx` and, when
893 /// `kind = MainAi` / `Composite`, bypasses `inner.spawn` and calls
894 /// `operator.execute` instead.
895 pub async fn register_operator(
896 &self,
897 id: impl Into<String>,
898 operator: Arc<dyn crate::operator::Operator>,
899 ) {
900 self.inner
901 .operators
902 .write()
903 .await
904 .insert(id.into(), operator);
905 }
906
907 /// Unregister a `SeniorBridge` by name (e.g. on WebSocket disconnect
908 /// or explicit teardown). A missing ID is a no-op.
909 pub async fn unregister_senior_bridge(&self, id: &str) {
910 self.inner.senior_bridges.write().await.remove(id);
911 }
912
913 /// Unregister a `SpawnHook` by name. A missing ID is a no-op.
914 pub async fn unregister_spawn_hook(&self, id: &str) {
915 self.inner.spawn_hooks.write().await.remove(id);
916 }
917
918 /// Unregister an `Operator` backend by name. A missing ID is a no-op.
919 pub async fn unregister_operator(&self, id: &str) {
920 self.inner.operators.write().await.remove(id);
921 }
922
923 /// Snapshot the list of registered `SpawnHook` IDs (for test
924 /// observation and debugging).
925 pub async fn list_spawn_hook_ids(&self) -> Vec<String> {
926 self.inner
927 .spawn_hooks
928 .read()
929 .await
930 .keys()
931 .cloned()
932 .collect()
933 }
934
935 /// Snapshot the list of registered `SeniorBridge` IDs.
936 pub async fn list_senior_bridge_ids(&self) -> Vec<String> {
937 self.inner
938 .senior_bridges
939 .read()
940 .await
941 .keys()
942 .cloned()
943 .collect()
944 }
945
946 /// Snapshot the list of registered `Operator` IDs.
947 pub async fn list_operator_ids(&self) -> Vec<String> {
948 self.inner.operators.read().await.keys().cloned().collect()
949 }
950
951 /// Attach specifying IDs directly. The caller is expected to have
952 /// pre-registered them via `register_senior_bridge` /
953 /// `register_spawn_hook` / `register_operator`. This is the canonical
954 /// path when persistence is in play.
955 ///
956 /// `kind` is the "Runtime Global" tier of the `OperatorKind` cascade
957 /// (stored verbatim on `OperatorSession.operator_kind`): `Some(_)` is
958 /// an explicit request (including `Some(OperatorKind::Automate)`) that
959 /// outranks the BP-level tiers; `None` leaves it unspecified so the
960 /// BP-level tiers / final default decide. See
961 /// `crate::core::ctx::collapse_operator_kind`.
962 #[allow(clippy::too_many_arguments)]
963 pub async fn attach_with_ids(
964 &self,
965 operator_id: impl Into<String>,
966 role: Role,
967 ttl: Duration,
968 kind: Option<OperatorKind>,
969 bridge_id: Option<String>,
970 hook_id: Option<String>,
971 operator_backend_id: Option<String>,
972 operator_kind_overrides: HashMap<String, OperatorKind>,
973 bp_agent_kinds: HashMap<String, OperatorKind>,
974 bp_global_kind: Option<OperatorKind>,
975 ) -> Result<CapToken, EngineError> {
976 let operator_id = operator_id.into();
977 let token = self
978 .inner
979 .signer
980 .session(operator_id.clone(), role, vec!["*".into()], ttl);
981 let session_id = SessionId::new();
982 let fp = token.fingerprint();
983 let now = now_unix();
984 let token_for_store = token.clone();
985
986 self.with_state("attach_with_ids", |s| {
987 s.tokens
988 .insert(fp.clone(), CapTokenRecord::from_token(token_for_store));
989 s.sessions.insert(
990 session_id.clone(),
991 OperatorSession {
992 id: session_id.clone(),
993 operator_id: operator_id.clone(),
994 role,
995 attached_at: now,
996 last_seen: now,
997 attached: true,
998 owned_task_ids: Vec::new(),
999 token_fp: fp.clone(),
1000 operator_kind: kind,
1001 runtime_agent_kinds: operator_kind_overrides,
1002 bp_agent_kinds,
1003 bp_global_kind,
1004 bridge_id,
1005 hook_id,
1006 operator_backend_id,
1007 },
1008 );
1009 s.push_event(Event::SessionAttached {
1010 session_id: session_id.clone(),
1011 role,
1012 });
1013 })
1014 .await?;
1015
1016 let _ = self
1017 .inner
1018 .event_tx
1019 .send(Event::SessionAttached { session_id, role });
1020 Ok(token)
1021 }
1022
1023 /// Build an `OperatorInfo` by looking up the session's registered IDs
1024 /// on the `BridgeRegistry`, plus resolving the 4-tier `OperatorKind`
1025 /// cascade for `agent_name` via `crate::core::ctx::collapse_operator_kind`.
1026 /// Used when `dispatch_attempt` injects `Ctx`. An unresolved ID
1027 /// (nothing registered) is silently `None` — the bridge / hook simply
1028 /// does not fire and the default behaviour applies.
1029 async fn resolve_operator_info(
1030 &self,
1031 session: &OperatorSession,
1032 agent_name: &str,
1033 ) -> OperatorInfo {
1034 let senior_bridge = if let Some(id) = &session.bridge_id {
1035 self.inner.senior_bridges.read().await.get(id).cloned()
1036 } else {
1037 None
1038 };
1039 let spawn_hook = if let Some(id) = &session.hook_id {
1040 self.inner.spawn_hooks.read().await.get(id).cloned()
1041 } else {
1042 None
1043 };
1044 let operator = if let Some(id) = &session.operator_backend_id {
1045 self.inner.operators.read().await.get(id).cloned()
1046 } else {
1047 None
1048 };
1049 let runtime_agent = session.runtime_agent_kinds.get(agent_name).copied();
1050 // "Runtime Global" tier: `Some(_)` is always an explicit request
1051 // (see the field doc on `OperatorSession.operator_kind`).
1052 let runtime_global = session.operator_kind;
1053 let bp_agent = session.bp_agent_kinds.get(agent_name).copied();
1054 let bp_global = session.bp_global_kind;
1055 let kind = crate::core::ctx::collapse_operator_kind(
1056 runtime_agent,
1057 runtime_global,
1058 bp_agent,
1059 bp_global,
1060 );
1061 OperatorInfo {
1062 kind,
1063 id: session.operator_id.clone(),
1064 senior_bridge,
1065 spawn_hook,
1066 operator,
1067 }
1068 }
1069
1070 /// Convenience attach that takes an `OperatorInfo` (three
1071 /// `Arc<dyn ...>` fields plus `kind`) **inline**.
1072 ///
1073 /// # Pipeline
1074 ///
1075 /// Each `Arc<dyn ...>` is auto-registered on the engine's registry
1076 /// under a synthetic ID (`br-<hex>` / `hk-<hex>` / `ob-<hex>`), and
1077 /// the session stores that synthetic ID. Subsequent `dispatch_attempt`
1078 /// calls rebuild the `Arc`s from those IDs via
1079 /// `resolve_operator_info`, and the three middlewares fire as usual.
1080 ///
1081 /// # ⚠ Non-persisted sessions only
1082 ///
1083 /// Because this API takes inline `Arc`s, the reattach path after
1084 /// session persistence cannot rebuild them — the synthetic IDs are
1085 /// not present in a freshly started process's registry. If you need
1086 /// persistence, use [`Self::attach_with_ids`] with `register_*` calls
1087 /// beforehand to go through **named IDs** instead.
1088 ///
1089 /// Handy for tests and short-lived in-process sessions. Production
1090 /// WebSocket callbacks and the like should prefer `attach_with_ids`
1091 /// as the canonical path.
1092 pub async fn attach_with(
1093 &self,
1094 operator_id: impl Into<String>,
1095 role: Role,
1096 ttl: Duration,
1097 operator_info: crate::core::ctx::OperatorInfo,
1098 ) -> Result<CapToken, EngineError> {
1099 let operator_id = operator_id.into();
1100 // The caller always hands in a fully-formed `OperatorInfo`
1101 // (including its `kind`), so it is stored as an explicit "Runtime
1102 // Global" tier request (`Some(kind)`) — this path never persists
1103 // BP-level tiers (both stay empty below), so `Some(kind)` resolves
1104 // to the same `kind` at dispatch either way; see
1105 // `OperatorSession.operator_kind` doc.
1106 let kind = operator_info.kind;
1107 // BridgeRegistry auto-register: when the caller hands in an
1108 // `Arc<dyn>` directly, register it under a synthesised ID (the inline
1109 // path aware of persistence). Callers who want to pre-register with a
1110 // named ID should use `register_senior_bridge` / `register_spawn_hook`
1111 // + `attach_with_ids`.
1112 let bridge_id = if let Some(bridge) = operator_info.senior_bridge.clone() {
1113 let id = format!("br-{}", crate::types::uid_hex(8));
1114 self.inner
1115 .senior_bridges
1116 .write()
1117 .await
1118 .insert(id.clone(), bridge);
1119 Some(id)
1120 } else {
1121 None
1122 };
1123 let hook_id = if let Some(hook) = operator_info.spawn_hook.clone() {
1124 let id = format!("hk-{}", crate::types::uid_hex(8));
1125 self.inner
1126 .spawn_hooks
1127 .write()
1128 .await
1129 .insert(id.clone(), hook);
1130 Some(id)
1131 } else {
1132 None
1133 };
1134 let operator_backend_id = if let Some(operator) = operator_info.operator.clone() {
1135 // `ob-` = operator-backend registry id. Renamed from `op-` in the
1136 // issue #11 prefix reconciliation: `op-` used to collide with the
1137 // WS operator sid shape (now unified into `S-<hex>` anyway), and a
1138 // shared prefix across two unrelated registries made log filtering
1139 // by prefix silently ambiguous.
1140 let id = format!("ob-{}", crate::types::uid_hex(8));
1141 self.inner
1142 .operators
1143 .write()
1144 .await
1145 .insert(id.clone(), operator);
1146 Some(id)
1147 } else {
1148 None
1149 };
1150
1151 let token = self
1152 .inner
1153 .signer
1154 .session(operator_id.clone(), role, vec!["*".into()], ttl);
1155 let session_id = SessionId::new();
1156 let fp = token.fingerprint();
1157 let now = now_unix();
1158 let token_for_store = token.clone();
1159
1160 self.with_state("attach_with", |s| {
1161 s.tokens
1162 .insert(fp.clone(), CapTokenRecord::from_token(token_for_store));
1163 s.sessions.insert(
1164 session_id.clone(),
1165 OperatorSession {
1166 id: session_id.clone(),
1167 operator_id,
1168 role,
1169 attached_at: now,
1170 last_seen: now,
1171 attached: true,
1172 owned_task_ids: Vec::new(),
1173 token_fp: fp.clone(),
1174 operator_kind: Some(kind),
1175 runtime_agent_kinds: HashMap::new(),
1176 bp_agent_kinds: HashMap::new(),
1177 bp_global_kind: None,
1178 bridge_id,
1179 hook_id,
1180 operator_backend_id,
1181 },
1182 );
1183 s.push_event(Event::SessionAttached {
1184 session_id: session_id.clone(),
1185 role,
1186 });
1187 })
1188 .await?;
1189
1190 let _ = self
1191 .inner
1192 .event_tx
1193 .send(Event::SessionAttached { session_id, role });
1194 Ok(token)
1195 }
1196
1197 /// Mark the session bound to `token` as detached (`attached = false`).
1198 /// Tasks are left in place — a later `attach`/`attach_with_ids` call
1199 /// carrying the same registered bridge/hook IDs can pick them back up.
1200 pub async fn detach(&self, token: &CapToken) -> Result<(), EngineError> {
1201 self.verify_token(token, Verb::DetachSession).await?;
1202 let fp = token.fingerprint();
1203 self.with_state("detach", move |s| {
1204 let sid = s
1205 .sessions
1206 .iter()
1207 .find(|(_, sess)| sess.token_fp == fp)
1208 .map(|(id, _)| id.clone());
1209 if let Some(sid) = sid {
1210 if let Some(sess) = s.sessions.get_mut(&sid) {
1211 sess.attached = false;
1212 }
1213 s.push_event(Event::SessionDetached {
1214 session_id: sid.clone(),
1215 });
1216 let _ = sid;
1217 }
1218 })
1219 .await?;
1220 Ok(())
1221 }
1222
1223 /// Refresh the session's `last_seen` timestamp and mark it `attached`.
1224 /// Called periodically by an attached client to avoid being flipped to
1225 /// detached by `start_detach_loop`.
1226 pub async fn heartbeat(&self, token: &CapToken) -> Result<(), EngineError> {
1227 self.verify_token(token, Verb::Heartbeat).await?;
1228 let now = now_unix();
1229 let fp = token.fingerprint();
1230 self.with_state("heartbeat", move |s| {
1231 if let Some(sess) = s.sessions.values_mut().find(|sess| sess.token_fp == fp) {
1232 sess.last_seen = now;
1233 sess.attached = true;
1234 }
1235 })
1236 .await?;
1237 Ok(())
1238 }
1239
1240 // ═══════════════════════════════════════════════════════════════════════
1241 // Task lifecycle
1242 // ═══════════════════════════════════════════════════════════════════════
1243
1244 /// Create a new `TaskState` from `spec` and register its initial
1245 /// prompt. When the calling token is a Worker (i.e. this is a
1246 /// recursive spawn), the new task inherits `parent.spawn_depth + 1`
1247 /// and is rejected with `SpawnDepthExceeded` once `max_spawn_depth` is
1248 /// hit; an Operator-issued call starts at depth 0.
1249 pub async fn start_task(
1250 &self,
1251 token: &CapToken,
1252 spec: TaskSpec,
1253 ) -> Result<StepId, EngineError> {
1254 self.verify_token(token, Verb::StartTask).await?;
1255 let task_id = StepId::new();
1256 let initial_directive = spec.initial_directive.clone();
1257 let task_id_clone = task_id.clone();
1258 let fp = token.fingerprint();
1259 let max_depth = self.inner.cfg.max_spawn_depth;
1260 self.with_state("start_task", move |s| {
1261 // Recursive swarm depth gate (recursion guard):
1262 // Worker tokens carry CapTokenRecord.parent_task_id. Give the
1263 // child parent's spawn_depth + 1; if it exceeds `max`, raise an
1264 // error. Operator tokens (parent_task_id=None) start at depth 0.
1265 let parent_depth_opt = s
1266 .tokens
1267 .get(&fp)
1268 .and_then(|rec| rec.task_id.as_ref())
1269 .and_then(|tid| s.tasks.get(tid))
1270 .map(|t| t.spawn_depth);
1271 let depth = match parent_depth_opt {
1272 Some(d) => {
1273 if d + 1 >= max_depth {
1274 return Err(EngineError::SpawnDepthExceeded {
1275 current: d + 1,
1276 max: max_depth,
1277 });
1278 }
1279 d + 1
1280 }
1281 None => 0,
1282 };
1283
1284 let mut task = TaskState::new(task_id_clone.clone(), spec);
1285 task.spawn_depth = depth;
1286 s.tasks.insert(task_id_clone.clone(), task);
1287 s.prompts
1288 .insert((task_id_clone.clone(), 1), initial_directive);
1289 // Link to the owner session (only Operator tokens match; Worker tokens have no session).
1290 if let Some(sess) = s.sessions.values_mut().find(|sess| sess.token_fp == fp) {
1291 sess.owned_task_ids.push(task_id_clone.clone());
1292 }
1293 s.push_event(Event::TaskCreated {
1294 task_id: task_id_clone.clone(),
1295 });
1296 Ok::<(), EngineError>(())
1297 })
1298 .await??;
1299 let _ = self.inner.event_tx.send(Event::TaskCreated {
1300 task_id: task_id.clone(),
1301 });
1302 Ok(task_id)
1303 }
1304
1305 /// Fetch a snapshot of `TaskState` for `task_id`, subject to the
1306 /// task-ownership gate (see `verify_token_for_task`).
1307 pub async fn read_task_state(
1308 &self,
1309 token: &CapToken,
1310 task_id: &StepId,
1311 ) -> Result<TaskState, EngineError> {
1312 self.verify_token_for_task(token, Verb::ReadTaskState, task_id)
1313 .await?;
1314 let task_id = task_id.clone();
1315 self.with_state("read_task_state", move |s| {
1316 s.tasks
1317 .get(&task_id)
1318 .cloned()
1319 .ok_or_else(|| EngineError::TaskNotFound(task_id.to_string()))
1320 })
1321 .await?
1322 }
1323
1324 /// Mark `task_id` as `Cancelled` and wake any caller blocked in
1325 /// `poll_task` for it.
1326 pub async fn cancel_task(&self, token: &CapToken, task_id: &StepId) -> Result<(), EngineError> {
1327 self.verify_token_for_task(token, Verb::CancelTask, task_id)
1328 .await?;
1329 let tid = task_id.clone();
1330 self.with_state("cancel_task", move |s| {
1331 let task = s
1332 .tasks
1333 .get_mut(&tid)
1334 .ok_or_else(|| EngineError::TaskNotFound(tid.to_string()))?;
1335 task.status = TaskStatus::Cancelled;
1336 task.updated_at = now_unix();
1337 s.push_event(Event::TaskCancelled {
1338 task_id: tid.clone(),
1339 });
1340 Ok::<(), EngineError>(())
1341 })
1342 .await??;
1343 self.wake_task(task_id).await?;
1344 Ok(())
1345 }
1346
1347 /// Dispatch a single attempt through the given `spawner`.
1348 ///
1349 /// The lock is only held for snapshot capture; the actual spawn and
1350 /// completion await happen outside the lock (R3 discipline).
1351 ///
1352 /// Sits on the Domain side of the Data / Domain split. The dispatch
1353 /// path itself does not touch big response bodies — those flow through
1354 /// the Data plane (`output_store` module + sink / input_inject
1355 /// `SpawnerLayer`s) around this method.
1356 ///
1357 /// The caller does the compile plus `service::linker::link` and
1358 /// carries the same stack through each dispatch. Because the spawner
1359 /// is passed per-request rather than looked up from engine-global
1360 /// state, parallel requests against a single `Engine` instance
1361 /// (different Blueprints, different spawners) do not race.
1362 ///
1363 /// `run_id`, when `Some` (issue #13 run_id propagation —
1364 /// `EngineDispatcher` threads it in from its `RunContext`), is
1365 /// inserted into `Ctx.meta.runtime["run_id"]` (a plain JSON string)
1366 /// alongside `worker_handle`, so `Operator::execute` implementations
1367 /// (e.g. `WSOperatorSession`) can read it back and surface it to the
1368 /// worker (Spawn directive / prompt). `None` (every pre-existing
1369 /// caller / test) omits the key entirely — unchanged behavior.
1370 pub async fn dispatch_attempt_with(
1371 &self,
1372 token: &CapToken,
1373 task_id: &StepId,
1374 spawner: &Arc<dyn SpawnerAdapter>,
1375 run_id: Option<&RunId>,
1376 ) -> Result<DispatchOutcome, EngineError> {
1377 self.verify_token(token, Verb::DispatchAttempt).await?;
1378 let task_id = task_id.clone();
1379
1380 // 1) Under the lock: increment the attempt number, mark Running, snapshot the
1381 // prompt, and pull `operator_info` from the session so we can inject it into Ctx.
1382 let fp = token.fingerprint();
1383 let tid_for_prep = task_id.clone();
1384 let (attempt, agent, session_snapshot, step_ctx) = self
1385 .with_state("dispatch.prep", move |s| {
1386 let task = s
1387 .tasks
1388 .get_mut(&tid_for_prep)
1389 .ok_or_else(|| EngineError::TaskNotFound(tid_for_prep.to_string()))?;
1390 task.attempt += 1;
1391 task.status = TaskStatus::Running;
1392 task.updated_at = now_unix();
1393 // The spawner pulls the prompt via engine.fetch_prompt. In prep,
1394 // if the prompts table has no entry for this attempt yet,
1395 // fall back and insert `initial_directive` so the subsequent
1396 // fetch_prompt succeeds.
1397 let attempt = task.attempt;
1398 let initial = task.spec.initial_directive.clone();
1399 s.prompts
1400 .entry((tid_for_prep.clone(), attempt))
1401 .or_insert(initial);
1402 let task = s
1403 .tasks
1404 .get(&tid_for_prep)
1405 .ok_or_else(|| EngineError::TaskNotFound(tid_for_prep.to_string()))?;
1406 let agent = task.spec.agent.clone();
1407 // GH #21 Phase 2: re-read `TaskSpec.step_ctx` on EVERY
1408 // attempt (not cached once at start_task) so retries and
1409 // Run-rekicks all carry the Step tier through to Ctx —
1410 // see TaskSpec.step_ctx's doc.
1411 let step_ctx = task.spec.step_ctx.clone();
1412 // Session snapshot (looked up by token nonce). When no session
1413 // exists (worker token invoked directly / test injection), fall
1414 // back to None → default OperatorInfo.
1415 let sess_clone = s
1416 .sessions
1417 .values()
1418 .find(|sess| sess.token_fp == fp)
1419 .cloned();
1420 Ok::<_, EngineError>((attempt, agent, sess_clone, step_ctx))
1421 })
1422 .await??;
1423 // BridgeRegistry lookup + per-agent OperatorKind cascade.
1424 let operator_info = match session_snapshot {
1425 Some(sess) => self.resolve_operator_info(&sess, &agent).await,
1426 None => OperatorInfo::default(),
1427 };
1428
1429 // 2) Outside the lock: worker token mint + spawn.
1430 //
1431 // Session-style mint (max_uses=None). Within one attempt the worker is
1432 // expected to hit `verify_token + fetch_prompt + fetch_data + post_result`
1433 // multiple times in order, so `one_time` would exhaust the token on the
1434 // very first verb. Capability is guarded by (a) the role × verb gate and
1435 // (b) the short TTL (1800s).
1436 let worker_token = self.inner.signer.session(
1437 format!("worker-of-{task_id}"),
1438 Role::Worker,
1439 vec!["*".into()],
1440 Duration::from_secs(1800),
1441 );
1442 let worker_fp = worker_token.fingerprint();
1443 let task_id_for_worker = task_id.clone();
1444 let worker_token_for_store = worker_token.clone();
1445 self.with_state("dispatch.mint_worker", move |s| {
1446 s.tokens.insert(
1447 worker_fp,
1448 CapTokenRecord::from_worker_token(worker_token_for_store, task_id_for_worker),
1449 );
1450 })
1451 .await?;
1452
1453 // Mint a short handle (`wh-XXXXXXXX`) and register it in worker_handles.
1454 // Used by the simplified Bearer path for SubAgents (short-handle form
1455 // avoids base64 copy-paste incidents).
1456 let worker_handle = self.mint_worker_handle(worker_token.fingerprint()).await?;
1457
1458 let mut ctx = Ctx::new(task_id.clone(), attempt, agent.clone());
1459 ctx.operator = operator_info; // activates MainAIMiddleware / Senior bridge
1460 ctx.meta
1461 .runtime
1462 .insert("worker_handle".to_string(), Value::String(worker_handle));
1463 if let Some(rid) = run_id {
1464 ctx.meta
1465 .runtime
1466 .insert(RUN_ID_KEY.to_string(), Value::String(rid.to_string()));
1467 }
1468 // GH #21 Phase 2: the Step tier's resolved context bundle (from
1469 // `TaskSpec.step_ctx`, re-read every attempt above) — consumed by
1470 // `AgentContextMiddleware`, which unpacks its keys ahead of the
1471 // Agent / BP-global tiers.
1472 if let Some(step_ctx) = step_ctx {
1473 ctx.meta.runtime.insert(STEP_CTX_KEY.to_string(), step_ctx);
1474 }
1475
1476 let worker = spawner
1477 .spawn(self, &ctx, task_id.clone(), attempt, worker_token)
1478 .await
1479 .map_err(|e| EngineError::DispatchFailed(e.to_string()))?;
1480
1481 // 3) Outside the lock: await worker.join() (signal-only). WorkerError is
1482 // stringified. The value is fetched via output_tail (sink path).
1483 let signal_result: Result<(), String> = worker.join().await.map_err(|e| e.to_string());
1484
1485 // Pull the last Final from output_tail and use it as the value. GH
1486 // #36 ST1 (named multi-part worker output): also fold every
1487 // `Artifact` the WORKER ITSELF staged on the same tail (via
1488 // `stage_worker_artifact_trusted` / `POST /v1/worker/artifact`)
1489 // into a `"parts"` object keyed by name — event order,
1490 // last-write-wins per name (a name staged twice overwrites,
1491 // mirroring `HashMap`/`Map` insert semantics, not an accumulating
1492 // list). `worker_artifact_names_for` is the allowlist that scopes
1493 // this to the worker's own opt-in parts — an `Artifact` some OTHER
1494 // producer appended to this same tail (e.g.
1495 // `AfterRunAuditMiddleware`'s `"audit:<step_ref>"` sidecar finding)
1496 // is left untouched (see `fold_final_and_parts`'s doc). When at
1497 // least one part was staged, the BP-chain value becomes `{"out":
1498 // <final value>, "parts": {...}}`; zero parts staged (the
1499 // pre-GH-#36 case, and every non-opt-in step) leaves the value
1500 // exactly the plain `Final` value, byte-identical to before this
1501 // change.
1502 let value_ok: Result<(Value, bool), String> = match signal_result {
1503 Ok(()) => {
1504 let tail = self.output_tail(&task_id, attempt).await;
1505 let staged_names = self.worker_artifact_names_for(&task_id, attempt).await;
1506 fold_final_and_parts(&tail, &staged_names)
1507 .ok_or_else(|| "no Final in output_tail".to_string())
1508 }
1509 Err(msg) => Err(msg),
1510 };
1511
1512 // 4) Under the lock: apply (split the borrow scope so push_event and task mut can co-exist).
1513 let outcome = self
1514 .with_state("dispatch.apply", |s| {
1515 if !s.tasks.contains_key(&task_id) {
1516 return Err(EngineError::TaskNotFound(task_id.to_string()));
1517 }
1518 match value_ok {
1519 Ok((value, ok)) => {
1520 let pass = ok;
1521 {
1522 let task = s.tasks.get_mut(&task_id).unwrap();
1523 task.last_result = Some(value.clone());
1524 task.updated_at = now_unix();
1525 task.status = if pass {
1526 TaskStatus::Pass
1527 } else {
1528 TaskStatus::Blocked
1529 };
1530 }
1531 s.push_event(Event::TaskAttemptCompleted {
1532 task_id: task_id.clone(),
1533 attempt,
1534 result: value.clone(),
1535 });
1536 if pass {
1537 s.push_event(Event::TaskPass {
1538 task_id: task_id.clone(),
1539 result: value.clone(),
1540 });
1541 Ok::<_, EngineError>(DispatchOutcome::Pass(value))
1542 } else {
1543 s.push_event(Event::TaskBlocked {
1544 task_id: task_id.clone(),
1545 result: value.clone(),
1546 });
1547 Ok(DispatchOutcome::Blocked(value))
1548 }
1549 }
1550 Err(msg) => {
1551 let task = s.tasks.get_mut(&task_id).unwrap();
1552 task.status = TaskStatus::Blocked;
1553 task.updated_at = now_unix();
1554 Err(EngineError::DispatchFailed(msg))
1555 }
1556 }
1557 })
1558 .await??;
1559
1560 // event broadcast (outside the lock — push_event feeds the in-memory tail; broadcast is a separate path).
1561 let _ = self.inner.event_tx.send(Event::TaskAttemptCompleted {
1562 task_id: task_id.clone(),
1563 attempt,
1564 result: match &outcome {
1565 DispatchOutcome::Pass(v) | DispatchOutcome::Blocked(v) => v.clone(),
1566 _ => Value::Null,
1567 },
1568 });
1569
1570 // Wake any callers waiting in poll_task.
1571 self.wake_task(&task_id).await?;
1572
1573 Ok(outcome)
1574 }
1575
1576 /// Dispatch a single attempt, opt-in to the replay-log Core primitive
1577 /// ([`crate::store::replay`]) via `run_ctx`.
1578 ///
1579 /// This is the [`Self::dispatch_attempt_with`] sibling used by callers
1580 /// that carry a `RunContext` with `replay_store` / `replay_cursor`
1581 /// populated. Behavior versus the plain `dispatch_attempt_with`:
1582 ///
1583 /// - **`run_ctx.replay_cursor` is `Some` AND the cursor has a matching
1584 /// `(step_ref, input_hash, occurrence)` row** — the stored value is
1585 /// returned verbatim as `DispatchOutcome::Pass(v)`; the `Adapter`
1586 /// (spawner + worker) is never touched. The task's `attempt` is
1587 /// still bumped and `TaskStatus` set to `Pass`, so downstream state
1588 /// (`task.last_result`, `TaskAttemptCompleted` / `TaskPass` events,
1589 /// `wake_task`) fires the same way an ordinary Pass would.
1590 /// - **Miss (or `replay_cursor: None`)** — the ordinary spawn path
1591 /// runs. When `run_ctx.replay_store` is `Some` AND the outcome is
1592 /// `Pass`, one `ReplayEntry` is appended carrying the whole `Ctx`
1593 /// snapshot (with `operator` dropped by `#[serde(skip)]`) plus the
1594 /// `step_output` value. `Blocked` / `Err` outcomes are never
1595 /// logged — a partial-failure row would poison the replay path
1596 /// after a subsequent successful retry.
1597 ///
1598 /// `run_ctx: None` collapses to the same behavior as
1599 /// `dispatch_attempt_with(token, task_id, spawner, None)` — no run
1600 /// tracing, no replay.
1601 pub async fn dispatch_attempt_with_run_ctx(
1602 &self,
1603 token: &CapToken,
1604 task_id: &StepId,
1605 spawner: &Arc<dyn SpawnerAdapter>,
1606 run_ctx: Option<&RunContext>,
1607 ) -> Result<DispatchOutcome, EngineError> {
1608 self.verify_token(token, Verb::DispatchAttempt).await?;
1609 let task_id = task_id.clone();
1610
1611 // 1) Under the lock: prep (bump attempt, snapshot agent/directive).
1612 let fp = token.fingerprint();
1613 let tid_for_prep = task_id.clone();
1614 let (attempt, agent, session_snapshot, step_ctx, initial_directive) = self
1615 .with_state("dispatch_run_ctx.prep", move |s| {
1616 let task = s
1617 .tasks
1618 .get_mut(&tid_for_prep)
1619 .ok_or_else(|| EngineError::TaskNotFound(tid_for_prep.to_string()))?;
1620 task.attempt += 1;
1621 task.status = TaskStatus::Running;
1622 task.updated_at = now_unix();
1623 let attempt = task.attempt;
1624 let initial = task.spec.initial_directive.clone();
1625 s.prompts
1626 .entry((tid_for_prep.clone(), attempt))
1627 .or_insert(initial.clone());
1628 let task = s
1629 .tasks
1630 .get(&tid_for_prep)
1631 .ok_or_else(|| EngineError::TaskNotFound(tid_for_prep.to_string()))?;
1632 let agent = task.spec.agent.clone();
1633 let step_ctx = task.spec.step_ctx.clone();
1634 let sess_clone = s
1635 .sessions
1636 .values()
1637 .find(|sess| sess.token_fp == fp)
1638 .cloned();
1639 Ok::<_, EngineError>((attempt, agent, sess_clone, step_ctx, initial))
1640 })
1641 .await??;
1642
1643 let operator_info = match session_snapshot {
1644 Some(sess) => self.resolve_operator_info(&sess, &agent).await,
1645 None => OperatorInfo::default(),
1646 };
1647
1648 // 2) Compute the replay key from step_ref (= agent) + hashed input.
1649 // Occurrence comes from the cursor's per-key counter (bumped
1650 // once per dispatch, so a loop that re-visits the same step
1651 // with the same input gets 0, 1, 2, … distinct rows).
1652 let step_ref = agent.clone();
1653 let input_hash = hash_input_value(&initial_directive);
1654 let (replay_hit_value, occurrence) = if let Some(rc) = run_ctx {
1655 if let Some(cursor) = &rc.replay_cursor {
1656 let mut guard = cursor.lock().expect("replay cursor mutex poisoned");
1657 let occ = guard.next_occurrence(&step_ref, &input_hash);
1658 let hit = guard.find(&step_ref, &input_hash, occ);
1659 (hit, occ)
1660 } else {
1661 (None, 0)
1662 }
1663 } else {
1664 (None, 0)
1665 };
1666
1667 // 3) Build the Ctx that (a) either the spawner will see on a miss,
1668 // or (b) we log alongside the replay row.
1669 let mut ctx = Ctx::new(task_id.clone(), attempt, agent.clone());
1670 ctx.operator = operator_info;
1671 if let Some(rc) = run_ctx {
1672 ctx.meta
1673 .runtime
1674 .insert(RUN_ID_KEY.to_string(), Value::String(rc.run_id.to_string()));
1675 }
1676 if let Some(step_ctx) = step_ctx {
1677 ctx.meta.runtime.insert(STEP_CTX_KEY.to_string(), step_ctx);
1678 }
1679
1680 // 4) Replay-hit shortcut: skip the spawn+join, return stored value.
1681 let was_replay_hit = replay_hit_value.is_some();
1682 let value_ok: Result<(Value, bool), String> = if let Some(stored) = replay_hit_value {
1683 tracing::info!(
1684 task_id = %task_id,
1685 step_ref = %step_ref,
1686 occurrence = occurrence,
1687 "replayed from log; worker dispatch skipped"
1688 );
1689 Ok((stored, true))
1690 } else {
1691 // 5) Ordinary spawn path — mint a worker token+handle, run the
1692 // spawner, join, and pull the last Final from output_tail.
1693 let worker_token = self.inner.signer.session(
1694 format!("worker-of-{task_id}"),
1695 Role::Worker,
1696 vec!["*".into()],
1697 Duration::from_secs(1800),
1698 );
1699 let worker_fp = worker_token.fingerprint();
1700 let task_id_for_worker = task_id.clone();
1701 let worker_token_for_store = worker_token.clone();
1702 self.with_state("dispatch_run_ctx.mint_worker", move |s| {
1703 s.tokens.insert(
1704 worker_fp,
1705 CapTokenRecord::from_worker_token(worker_token_for_store, task_id_for_worker),
1706 );
1707 })
1708 .await?;
1709 let worker_handle = self.mint_worker_handle(worker_token.fingerprint()).await?;
1710 ctx.meta
1711 .runtime
1712 .insert("worker_handle".to_string(), Value::String(worker_handle));
1713
1714 let worker = spawner
1715 .spawn(self, &ctx, task_id.clone(), attempt, worker_token)
1716 .await
1717 .map_err(|e| EngineError::DispatchFailed(e.to_string()))?;
1718 let signal_result: Result<(), String> = worker.join().await.map_err(|e| e.to_string());
1719 match signal_result {
1720 Ok(()) => {
1721 let tail = self.output_tail(&task_id, attempt).await;
1722 let staged_names = self.worker_artifact_names_for(&task_id, attempt).await;
1723 fold_final_and_parts(&tail, &staged_names)
1724 .ok_or_else(|| "no Final in output_tail".to_string())
1725 }
1726 Err(msg) => Err(msg),
1727 }
1728 };
1729
1730 // 6) Apply — mirrors `dispatch_attempt_with`'s apply arm exactly
1731 // (task.last_result / status update + TaskAttemptCompleted /
1732 // TaskPass / TaskBlocked events).
1733 let outcome = self
1734 .with_state("dispatch_run_ctx.apply", |s| {
1735 if !s.tasks.contains_key(&task_id) {
1736 return Err(EngineError::TaskNotFound(task_id.to_string()));
1737 }
1738 match value_ok {
1739 Ok((value, ok)) => {
1740 let pass = ok;
1741 {
1742 let task = s.tasks.get_mut(&task_id).unwrap();
1743 task.last_result = Some(value.clone());
1744 task.updated_at = now_unix();
1745 task.status = if pass {
1746 TaskStatus::Pass
1747 } else {
1748 TaskStatus::Blocked
1749 };
1750 }
1751 s.push_event(Event::TaskAttemptCompleted {
1752 task_id: task_id.clone(),
1753 attempt,
1754 result: value.clone(),
1755 });
1756 if pass {
1757 s.push_event(Event::TaskPass {
1758 task_id: task_id.clone(),
1759 result: value.clone(),
1760 });
1761 Ok::<_, EngineError>(DispatchOutcome::Pass(value))
1762 } else {
1763 s.push_event(Event::TaskBlocked {
1764 task_id: task_id.clone(),
1765 result: value.clone(),
1766 });
1767 Ok(DispatchOutcome::Blocked(value))
1768 }
1769 }
1770 Err(msg) => {
1771 let task = s.tasks.get_mut(&task_id).unwrap();
1772 task.status = TaskStatus::Blocked;
1773 task.updated_at = now_unix();
1774 Err(EngineError::DispatchFailed(msg))
1775 }
1776 }
1777 })
1778 .await??;
1779
1780 // 7) On MISS + Pass + replay_store present, append a replay row.
1781 // Replay-HIT rows are already logged from the original run and
1782 // must never be double-logged (Core primitive contract). A
1783 // secondary-persistence failure here (`tracing::warn!` +
1784 // swallow) matches the `run_ctx.run_store.append_step_entry`
1785 // convention in `EngineDispatcher::dispatch`: it must not mask
1786 // the primary dispatch outcome the caller already has in hand.
1787 if !was_replay_hit {
1788 if let (Some(rc), DispatchOutcome::Pass(v)) = (run_ctx, &outcome) {
1789 if let Some(store) = &rc.replay_store {
1790 match ReplayEntry::from_completion(
1791 rc.run_id.clone(),
1792 step_ref.clone(),
1793 input_hash.clone(),
1794 occurrence,
1795 &ctx,
1796 v,
1797 ) {
1798 Ok(entry) => {
1799 if let Err(e) = store.append(entry).await {
1800 tracing::warn!(
1801 run_id = %rc.run_id,
1802 step_ref = %step_ref,
1803 occurrence = occurrence,
1804 error = %e,
1805 "dispatch_attempt_with_run_ctx: replay_store.append failed"
1806 );
1807 }
1808 }
1809 Err(e) => {
1810 tracing::warn!(
1811 run_id = %rc.run_id,
1812 step_ref = %step_ref,
1813 occurrence = occurrence,
1814 error = %e,
1815 "dispatch_attempt_with_run_ctx: ReplayEntry encode failed"
1816 );
1817 }
1818 }
1819 }
1820 }
1821 }
1822
1823 let _ = self.inner.event_tx.send(Event::TaskAttemptCompleted {
1824 task_id: task_id.clone(),
1825 attempt,
1826 result: match &outcome {
1827 DispatchOutcome::Pass(v) | DispatchOutcome::Blocked(v) => v.clone(),
1828 _ => Value::Null,
1829 },
1830 });
1831
1832 self.wake_task(&task_id).await?;
1833
1834 Ok(outcome)
1835 }
1836
1837 // ═══════════════════════════════════════════════════════════════════════
1838 // Worker-side API (= prompt / data fetch + result post)
1839 // ═══════════════════════════════════════════════════════════════════════
1840
1841 /// Fetch the directive/prompt `Value` for `task_id`'s current attempt.
1842 /// Falls back to `initial_directive` when no prompt has been recorded
1843 /// yet for that attempt. Returns the `Value` end-to-end (issue #18);
1844 /// the render down to `String` happens only at the two consumer
1845 /// boundaries — the Worker HTTP path (`fetch_worker_payload*` →
1846 /// `WorkerPayload.prompt: String`) and the WS Spawn frame text
1847 /// render (`operator_ws::session`).
1848 pub async fn fetch_prompt(
1849 &self,
1850 token: &CapToken,
1851 task_id: &StepId,
1852 ) -> Result<Value, EngineError> {
1853 self.verify_token_for_task(token, Verb::FetchPrompt, task_id)
1854 .await?;
1855 let task_id = task_id.clone();
1856 self.with_state("fetch_prompt", move |s| {
1857 let task = s
1858 .tasks
1859 .get(&task_id)
1860 .ok_or_else(|| EngineError::TaskNotFound(task_id.to_string()))?;
1861 s.prompts
1862 .get(&(task_id.clone(), task.attempt.max(1)))
1863 .cloned()
1864 .ok_or_else(|| {
1865 EngineError::ResourceNotFound(format!(
1866 "prompt({}, attempt={})",
1867 task_id, task.attempt
1868 ))
1869 })
1870 })
1871 .await?
1872 }
1873
1874 /// Combined fetch for `HTTP /v1/worker/prompt`: returns `prompt` +
1875 /// (optional) `system` + `agent` + `attempt` in a single round trip.
1876 /// The verb gate reuses `FetchPrompt` — same semantics as "the worker
1877 /// pulls its task input".
1878 ///
1879 /// `system` is the value written by `OperatorSpawner::spawn` through
1880 /// `bake_worker_system_prompt` when it ran; otherwise `None` (no
1881 /// profile present, or the bake never happened).
1882 pub async fn fetch_worker_payload(
1883 &self,
1884 token: &CapToken,
1885 task_id: &StepId,
1886 ) -> Result<crate::types::WorkerPayload, EngineError> {
1887 self.verify_token_for_task(token, Verb::FetchPrompt, task_id)
1888 .await?;
1889 let task_id_clone = task_id.clone();
1890 let mut payload = self
1891 .with_state("fetch_worker_payload", move |s| {
1892 let task = s
1893 .tasks
1894 .get(&task_id_clone)
1895 .ok_or_else(|| EngineError::TaskNotFound(task_id_clone.to_string()))?;
1896 let attempt = task.attempt.max(1);
1897 let prompt = s
1898 .prompts
1899 .get(&(task_id_clone.clone(), attempt))
1900 .cloned()
1901 .ok_or_else(|| {
1902 EngineError::ResourceNotFound(format!(
1903 "prompt({}, attempt={})",
1904 task_id_clone, attempt
1905 ))
1906 })?;
1907 let system = s
1908 .systems
1909 .get(&(task_id_clone.clone(), attempt))
1910 .cloned()
1911 .unwrap_or(None);
1912 let agent = task.spec.agent.clone();
1913 let context = s
1914 .agent_ctx
1915 .get(&(task_id_clone.clone(), attempt))
1916 .map(|e| e.view.clone());
1917 Ok::<_, EngineError>(crate::types::WorkerPayload {
1918 task_id: task_id_clone.clone(),
1919 attempt,
1920 agent,
1921 prompt: render_directive_to_string(&prompt),
1922 system,
1923 context,
1924 system_ref: None,
1925 })
1926 })
1927 .await??;
1928 self.apply_system_ref_threshold(&mut payload).await?;
1929 Ok(payload)
1930 }
1931
1932 /// Fetch a worker payload via a short handle. Skips token verification
1933 /// and returns `prompt` + `system` + `agent` + `attempt` in a thin
1934 /// path. The caller is expected to have already resolved `task_id`
1935 /// via `task_id_from_handle` — the handle's presence in
1936 /// `worker_handles` means it was minted server-side and is therefore
1937 /// trusted.
1938 pub async fn fetch_worker_payload_trusted(
1939 &self,
1940 task_id: &StepId,
1941 ) -> Result<crate::types::WorkerPayload, EngineError> {
1942 let task_id_clone = task_id.clone();
1943 let mut payload = self
1944 .with_state("fetch_worker_payload_trusted", move |s| {
1945 let task = s
1946 .tasks
1947 .get(&task_id_clone)
1948 .ok_or_else(|| EngineError::TaskNotFound(task_id_clone.to_string()))?;
1949 let attempt = task.attempt.max(1);
1950 let prompt = s
1951 .prompts
1952 .get(&(task_id_clone.clone(), attempt))
1953 .cloned()
1954 .ok_or_else(|| {
1955 EngineError::ResourceNotFound(format!(
1956 "prompt({}, attempt={})",
1957 task_id_clone, attempt
1958 ))
1959 })?;
1960 let system = s
1961 .systems
1962 .get(&(task_id_clone.clone(), attempt))
1963 .cloned()
1964 .unwrap_or(None);
1965 let agent = task.spec.agent.clone();
1966 let context = s
1967 .agent_ctx
1968 .get(&(task_id_clone.clone(), attempt))
1969 .map(|e| e.view.clone());
1970 Ok::<_, EngineError>(crate::types::WorkerPayload {
1971 task_id: task_id_clone.clone(),
1972 attempt,
1973 agent,
1974 prompt: render_directive_to_string(&prompt),
1975 system,
1976 context,
1977 system_ref: None,
1978 })
1979 })
1980 .await??;
1981 self.apply_system_ref_threshold(&mut payload).await?;
1982 Ok(payload)
1983 }
1984
1985 /// GH #31: shared threshold-branch tail for
1986 /// [`Self::fetch_worker_payload`] / [`Self::fetch_worker_payload_trusted`].
1987 /// Both build a raw `WorkerPayload` inside `with_state` with `system`
1988 /// populated as before and `system_ref: None`; this runs *outside* any
1989 /// lock (R3 — `SystemRefMode::File`'s `tokio::fs` write is a genuine
1990 /// `.await`, which `with_state`'s sync-closure contract forbids inside
1991 /// the lock) and rewrites `payload.system` / `payload.system_ref` in
1992 /// place per `SystemRefConfig.threshold_bytes`: over-threshold clears
1993 /// `system` and populates `system_ref`; at-or-under-threshold leaves
1994 /// `system` as-is and `system_ref` stays `None`. A no-op when
1995 /// `payload.system` is already `None` (no `system_prompt` was baked).
1996 async fn apply_system_ref_threshold(
1997 &self,
1998 payload: &mut crate::types::WorkerPayload,
1999 ) -> Result<(), EngineError> {
2000 let Some(rendered) = payload.system.take() else {
2001 return Ok(());
2002 };
2003 let cfg = self.cfg().system_ref.clone();
2004 if rendered.len() <= cfg.threshold_bytes {
2005 payload.system = Some(rendered);
2006 return Ok(());
2007 }
2008 use sha2::Digest;
2009 let size_bytes = rendered.len() as u64;
2010 let sha256 = hex::encode(sha2::Sha256::digest(rendered.as_bytes()));
2011 let task_id = &payload.task_id;
2012 let attempt = payload.attempt;
2013 let system_ref = match cfg.mode {
2014 crate::types::SystemRefMode::Http => crate::types::SystemRef {
2015 // The engine has no knowledge of scheme/host here — see
2016 // `SystemRefMode::Http`'s doc for who fills that in.
2017 uri: format!("/v1/worker/prompt/system?task_id={task_id}&attempt={attempt}"),
2018 sha256,
2019 size_bytes,
2020 mode: crate::types::SystemRefMode::Http,
2021 },
2022 crate::types::SystemRefMode::File => {
2023 tokio::fs::create_dir_all(&cfg.store_dir).await?;
2024 let path = cfg.store_dir.join(format!("{task_id}-{attempt}.md"));
2025 tokio::fs::write(&path, rendered.as_bytes()).await?;
2026 crate::types::SystemRef {
2027 uri: format!("file://{}", path.display()),
2028 sha256,
2029 size_bytes,
2030 mode: crate::types::SystemRefMode::File,
2031 }
2032 }
2033 };
2034 payload.system = None;
2035 payload.system_ref = Some(system_ref);
2036 Ok(())
2037 }
2038
2039 /// Returns the effective [`mlua_swarm_schema::ContextPolicy`]
2040 /// `AgentContextMiddleware` resolved and snapshotted for `(task_id,
2041 /// attempt)` at spawn time (the same policy already applied to that
2042 /// key's `EngineState.agent_ctx` entry's `.view`, GH #23 fold).
2043 /// Pass-all (`ContextPolicy::default()`) when no entry exists — either
2044 /// a pre-ST5 spawn, or a spawner stack that never layered
2045 /// `AgentContextMiddleware` (fail-open, mirroring [`Self::output_tail`]'s
2046 /// "no entry = empty default" convention).
2047 ///
2048 /// `crates/mlua-swarm-server/src/worker.rs`'s `GET /v1/worker/prompt`
2049 /// handler reads this back to filter `WorkerPayload.context.steps` via
2050 /// `ContextPolicy::allows_step`, without re-deriving the policy from
2051 /// the Blueprint at fetch time (`projection-adapter` ST5).
2052 pub async fn context_policy_for(
2053 &self,
2054 task_id: &StepId,
2055 attempt: u32,
2056 ) -> mlua_swarm_schema::ContextPolicy {
2057 let key = (task_id.clone(), attempt);
2058 self.with_state("context_policy_for", move |s| {
2059 s.agent_ctx
2060 .get(&key)
2061 .map(|e| e.policy.clone())
2062 .unwrap_or_default()
2063 })
2064 .await
2065 .unwrap_or_default()
2066 }
2067
2068 /// GH #23: returns the Blueprint-wide
2069 /// [`crate::core::step_naming::StepNaming`] table snapshotted for
2070 /// `task_id` (the same `Arc` `crate::blueprint::EngineDispatcher::dispatch`
2071 /// stashed into `EngineState.step_namings` at dispatch time —
2072 /// `Self::start_task`'s `StepId`, not the `TaskId` work item). `None`
2073 /// when no entry exists — either the dispatcher was never given a
2074 /// `StepNaming` (`EngineDispatcher::with_step_naming` not called) or
2075 /// the lock could not be acquired; callers are expected to fall back
2076 /// to the pre-GH-#23 runtime union rule in that case (subtask-2/3
2077 /// consumers).
2078 pub async fn step_naming_for(
2079 &self,
2080 task_id: &StepId,
2081 ) -> Option<Arc<crate::core::step_naming::StepNaming>> {
2082 let key = task_id.clone();
2083 self.with_state("step_naming_for", move |s| {
2084 s.step_namings.get(&key).cloned()
2085 })
2086 .await
2087 .ok()
2088 .flatten()
2089 }
2090
2091 /// GH #27 (follow-up to #23): returns the Blueprint-wide
2092 /// [`crate::core::projection_placement::ProjectionPlacement`] resolver
2093 /// snapshotted for `task_id` (the same `Arc`
2094 /// `crate::blueprint::EngineDispatcher::dispatch` stashed into
2095 /// `EngineState.projection_placements` at dispatch time — mirroring
2096 /// [`Self::step_naming_for`]'s contract exactly). `None` when no entry
2097 /// exists — either the dispatcher was never given a
2098 /// `ProjectionPlacement` (`EngineDispatcher::with_projection_placement`
2099 /// not called) or the lock could not be acquired; callers are expected
2100 /// to fall back to `ProjectionPlacement::default()` (byte-compat with
2101 /// the pre-#27 hardcoded layout) in that case.
2102 pub async fn projection_placement_for(
2103 &self,
2104 task_id: &StepId,
2105 ) -> Option<Arc<crate::core::projection_placement::ProjectionPlacement>> {
2106 let key = task_id.clone();
2107 self.with_state("projection_placement_for", move |s| {
2108 s.projection_placements.get(&key).cloned()
2109 })
2110 .await
2111 .ok()
2112 .flatten()
2113 }
2114
2115 /// Returns the [`crate::core::agent_context::AgentContextView`]
2116 /// snapshotted for `(task_id, attempt)`, if `AgentContextMiddleware`
2117 /// stashed one — the same lookup [`Self::fetch_worker_payload`] /
2118 /// [`Self::fetch_worker_payload_trusted`] perform inline, exposed
2119 /// standalone for callers that only need the view (not a full
2120 /// `WorkerPayload`) — e.g. the HTTP debug-plane `GET
2121 /// /v1/tasks/:id/runs/:run/steps*` handlers resolving a
2122 /// materialized-file root for a step *other than* the one currently
2123 /// fetching its own prompt (`projection-adapter` ST5).
2124 pub async fn agent_context_for(
2125 &self,
2126 task_id: &StepId,
2127 attempt: u32,
2128 ) -> Option<crate::core::agent_context::AgentContextView> {
2129 let key = (task_id.clone(), attempt);
2130 self.with_state("agent_context_for", move |s| {
2131 s.agent_ctx.get(&key).map(|e| e.view.clone())
2132 })
2133 .await
2134 .ok()
2135 .flatten()
2136 }
2137
2138 /// Read the current attempt number for a task (server-side lookup, no
2139 /// token verification). Used on `HTTP /v1/worker/result` when the
2140 /// worker omits `attempt` and the server has to fill it in.
2141 pub async fn task_attempt(&self, task_id: &StepId) -> Result<u32, EngineError> {
2142 let task_id = task_id.clone();
2143 self.with_state("task_attempt", move |s| {
2144 s.tasks
2145 .get(&task_id)
2146 .map(|t| t.attempt)
2147 .ok_or_else(|| EngineError::TaskNotFound(task_id.to_string()))
2148 })
2149 .await?
2150 }
2151
2152 /// Server-side admin API that lets `OperatorSpawner::spawn` bake the
2153 /// rendered `system_prompt` into engine state. There is no verb gate
2154 /// — the only expected caller is inside the spawner. SubAgents fetch
2155 /// this alongside the prompt on the `/v1/worker/prompt` path.
2156 pub async fn bake_worker_system_prompt(
2157 &self,
2158 task_id: &StepId,
2159 attempt: u32,
2160 system: Option<String>,
2161 ) -> Result<(), EngineError> {
2162 let task_id = task_id.clone();
2163 self.with_state("bake_worker_system_prompt", move |s| {
2164 // GH #31: record this agent's most-recently-baked render size
2165 // before `system` is moved into `s.systems.insert` below. Same
2166 // `s.tasks.get(&task_id)` → `.spec.agent` lookup pattern
2167 // `fetch_worker_payload` uses (see its doc for why this keying
2168 // is load-bearing for a later `bp_doctor` route).
2169 if let Some(rendered) = system.as_ref() {
2170 if let Some(agent) = s.tasks.get(&task_id).map(|t| t.spec.agent.clone()) {
2171 s.agent_render_sizes.insert(agent, rendered.len());
2172 }
2173 }
2174 s.systems.insert((task_id, attempt), system);
2175 })
2176 .await?;
2177 Ok(())
2178 }
2179
2180 /// GH #31: the most-recently-baked `system_prompt` render size (in
2181 /// bytes) observed for `agent_name`, if `bake_worker_system_prompt` has
2182 /// ever recorded one — last-write-wins across every `(task_id,
2183 /// attempt)` dispatch of that agent. `None` when no `system_prompt`
2184 /// has ever been baked for this agent name. Read by the `bp_doctor`
2185 /// route this subtask's follow-up adds.
2186 pub async fn agent_last_rendered_size(&self, agent_name: &str) -> Option<usize> {
2187 let agent_name = agent_name.to_string();
2188 self.with_state("agent_last_rendered_size", move |s| {
2189 s.agent_render_sizes.get(&agent_name).copied()
2190 })
2191 .await
2192 .ok()
2193 .flatten()
2194 }
2195
2196 /// GH #31: plain read-through of the baked `system` string for
2197 /// `(task_id, attempt)` from `EngineState.systems`, with no threshold
2198 /// branching. Backs `GET /v1/worker/prompt/system` (the `Http`-mode
2199 /// fetch target `system_ref.uri` points at) — that route needs the
2200 /// exact raw bytes to serve as the response body for the client's
2201 /// sha256 verification, not a `WorkerPayload`-wrapped value.
2202 ///
2203 /// Distinct from `apply_system_ref_threshold` (private, mutates an
2204 /// already-built `WorkerPayload` in place after full construction):
2205 /// this accessor has no threshold logic and is `pub` so
2206 /// `mlua-swarm-server`'s `worker` module can call it directly.
2207 ///
2208 /// Returns `Ok(None)` if no baked system exists for that `(task_id,
2209 /// attempt)` (either the task/attempt has no entry in `s.systems`, or
2210 /// the entry is present but stores `None`) — the caller maps this to
2211 /// a 404.
2212 pub async fn raw_system_prompt(
2213 &self,
2214 task_id: &StepId,
2215 attempt: u32,
2216 ) -> Result<Option<String>, EngineError> {
2217 let task_id = task_id.clone();
2218 self.with_state("raw_system_prompt", move |s| {
2219 s.systems.get(&(task_id, attempt)).cloned().unwrap_or(None)
2220 })
2221 .await
2222 }
2223
2224 /// Fetch an arbitrary named resource previously stored via
2225 /// `set_resource`. Not task-scoped — any valid token with the
2226 /// `FetchData` verb may read any key.
2227 pub async fn fetch_data(&self, token: &CapToken, key: &str) -> Result<Value, EngineError> {
2228 self.verify_token(token, Verb::FetchData).await?;
2229 let key = key.to_string();
2230 self.with_state("fetch_data", move |s| {
2231 s.resources
2232 .get(&key)
2233 .cloned()
2234 .ok_or(EngineError::ResourceNotFound(key))
2235 })
2236 .await?
2237 }
2238
2239 // ───────────────────────────────────────────────────────────────────────
2240 // Output path.
2241 // ───────────────────────────────────────────────────────────────────────
2242
2243 /// Send one output event from inside a `SpawnerAdapter` or worker.
2244 /// Structuring is assumed to be complete by the time we cross the
2245 /// `SpawnerAdapter` boundary; this API just appends to the
2246 /// `OutputStore`, pushes to the `EventLog`, and (for `Final`) emits
2247 /// the `TaskAttemptCompleted` event.
2248 ///
2249 /// This is Domain-side plumbing: it feeds the engine's verdict flow,
2250 /// not the Data-plane store in the `output_store` module. It also
2251 /// does not wake the dispatch path — that is done through the
2252 /// spawner's completion oneshot when the worker terminates.
2253 ///
2254 /// # Submit-time projection sink (subtask-4 / ST2 rework)
2255 ///
2256 /// A `Final` event additionally fans out to the submit-time projection
2257 /// sink ([`Self::materialize_final_submission`]): (a) when
2258 /// [`Self::set_output_store`] has wired a Data-plane
2259 /// [`crate::store::output::OutputStore`], the event is dual-written
2260 /// there (`producer_agent` = `TaskState.spec.agent`, resolved to its
2261 /// GH #23 canonical projection name — see below), and (b) when this
2262 /// task's spawn ran through `AgentContextMiddleware` (so
2263 /// `EngineState.agent_ctx` has a `.view.work_dir` / `.view.project_root`
2264 /// for it), the value is additionally materialized to the
2265 /// [`crate::core::projection_placement::ProjectionPlacement`]
2266 /// resolver's target (byte-compat default layout
2267 /// `<root>/workspace/tasks/<task_id>/ctx/<canonical_agent>.md`) — see
2268 /// `crate::core::projection`'s module doc.
2269 ///
2270 /// **GH #23 subtask-2 (canonical sink):** both writes above key off the
2271 /// canonical name — `Engine::step_naming_for(task_id)`'s
2272 /// `StepNaming::canonical_of_producer(producer_agent)` when a table was
2273 /// snapshotted for this task (`EngineDispatcher::with_step_naming`),
2274 /// else `producer_agent` unchanged (fail-open, byte-identical to
2275 /// pre-GH-#23 behavior — see [`crate::core::step_naming`]'s module
2276 /// doc).
2277 ///
2278 /// **Invariants** (Subtask 4): (1) this sink is fail-open — an
2279 /// unresolved root, an unconfigured `OutputStore`, or either one
2280 /// erroring, only logs a `tracing::warn!` and never turns this
2281 /// `Ok(())` into an `Err`; (2) the wired `OutputStore` stays the single
2282 /// source of truth for cross-step queries — the materialized file is a
2283 /// projection of it, not a second store; (3) core does not depend on
2284 /// `mlua-swarm-server` — everything this sink touches
2285 /// (`crate::store::output` / `crate::core::projection`) already lives
2286 /// in this crate.
2287 ///
2288 /// # `Artifact` dual-write (GH #34 subtask-3 gap fix)
2289 ///
2290 /// An `Artifact` event ALSO fans out to the Data-plane, via
2291 /// [`Self::materialize_artifact_submission`] — general-form: every
2292 /// `Artifact` submitted through this API dual-writes, no name-prefix
2293 /// gate. Unlike `Final`, the dual-write key is the artifact's own
2294 /// `name` field, verbatim — NOT resolved through the GH #23 canonical
2295 /// `StepNaming` table. An artifact's `name` IS its identity (mirrors
2296 /// [`crate::store::output::OutputStore::get_latest_by_name`]'s doc),
2297 /// so no canonicalization applies. Same fail-open discipline as
2298 /// `Final` (Invariant 1 above), but `Artifact` does NOT drive the
2299 /// file-materialize half (b) — artifact findings (e.g.
2300 /// `AfterRunAuditMiddleware`'s `"audit:<step_ref>"`) are observational
2301 /// sidecar data, not a step's own submission a work_dir/project_root
2302 /// projection needs to track. `Progress` / `Partial` events are
2303 /// unaffected — no behavior change.
2304 pub async fn submit_output(
2305 &self,
2306 token: &crate::types::CapToken,
2307 task_id: &StepId,
2308 attempt: u32,
2309 event: crate::worker::output::OutputEvent,
2310 ) -> Result<(), EngineError> {
2311 self.verify_token_for_task(token, crate::types::Verb::EmitOutput, task_id)
2312 .await?;
2313 // GH #51 — completion-time verdict-contract enforcement, embedded
2314 // choke point 2 of 2 (see `Self::verdict_contract_completion_check`'s
2315 // doc). Guarded to `Final` only — the ONLY `OutputEvent` variant a
2316 // verdict contract's completion can meaningfully address; this
2317 // guard is defensive (this function is empirically called with
2318 // `Final` only today, both from `worker.rs`'s `worker_result` and
2319 // from `operator.rs`'s WS fallback) but costs nothing and protects
2320 // against a future non-`Final` caller. Runs BEFORE the
2321 // `output_tail` write immediately below: on `Err`, this returns
2322 // immediately and the write never happens — a rejected value
2323 // never reaches `output_tail` / the flow ctx.
2324 if let crate::worker::output::OutputEvent::Final { content, ok } = &event {
2325 let comparable_value = content_ref_to_comparable_string(content.clone());
2326 self.verdict_contract_completion_check(task_id, attempt, *ok, &comparable_value)
2327 .await?;
2328 }
2329 let task_id_for_apply = task_id.clone();
2330 let event_clone = event.clone();
2331 self.with_state("submit_output", move |s| {
2332 s.output_store
2333 .entry((task_id_for_apply.clone(), attempt))
2334 .or_default()
2335 .push(event_clone.clone());
2336 s.push_event(crate::core::state::Event::WorkerOutput {
2337 task_id: task_id_for_apply,
2338 attempt,
2339 event: event_clone,
2340 });
2341 })
2342 .await?;
2343 match &event {
2344 crate::worker::output::OutputEvent::Final { content, ok } => {
2345 self.materialize_final_submission(task_id, attempt, content, *ok)
2346 .await?;
2347 }
2348 crate::worker::output::OutputEvent::Artifact { name, content } => {
2349 self.materialize_artifact_submission(task_id, attempt, name, content)
2350 .await?;
2351 }
2352 _ => {}
2353 }
2354 Ok(())
2355 }
2356
2357 /// Submit-time projection sink (subtask-4 / ST2 rework) shared by
2358 /// [`Self::submit_output`] and [`Self::submit_worker_result_trusted`].
2359 /// Best-effort / fail-open throughout (see `submit_output`'s doc
2360 /// Invariants): every failure path only `tracing::warn!`s and returns.
2361 ///
2362 /// Reads `(producer_agent, view)` via one read-only [`Self::with_state`]
2363 /// call — `producer_agent` off `TaskState.spec.agent`, `view` (the
2364 /// full [`crate::core::agent_context::AgentContextView`]) off
2365 /// `EngineState.agent_ctx[(task_id, attempt)]`, the same snapshot
2366 /// `crate::middleware::agent_context::AgentContextMiddleware` writes at
2367 /// spawn time — then does its actual (dual-write / file-write) work
2368 /// *outside* that lock, so a slow disk write or Data-plane store call
2369 /// never holds up unrelated `Engine::with_state` callers. `root` itself
2370 /// is resolved from `view` AFTER the lock via
2371 /// [`crate::core::projection_placement::ProjectionPlacement::resolve_root`]
2372 /// (GH #27, follow-up to #23) — the SAME resolver
2373 /// [`Self::step_naming_for`]'s sibling accessor
2374 /// [`Self::projection_placement_for`] snapshotted at dispatch time, so
2375 /// this sink's root-preference / fallback order is identical to the
2376 /// server read-back and the spawn-time pointer.
2377 async fn materialize_final_submission(
2378 &self,
2379 task_id: &StepId,
2380 attempt: u32,
2381 content: &crate::worker::output::ContentRef,
2382 ok: bool,
2383 ) -> Result<(), EngineError> {
2384 let server_policy = self.cfg().check_policy;
2385 let task_id_for_lookup = task_id.clone();
2386 let lookup = self
2387 .with_state("materialize_final_submission.lookup", move |s| {
2388 let entry = s.tasks.get(&task_id_for_lookup);
2389 let producer_agent = entry.map(|t| t.spec.agent.clone());
2390 let task_policy = entry.and_then(|t| t.spec.check_policy);
2391 let view = s
2392 .agent_ctx
2393 .get(&(task_id_for_lookup.clone(), attempt))
2394 .map(|e| e.view.clone());
2395 (producer_agent, task_policy, view)
2396 })
2397 .await;
2398 // Per-task `TaskSpec.check_policy` (ST1c) wins
2399 // over the server-wide `EngineCfg.check_policy` when set — a
2400 // per-run override forwarded from the launch entry point (see
2401 // `TaskLaunchRequest.check_policy` /
2402 // `TaskLaunchInput.check_policy`). `None` leaves the server
2403 // default in effect (backward compat).
2404 let policy = lookup
2405 .as_ref()
2406 .ok()
2407 .and_then(|(_, tp, _)| *tp)
2408 .unwrap_or(server_policy);
2409 let (producer_agent, view) = match lookup.map(|(pa, _, view)| (pa, view)) {
2410 Ok(pair) => pair,
2411 Err(err) => {
2412 if !matches!(policy, crate::core::config::CheckPolicy::Silent) {
2413 tracing::warn!(
2414 %task_id,
2415 error = %err,
2416 "submit-time projection sink: state lookup failed; skipping (fail-open)"
2417 );
2418 }
2419 apply_check_policy(
2420 policy,
2421 "submit-time projection sink: state lookup",
2422 "state lookup failed; skipping (fail-open)",
2423 )?;
2424 return Ok(());
2425 }
2426 };
2427 let Some(producer_agent) = producer_agent else {
2428 // Defensive only: `task_id` is always a just-looked-up task at
2429 // every real call site. No task, no addressable producer name
2430 // — nothing to project. Not gated by `CheckPolicy` — a missing
2431 // task is an intentional early-exit path, not a fail-open
2432 // condition to surface.
2433 return Ok(());
2434 };
2435 let placement = self
2436 .projection_placement_for(task_id)
2437 .await
2438 .unwrap_or_default();
2439 let root = view.and_then(|v| placement.resolve_root(&v));
2440
2441 // GH #23 subtask-2: resolve `producer_agent` to its canonical
2442 // projection name via the Blueprint-wide `StepNaming` table
2443 // snapshotted at dispatch time (`Engine::step_naming_for`). Both
2444 // write paths below ((a) data-plane, (b) file stem) use the
2445 // *canonical* name — `StepNaming::canonical_of_producer` returns
2446 // `producer_agent` unchanged for undeclared steps (byte-identical
2447 // to pre-GH-#23 behavior), and `None` (no table for this
2448 // `task_id`, e.g. a spawn that never went through
2449 // `EngineDispatcher::with_step_naming`) is a defensive fail-open
2450 // to the raw `producer_agent`, same discipline as the rest of this
2451 // sink.
2452 let canonical_agent = self
2453 .step_naming_for(task_id)
2454 .await
2455 .and_then(|naming| {
2456 naming
2457 .canonical_of_producer(&producer_agent)
2458 .map(str::to_string)
2459 })
2460 .unwrap_or_else(|| producer_agent.clone());
2461
2462 // (a) Data-plane dual-write, when an OutputStore backend is wired.
2463 if let Some(store) = self.output_store_backend() {
2464 if let Err(err) = store
2465 .append(
2466 task_id.as_str(),
2467 attempt,
2468 &canonical_agent,
2469 crate::worker::output::OutputEvent::Final {
2470 content: content.clone(),
2471 ok,
2472 },
2473 Vec::new(),
2474 )
2475 .await
2476 {
2477 if !matches!(policy, crate::core::config::CheckPolicy::Silent) {
2478 tracing::warn!(
2479 %task_id,
2480 agent = %producer_agent,
2481 canonical = %canonical_agent,
2482 error = %err,
2483 "submit-time projection sink: OutputStore dual-write failed (fail-open)"
2484 );
2485 }
2486 apply_check_policy(
2487 policy,
2488 "submit-time projection sink: OutputStore dual-write",
2489 "OutputStore dual-write failed (fail-open)",
2490 )?;
2491 }
2492 }
2493
2494 // (b) File materialize, when a root resolved.
2495 let Some(root) = root else {
2496 if !matches!(policy, crate::core::config::CheckPolicy::Silent) {
2497 tracing::warn!(
2498 %task_id,
2499 agent = %producer_agent,
2500 canonical = %canonical_agent,
2501 "submit-time projection sink: no work_dir/project_root resolved; skipping file materialize (fail-open)"
2502 );
2503 }
2504 apply_check_policy(
2505 policy,
2506 "submit-time projection sink: file materialize",
2507 "no work_dir/project_root resolved; skipping file materialize (fail-open)",
2508 )?;
2509 return Ok(());
2510 };
2511 let value = match content {
2512 crate::worker::output::ContentRef::Inline { value } => value.clone(),
2513 crate::worker::output::ContentRef::FileRef {
2514 path,
2515 mime,
2516 size_hint,
2517 } => serde_json::json!({
2518 "file_ref": path.to_string_lossy(),
2519 "mime": mime,
2520 "size_hint": size_hint,
2521 }),
2522 };
2523 let key = crate::core::projection::ProjectionKey {
2524 task_id: task_id.to_string(),
2525 run_id: None,
2526 step: Some(canonical_agent.clone()),
2527 path: None,
2528 };
2529 let adapter = crate::core::projection::FileProjectionAdapter::with_placement(
2530 root,
2531 (*placement).clone(),
2532 );
2533 if let Err(err) = adapter.materialize_submission(&key, &value, attempt, ok) {
2534 if !matches!(policy, crate::core::config::CheckPolicy::Silent) {
2535 tracing::warn!(
2536 %task_id,
2537 agent = %producer_agent,
2538 canonical = %canonical_agent,
2539 error = %err,
2540 "submit-time projection sink: file materialize failed (fail-open)"
2541 );
2542 }
2543 apply_check_policy(
2544 policy,
2545 "submit-time projection sink: file materialize",
2546 "file materialize failed (fail-open)",
2547 )?;
2548 }
2549 Ok(())
2550 }
2551
2552 /// Submit-time projection sink for `OutputEvent::Artifact` (GH #34
2553 /// subtask-3, later extended to drive the file half too). Two halves, the
2554 /// [`Self::materialize_final_submission`] mirror for staged named parts:
2555 ///
2556 /// - **Data-plane dual-write** — when [`Self::set_output_store`] has
2557 /// wired a [`crate::store::output::OutputStore`], the artifact
2558 /// dual-writes there under its own `name`, verbatim (general form:
2559 /// every `Artifact` staged via [`Self::submit_output`] /
2560 /// [`Self::stage_worker_artifact_trusted`] materializes this way, no
2561 /// name-prefix gate).
2562 /// - **File materialize** — when a `root` resolves off the spawn-time
2563 /// [`crate::core::agent_context::AgentContextView`], the part's
2564 /// content is written raw to `<ctx-dir>/<name>` via
2565 /// [`crate::core::projection::FileProjectionAdapter::materialize_part`].
2566 /// That file is the IN file the *next* Agent step reads: materializing
2567 /// a Step's OUTPUT to disk is the
2568 /// [`crate::core::projection::FileProjectionAdapter`]'s
2569 /// responsibility, and a staged named part is as much an OUTPUT the
2570 /// next step consumes as a `Final` is — so the sink materializes it
2571 /// too, rather than leaving parts Data-plane-only.
2572 ///
2573 /// Unlike the Final sink, no `StepNaming` canonicalization is applied:
2574 /// an artifact's `name` already IS the key both halves address (it
2575 /// names the file directly, extension included — `plan.md` — so
2576 /// `materialize_part` writes it verbatim, not through the `<stem>.md`
2577 /// synthesis the Final sink's canonical-agent path uses).
2578 ///
2579 /// Fail-open throughout, the same `check_policy` cascade as
2580 /// [`Self::materialize_final_submission`]: a per-task lookup error falls
2581 /// back to the server default (and a `None` view ⇒ the file half's
2582 /// unresolved-root path), an unconfigured `OutputStore` skips the
2583 /// dual-write, an unresolved root skips the file half, and a
2584 /// dual-write / file-write / name-guard error only `tracing::warn!`s
2585 /// (`Silent` suppresses even that) before applying [`apply_check_policy`]
2586 /// (`Strict` surfaces an [`EngineError`], `Warn` / `Silent` return
2587 /// `Ok(())`) — a staged part never turns a would-have-succeeded submit
2588 /// into a failure under the default policy.
2589 async fn materialize_artifact_submission(
2590 &self,
2591 task_id: &StepId,
2592 attempt: u32,
2593 name: &str,
2594 content: &crate::worker::output::ContentRef,
2595 ) -> Result<(), EngineError> {
2596 // Per-task `TaskSpec.check_policy` override + the `AgentContextView`
2597 // snapshot, resolved in ONE read-only `with_state` (the same lock
2598 // the policy lookup already needed — no extra `with_state` for the
2599 // view). Silent per-task lookup failure (`with_state` error) falls
2600 // back to the server-wide default and a `None` view (⇒ the file
2601 // half's own unresolved-root fail-open path); this sink never
2602 // surfaces the lookup error itself as a step failure.
2603 let server_policy = self.cfg().check_policy;
2604 let task_id_for_lookup = task_id.clone();
2605 let lookup = self
2606 .with_state("materialize_artifact_submission.lookup", move |s| {
2607 let task_policy = s
2608 .tasks
2609 .get(&task_id_for_lookup)
2610 .and_then(|t| t.spec.check_policy);
2611 let view = s
2612 .agent_ctx
2613 .get(&(task_id_for_lookup.clone(), attempt))
2614 .map(|e| e.view.clone());
2615 (task_policy, view)
2616 })
2617 .await
2618 .ok();
2619 let policy = lookup
2620 .as_ref()
2621 .and_then(|(tp, _)| *tp)
2622 .unwrap_or(server_policy);
2623 let view = lookup.and_then(|(_, view)| view);
2624
2625 // (a) Data-plane dual-write, when an OutputStore backend is wired —
2626 // the artifact's own `name` is its Data-plane key (no
2627 // canonicalization, unlike the Final sink's `StepNaming`
2628 // resolution).
2629 if let Some(store) = self.output_store_backend() {
2630 if let Err(err) = store
2631 .append(
2632 task_id.as_str(),
2633 attempt,
2634 name,
2635 crate::worker::output::OutputEvent::Artifact {
2636 name: name.to_string(),
2637 content: content.clone(),
2638 },
2639 Vec::new(),
2640 )
2641 .await
2642 {
2643 if !matches!(policy, crate::core::config::CheckPolicy::Silent) {
2644 tracing::warn!(
2645 %task_id,
2646 artifact = %name,
2647 error = %err,
2648 "submit-time projection sink: OutputStore dual-write failed for Artifact (fail-open)"
2649 );
2650 }
2651 apply_check_policy(
2652 policy,
2653 "submit-time projection sink: Artifact OutputStore dual-write",
2654 "OutputStore dual-write failed for Artifact (fail-open)",
2655 )?;
2656 }
2657 }
2658
2659 // (b) File materialize, when a root resolved — writes the staged
2660 // part raw to `<ctx-dir>/<name>`, the IN file the next Agent step
2661 // reads (see `FileProjectionAdapter::materialize_part`'s doc for
2662 // why raw / why the name is verbatim). A name-guard violation lands
2663 // on the same fail-open path as any other write error below.
2664 let placement = self
2665 .projection_placement_for(task_id)
2666 .await
2667 .unwrap_or_default();
2668 let Some(root) = view.and_then(|v| placement.resolve_root(&v)) else {
2669 if !matches!(policy, crate::core::config::CheckPolicy::Silent) {
2670 tracing::warn!(
2671 %task_id,
2672 artifact = %name,
2673 "submit-time projection sink: no work_dir/project_root resolved; skipping part file materialize (fail-open)"
2674 );
2675 }
2676 apply_check_policy(
2677 policy,
2678 "submit-time projection sink: part file materialize",
2679 "no work_dir/project_root resolved; skipping part file materialize (fail-open)",
2680 )?;
2681 return Ok(());
2682 };
2683 let value = match content {
2684 crate::worker::output::ContentRef::Inline { value } => value.clone(),
2685 crate::worker::output::ContentRef::FileRef {
2686 path,
2687 mime,
2688 size_hint,
2689 } => serde_json::json!({
2690 "file_ref": path.to_string_lossy(),
2691 "mime": mime,
2692 "size_hint": size_hint,
2693 }),
2694 };
2695 let adapter = crate::core::projection::FileProjectionAdapter::with_placement(
2696 root,
2697 (*placement).clone(),
2698 );
2699 if let Err(err) = adapter.materialize_part(task_id.as_str(), name, &value) {
2700 if !matches!(policy, crate::core::config::CheckPolicy::Silent) {
2701 tracing::warn!(
2702 %task_id,
2703 artifact = %name,
2704 error = %err,
2705 "submit-time projection sink: part file materialize failed (fail-open)"
2706 );
2707 }
2708 apply_check_policy(
2709 policy,
2710 "submit-time projection sink: part file materialize",
2711 "part file materialize failed (fail-open)",
2712 )?;
2713 }
2714 Ok(())
2715 }
2716
2717 /// Snapshot the entire output tail for a given `(task_id, attempt)`.
2718 /// Used by the dispatch path when pulling `Final`, and by observers
2719 /// reading the trace.
2720 pub async fn output_tail(
2721 &self,
2722 task_id: &StepId,
2723 attempt: u32,
2724 ) -> Vec<crate::worker::output::OutputEvent> {
2725 let key = (task_id.clone(), attempt);
2726 self.with_state("output_tail", move |s| {
2727 s.output_store.get(&key).cloned().unwrap_or_default()
2728 })
2729 .await
2730 .unwrap_or_default()
2731 }
2732
2733 /// Record an interim `last_result` for `task_id` without changing its
2734 /// `status`. Distinct from the terminal `Final` output event handled
2735 /// through `submit_output` / `dispatch_attempt_with`.
2736 pub async fn post_result(
2737 &self,
2738 token: &CapToken,
2739 task_id: &StepId,
2740 result: Value,
2741 ) -> Result<(), EngineError> {
2742 self.verify_token_for_task(token, Verb::PostResult, task_id)
2743 .await?;
2744 let task_id = task_id.clone();
2745 let result_clone = result.clone();
2746 self.with_state("post_result", move |s| {
2747 let task = s
2748 .tasks
2749 .get_mut(&task_id)
2750 .ok_or_else(|| EngineError::TaskNotFound(task_id.to_string()))?;
2751 task.last_result = Some(result_clone);
2752 task.updated_at = now_unix();
2753 Ok::<(), EngineError>(())
2754 })
2755 .await??;
2756 Ok(())
2757 }
2758
2759 /// Store a named resource value, retrievable later via `fetch_data`.
2760 /// No token is required — this is a server-side/admin-style setter
2761 /// (mirrors `bake_worker_system_prompt`).
2762 pub async fn set_resource(
2763 &self,
2764 key: impl Into<String>,
2765 value: Value,
2766 ) -> Result<(), EngineError> {
2767 let key = key.into();
2768 self.with_state("set_resource", move |s| {
2769 s.resources.insert(key, value);
2770 })
2771 .await?;
2772 Ok(())
2773 }
2774
2775 // ═══════════════════════════════════════════════════════════════════════
2776 // Senior suspend / resume
2777 // ═══════════════════════════════════════════════════════════════════════
2778
2779 /// Ask a question of the Senior, mark the task `Suspended`, and
2780 /// return a `ResumeKey`. The suspended state persists until another
2781 /// task calls `resume(key, answer)`.
2782 ///
2783 /// Resume-side waiting is `Notify`-based, so a caller (typically
2784 /// MainAI) can detach, reattach from a different process, and still
2785 /// pull the answer out via `await_resume(key, timeout)` — the answer
2786 /// is stored inside `EngineState`.
2787 pub async fn query_senior(
2788 &self,
2789 token: &CapToken,
2790 task_id: &StepId,
2791 question: Value,
2792 ) -> Result<ResumeKey, EngineError> {
2793 self.verify_token(token, Verb::QuerySenior).await?;
2794 let task_id = task_id.clone();
2795 let key = ResumeKey::for_senior(&task_id);
2796 let task_notify = self
2797 .with_state("query_senior.notify_ensure", |s| {
2798 s.ensure_task_notify(&task_id)
2799 })
2800 .await?;
2801
2802 let key_clone = key.clone();
2803 let task_id_inner = task_id.clone();
2804 let question_clone = question.clone();
2805 self.with_state("query_senior.suspend", move |s| {
2806 let task = s
2807 .tasks
2808 .get_mut(&task_id_inner)
2809 .ok_or_else(|| EngineError::TaskNotFound(task_id_inner.to_string()))?;
2810 task.status = TaskStatus::Suspended;
2811 task.suspended_on = Some(key_clone.clone());
2812 task.updated_at = now_unix();
2813 s.pending_resumes
2814 .insert(key_clone.clone(), ResumePending::new());
2815 s.push_event(Event::SeniorQueried {
2816 task_id: task_id_inner.clone(),
2817 question: question_clone.clone(),
2818 });
2819 s.push_event(Event::TaskSuspended {
2820 task_id: task_id_inner.clone(),
2821 key: key_clone.clone(),
2822 });
2823 Ok::<(), EngineError>(())
2824 })
2825 .await??;
2826
2827 // Notify callers waiting for a task status change (Running → Suspended).
2828 task_notify.notify_waiters();
2829
2830 let _ = self
2831 .inner
2832 .event_tx
2833 .send(Event::SeniorQueried { task_id, question });
2834 Ok(key)
2835 }
2836
2837 /// Store the answer for a `ResumeKey` in `EngineState` and wake the
2838 /// waiting caller via `Notify`. Also flips the suspended task's
2839 /// status back to `Running` and fires the per-task notifier.
2840 pub async fn resume(&self, key: ResumeKey, answer: Value) -> Result<(), EngineError> {
2841 let answer_for_state = answer.clone();
2842 let answer_for_event = answer.clone();
2843 let key_clone = key.clone();
2844 let (notify, task_notify, task_id_opt) = self
2845 .with_state("resume.set", move |s| {
2846 let pending = s
2847 .pending_resumes
2848 .get_mut(&key_clone)
2849 .ok_or(EngineError::ResumeKeyNotFound)?;
2850 pending.answer = Some(answer_for_state);
2851 let notify = pending.notify.clone();
2852
2853 let task_id = s
2854 .tasks
2855 .iter()
2856 .find(|(_, t)| t.suspended_on.as_ref() == Some(&key_clone))
2857 .map(|(id, _)| id.clone());
2858
2859 let task_notify = task_id.as_ref().map(|tid| s.ensure_task_notify(tid));
2860
2861 if let Some(tid) = &task_id {
2862 if let Some(task) = s.tasks.get_mut(tid) {
2863 task.suspended_on = None;
2864 task.status = TaskStatus::Running;
2865 task.updated_at = now_unix();
2866 }
2867 s.push_event(Event::TaskResumed {
2868 task_id: tid.clone(),
2869 key: key_clone.clone(),
2870 });
2871 s.push_event(Event::SeniorAnswered {
2872 task_id: tid.clone(),
2873 answer: answer_for_event.clone(),
2874 });
2875 }
2876 Ok::<_, EngineError>((notify, task_notify, task_id))
2877 })
2878 .await??;
2879
2880 // Outside the lock: notify_waiters for both the ResumePending and task-status waits.
2881 notify.notify_waiters();
2882 if let Some(n) = task_notify {
2883 n.notify_waiters();
2884 }
2885
2886 if let Some(tid) = task_id_opt {
2887 let _ = self
2888 .inner
2889 .event_tx
2890 .send(Event::TaskResumed { task_id: tid, key });
2891 }
2892 Ok(())
2893 }
2894
2895 /// Wait for the resume answer. Even if the caller (an Operator)
2896 /// detached and reattached, the answer is available immediately here
2897 /// — if it was already stored, this returns without waiting on the
2898 /// notifier.
2899 ///
2900 /// `timeout = Duration::ZERO` performs an instant check without
2901 /// waiting.
2902 pub async fn await_resume(
2903 &self,
2904 key: ResumeKey,
2905 timeout: Duration,
2906 ) -> Result<Value, EngineError> {
2907 // (1) Under the lock: clone the notify handle and check for an existing answer.
2908 let key_clone = key.clone();
2909 let (notify, existing) = self
2910 .with_state("await_resume.snapshot", move |s| {
2911 let pending = s
2912 .pending_resumes
2913 .get(&key_clone)
2914 .ok_or(EngineError::ResumeKeyNotFound)?;
2915 Ok::<_, EngineError>((pending.notify.clone(), pending.answer.clone()))
2916 })
2917 .await??;
2918
2919 // (2) If an answer has already been stored, return immediately (detach / reattach pattern).
2920 if let Some(v) = existing {
2921 return Ok(v);
2922 }
2923
2924 // (3) Outside the lock: wait on the notify with a timeout.
2925 if timeout.is_zero() {
2926 return Err(EngineError::PollTimeout);
2927 }
2928 let waited = tokio::time::timeout(timeout, notify.notified()).await;
2929 if waited.is_err() {
2930 return Err(EngineError::PollTimeout);
2931 }
2932
2933 // (4) Under the lock: re-read the answer (should be present now that we were notified).
2934 let key_clone = key.clone();
2935 self.with_state("await_resume.read", move |s| {
2936 let pending = s
2937 .pending_resumes
2938 .get(&key_clone)
2939 .ok_or(EngineError::ResumeKeyNotFound)?;
2940 pending
2941 .answer
2942 .clone()
2943 .ok_or_else(|| EngineError::Internal("notified but answer missing".into()))
2944 })
2945 .await?
2946 }
2947
2948 // ═══════════════════════════════════════════════════════════════════════
2949 // poll_task — the "wait" path that waits for task-status changes (works for long-poll and regular wait).
2950 // ═══════════════════════════════════════════════════════════════════════
2951
2952 /// Wait until the task's status **transitions to terminal or
2953 /// `Suspended`**, then return the latest `TaskState`. Returns
2954 /// immediately if the task is already in a terminal state.
2955 /// Exceeding the timeout returns `EngineError::PollTimeout`.
2956 ///
2957 /// A `hold` of `Duration::from_secs(0)` returns a snapshot immediately
2958 /// (no wait). Larger holds — tens of minutes up to days — are fine;
2959 /// the wait state is kept in memory inside the engine and does not
2960 /// degrade.
2961 pub async fn poll_task(
2962 &self,
2963 token: &CapToken,
2964 task_id: &StepId,
2965 hold: Duration,
2966 ) -> Result<TaskState, EngineError> {
2967 self.verify_token_for_task(token, Verb::PollTask, task_id)
2968 .await?;
2969 let task_id_inner = task_id.clone();
2970
2971 // (1) Under the lock: take a snapshot and clone task_notify.
2972 let (state, notify) = self
2973 .with_state("poll_task.snapshot", move |s| {
2974 let task = s
2975 .tasks
2976 .get(&task_id_inner)
2977 .cloned()
2978 .ok_or_else(|| EngineError::TaskNotFound(task_id_inner.to_string()))?;
2979 let notify = s.ensure_task_notify(&task_id_inner);
2980 Ok::<_, EngineError>((task, notify))
2981 })
2982 .await??;
2983
2984 // (2) Immediate-return condition: already terminal / Suspended (nothing left to wait on).
2985 if matches!(
2986 state.status,
2987 TaskStatus::Pass | TaskStatus::Blocked | TaskStatus::Cancelled | TaskStatus::Suspended
2988 ) {
2989 return Ok(state);
2990 }
2991 if hold.is_zero() {
2992 return Ok(state);
2993 }
2994
2995 // (3) Outside the lock: wait on Notify with a timeout.
2996 let waited = tokio::time::timeout(hold, notify.notified()).await;
2997 if waited.is_err() {
2998 return Err(EngineError::PollTimeout);
2999 }
3000
3001 // (4) Under the lock: take a fresh snapshot.
3002 let task_id_inner = task_id.clone();
3003 self.with_state("poll_task.reread", move |s| {
3004 s.tasks
3005 .get(&task_id_inner)
3006 .cloned()
3007 .ok_or_else(|| EngineError::TaskNotFound(task_id_inner.to_string()))
3008 })
3009 .await?
3010 }
3011
3012 // ═══════════════════════════════════════════════════════════════════════
3013 // Background: heartbeat miss → detach loop
3014 // ═══════════════════════════════════════════════════════════════════════
3015
3016 /// Background loop that scans sessions every `heartbeat_interval` and
3017 /// flips `attached = false` on any session whose `last_seen` exceeds
3018 /// `heartbeat_miss_threshold * interval`.
3019 ///
3020 /// The tasks themselves are kept (assuming
3021 /// `keepalive_on_idle = true`), so another client can reattach with
3022 /// the same token and resume immediately. Dropping the returned
3023 /// `JoinHandle` does not stop the loop — the handle exists so callers
3024 /// who want to abort can hold onto it.
3025 pub fn start_detach_loop(&self) -> tokio::task::JoinHandle<()> {
3026 let engine = self.clone();
3027 let cfg = self.inner.cfg.long_hold.clone();
3028 let interval = cfg.heartbeat_interval;
3029 let miss_secs = cfg.heartbeat_interval.as_secs() * cfg.heartbeat_miss_threshold as u64;
3030
3031 tokio::spawn(async move {
3032 let mut ticker = tokio::time::interval(interval);
3033 ticker.tick().await; // first tick is immediate
3034 loop {
3035 ticker.tick().await;
3036 let now = now_unix();
3037 let detached = engine
3038 .with_state("detach_loop.scan", |s| {
3039 let mut detached = Vec::new();
3040 for (sid, sess) in s.sessions.iter_mut() {
3041 if !sess.attached {
3042 continue;
3043 }
3044 if now.saturating_sub(sess.last_seen) >= miss_secs {
3045 sess.attached = false;
3046 detached.push(sid.clone());
3047 }
3048 }
3049 for sid in &detached {
3050 s.push_event(Event::SessionDetached {
3051 session_id: sid.clone(),
3052 });
3053 }
3054 detached
3055 })
3056 .await
3057 .unwrap_or_default();
3058 for sid in detached {
3059 let _ = engine
3060 .inner
3061 .event_tx
3062 .send(Event::SessionDetached { session_id: sid });
3063 }
3064 }
3065 })
3066 }
3067
3068 /// Helper: wake a task whose status has changed. Called from the
3069 /// method body outside the lock.
3070 async fn wake_task(&self, task_id: &StepId) -> Result<(), EngineError> {
3071 let task_id = task_id.clone();
3072 let notify_opt = self
3073 .with_state("wake_task.get_notify", move |s| {
3074 s.task_notifies.get(&task_id).cloned()
3075 })
3076 .await?;
3077 if let Some(n) = notify_opt {
3078 n.notify_waiters();
3079 }
3080 Ok(())
3081 }
3082}
3083
3084/// Decide what a submit-time projection sink should do at a fail-open
3085/// branch given the configured [`crate::core::config::CheckPolicy`].
3086///
3087/// Returns `Ok(())` under [`CheckPolicy::Silent`] and
3088/// [`CheckPolicy::Warn`] — the caller continues with fail-open. Returns
3089/// [`EngineError::CheckPolicyStrict`] under [`CheckPolicy::Strict`],
3090/// carrying the caller-supplied `context` (call-site identifier) and
3091/// `message` (the pre-existing warn-log message literal, preserved
3092/// verbatim for log parse compatibility).
3093///
3094/// This helper deliberately does **not** call `tracing::warn!` itself —
3095/// the caller is responsible for firing the existing warn! (with its
3096/// full structured-field payload — `%task_id`, `agent`, `canonical`,
3097/// `error`, etc.) under `Warn` mode, and for skipping the warn! under
3098/// `Silent` mode. Keeping the warn! at the call site preserves the
3099/// exact structured-field shape every existing log-parse consumer sees;
3100/// forwarding it through the helper would either drop those fields or
3101/// require a macro (deferred, see subtask-1b).
3102///
3103/// Design intent: the fail-open discipline of every submit-time
3104/// projection sink is byte-identical to the pre-`CheckPolicy` behaviour
3105/// under the default [`CheckPolicy::Warn`]. `Silent` is a per-run opt-in
3106/// to suppress noise (e.g., a caller that has already verified upstream
3107/// invariants); `Strict` is a per-run opt-in to fail loudly (e.g., a
3108/// caller that requires all parts to materialize). See
3109/// [`crate::core::config::CheckPolicy`] for the "state dirty on fail"
3110/// semantics of `Strict`.
3111pub(crate) fn apply_check_policy(
3112 policy: crate::core::config::CheckPolicy,
3113 context: &str,
3114 message: &str,
3115) -> Result<(), EngineError> {
3116 match policy {
3117 crate::core::config::CheckPolicy::Silent | crate::core::config::CheckPolicy::Warn => Ok(()),
3118 crate::core::config::CheckPolicy::Strict => Err(EngineError::CheckPolicyStrict {
3119 context: context.to_string(),
3120 message: message.to_string(),
3121 }),
3122 }
3123}
3124
3125#[cfg(test)]
3126mod check_policy_helper_tests {
3127 use super::apply_check_policy;
3128 use crate::core::config::CheckPolicy;
3129 use crate::core::errors::EngineError;
3130
3131 /// `Silent` returns `Ok(())` without producing an error. Log
3132 /// suppression (the "no `tracing::warn!`" half of the semantics) is
3133 /// enforced at the call site, not inside the helper — see the
3134 /// helper's doc comment for why.
3135 #[test]
3136 fn silent_returns_ok() {
3137 let result = apply_check_policy(CheckPolicy::Silent, "call/site", "sink message");
3138 assert!(matches!(result, Ok(())));
3139 }
3140
3141 /// `Warn` (the default) returns `Ok(())` — the caller continues
3142 /// with fail-open, having already fired its own `tracing::warn!`
3143 /// with the full structured-field payload.
3144 #[test]
3145 fn warn_returns_ok() {
3146 let result = apply_check_policy(CheckPolicy::Warn, "call/site", "sink message");
3147 assert!(matches!(result, Ok(())));
3148 }
3149
3150 /// `Strict` returns
3151 /// [`EngineError::CheckPolicyStrict`] with `context` and `message`
3152 /// copied verbatim from the caller — the completion route surfaces
3153 /// this as a step / launch error so a caller that has opted in can
3154 /// fail fast instead of proceeding with a partially-realized
3155 /// submission.
3156 #[test]
3157 fn strict_returns_error_with_context_and_message() {
3158 let result = apply_check_policy(
3159 CheckPolicy::Strict,
3160 "submit-time projection sink: file materialize",
3161 "no work_dir/project_root resolved; skipping file materialize (fail-open)",
3162 );
3163 match result {
3164 Err(EngineError::CheckPolicyStrict { context, message }) => {
3165 assert_eq!(context, "submit-time projection sink: file materialize");
3166 assert_eq!(
3167 message,
3168 "no work_dir/project_root resolved; skipping file materialize (fail-open)"
3169 );
3170 }
3171 other => panic!("expected CheckPolicyStrict, got {:?}", other),
3172 }
3173 }
3174}
3175
3176// ─── UT: issue #14 — token store keyed by fingerprint, not nonce ────────────
3177#[cfg(test)]
3178mod token_fingerprint_store_tests {
3179 use super::*;
3180
3181 /// A token that was never attached fails verify with a `TokenNotFound`
3182 /// that carries the fingerprint — never the nonce. The error string can
3183 /// surface in HTTP error bodies, so this is the secret-hygiene contract.
3184 #[tokio::test]
3185 async fn verify_unknown_token_reports_fingerprint_not_nonce() {
3186 let engine = Engine::new(EngineCfg::default());
3187 // Signed by the engine's own signer (sig passes) but never inserted
3188 // into the store — verify must fail at step (4), the store lookup.
3189 let token = engine.signer().session(
3190 "ghost",
3191 Role::Operator,
3192 vec!["*".into()],
3193 Duration::from_secs(60),
3194 );
3195 let err = engine
3196 .verify_token(&token, Verb::ReadTaskState)
3197 .await
3198 .expect_err("token is not in the store");
3199 let msg = err.to_string();
3200 assert!(
3201 msg.contains(&token.fingerprint()),
3202 "error must carry the fingerprint: {msg}"
3203 );
3204 assert!(
3205 !msg.contains(&token.nonce),
3206 "error must not leak the nonce: {msg}"
3207 );
3208 }
3209
3210 /// attach → verify → heartbeat → detach all resolve the session /
3211 /// token record through fingerprint keys (mint/verify lifecycle
3212 /// regression guard for the issue #14 key migration).
3213 #[tokio::test]
3214 async fn attach_verify_heartbeat_detach_cycle_with_fp_keying() {
3215 let engine = Engine::new(EngineCfg::default());
3216 let token = engine
3217 .attach("op-1", Role::Operator, Duration::from_secs(60))
3218 .await
3219 .expect("attach");
3220 engine
3221 .verify_token(&token, Verb::ReadTaskState)
3222 .await
3223 .expect("verify consumes via fp key");
3224 engine
3225 .heartbeat(&token)
3226 .await
3227 .expect("heartbeat finds the session by fp");
3228 engine
3229 .detach(&token)
3230 .await
3231 .expect("detach finds the session by fp");
3232 }
3233}
3234
3235// ─── UT: `OperatorKind` "Runtime Global" tier — `Option` semantics ─────────
3236//
3237// Regression coverage for the "explicit Automate is indistinguishable from
3238// unspecified" defect: `OperatorSession.operator_kind` (and the
3239// `attach_with_ids` `kind` parameter it stores) is `Option<OperatorKind>`,
3240// so `Some(Automate)` is an explicit Runtime Global request that must
3241// outrank `bp_global`, while `None` must let `bp_global` decide. Exercises
3242// the real `resolve_operator_info` cascade path (not just
3243// `collapse_operator_kind` in isolation), attaching via `attach_with_ids`
3244// exactly as `TaskLaunchService::launch` does.
3245#[cfg(test)]
3246mod resolve_operator_info_runtime_global_tests {
3247 use super::*;
3248
3249 async fn attach_and_resolve(
3250 runtime_global: Option<OperatorKind>,
3251 bp_global: Option<OperatorKind>,
3252 ) -> OperatorInfo {
3253 let engine = Engine::new(EngineCfg::default());
3254 let token = engine
3255 .attach_with_ids(
3256 "ut-op",
3257 Role::Operator,
3258 Duration::from_secs(30),
3259 runtime_global,
3260 None,
3261 None,
3262 None,
3263 HashMap::new(),
3264 HashMap::new(),
3265 bp_global,
3266 )
3267 .await
3268 .expect("attach_with_ids ok");
3269 let session = engine
3270 .with_state("test.find_session", |s| {
3271 s.sessions
3272 .values()
3273 .find(|sess| sess.token_fp == token.fingerprint())
3274 .cloned()
3275 })
3276 .await
3277 .expect("with_state ok")
3278 .expect("session present after attach_with_ids");
3279 engine.resolve_operator_info(&session, "agent-x").await
3280 }
3281
3282 #[tokio::test]
3283 async fn explicit_some_automate_outranks_bp_global_main_ai() {
3284 // Runtime Global explicitly requests Automate; bp_global is MainAi.
3285 // The explicit `Some(Automate)` must win — this is exactly the case
3286 // the old `== OperatorKind::default()` convention got wrong (it
3287 // could not tell "explicitly Automate" from "unspecified" and would
3288 // have let `bp_global` (MainAi) take over instead).
3289 let info =
3290 attach_and_resolve(Some(OperatorKind::Automate), Some(OperatorKind::MainAi)).await;
3291 assert_eq!(
3292 info.kind,
3293 OperatorKind::Automate,
3294 "explicit Some(Automate) runtime_global must outrank bp_global MainAi"
3295 );
3296 }
3297
3298 #[tokio::test]
3299 async fn none_lets_bp_global_main_ai_win() {
3300 // Runtime Global left unspecified (`None`); bp_global is MainAi.
3301 // With nothing more specific set, `bp_global` must decide.
3302 let info = attach_and_resolve(None, Some(OperatorKind::MainAi)).await;
3303 assert_eq!(
3304 info.kind,
3305 OperatorKind::MainAi,
3306 "None runtime_global must let bp_global MainAi win"
3307 );
3308 }
3309}
3310
3311/// issue #13 run_id propagation: `dispatch_attempt_with`'s `run_id` param
3312/// must land in `Ctx.meta.runtime["run_id"]` (the same slot pattern as the
3313/// pre-existing `worker_handle`), or be omitted entirely when `None`. Same
3314/// `CtxProbe` shape as `middleware::worker_binding`'s test module — an
3315/// inner `SpawnerAdapter` that snapshots the `Ctx` it was called with and
3316/// fails the spawn (only the ctx snapshot matters here).
3317#[cfg(test)]
3318mod dispatch_attempt_with_run_id_tests {
3319 use super::*;
3320 use crate::worker::adapter::{SpawnError, SpawnerAdapter};
3321 use crate::worker::Worker;
3322 use std::sync::Mutex as StdMutex;
3323
3324 struct CtxProbe {
3325 seen: Arc<StdMutex<Option<Ctx>>>,
3326 }
3327
3328 #[async_trait::async_trait]
3329 impl SpawnerAdapter for CtxProbe {
3330 async fn spawn(
3331 &self,
3332 _engine: &Engine,
3333 ctx: &Ctx,
3334 _task_id: StepId,
3335 _attempt: u32,
3336 _token: CapToken,
3337 ) -> Result<Box<dyn Worker>, SpawnError> {
3338 *self.seen.lock().unwrap() = Some(ctx.clone());
3339 Err(SpawnError::Internal("probe stop".into()))
3340 }
3341 }
3342
3343 async fn dispatch_with_probe(run_id: Option<&RunId>) -> Ctx {
3344 let engine = Engine::new(EngineCfg::default());
3345 let token = engine
3346 .attach("ut-op", Role::Operator, Duration::from_secs(30))
3347 .await
3348 .expect("attach");
3349 let tid = engine
3350 .start_task(
3351 &token,
3352 TaskSpec {
3353 agent: "probe".into(),
3354 initial_directive: "hi".into(),
3355 step_ctx: None,
3356 check_policy: None,
3357 },
3358 )
3359 .await
3360 .expect("start_task");
3361 let seen: Arc<StdMutex<Option<Ctx>>> = Arc::new(StdMutex::new(None));
3362 let spawner: Arc<dyn SpawnerAdapter> = Arc::new(CtxProbe { seen: seen.clone() });
3363 // The probe always errors the spawn (`SpawnError::Internal`); we
3364 // only care about the `Ctx` snapshot it captured, so the dispatch
3365 // outcome itself (`Err`) is discarded.
3366 let _ = engine
3367 .dispatch_attempt_with(&token, &tid, &spawner, run_id)
3368 .await;
3369 let captured = seen.lock().unwrap().clone();
3370 captured.expect("inner ctx captured")
3371 }
3372
3373 #[tokio::test]
3374 async fn run_id_lands_in_ctx_meta_runtime_when_some() {
3375 let run_id = RunId::new();
3376 let observed = dispatch_with_probe(Some(&run_id)).await;
3377 assert_eq!(
3378 observed.meta.runtime.get("run_id").and_then(|v| v.as_str()),
3379 Some(run_id.as_str()),
3380 "ctx.meta.runtime[\"run_id\"] must carry the run_id passed to dispatch_attempt_with"
3381 );
3382 }
3383
3384 #[tokio::test]
3385 async fn run_id_key_absent_when_none() {
3386 let observed = dispatch_with_probe(None).await;
3387 assert!(
3388 !observed.meta.runtime.contains_key("run_id"),
3389 "no run_id key must be injected when dispatch_attempt_with is called with None"
3390 );
3391 }
3392}
3393
3394/// GH #21 Phase 2: `TaskSpec.step_ctx` must land in
3395/// `Ctx.meta.runtime[STEP_CTX_KEY]` — re-read from the spec on EVERY
3396/// attempt (the prep closure re-reads `task.spec.step_ctx` every call, not
3397/// caching it once at `start_task`), so a retry (attempt 2) carries it
3398/// too. Same `CtxProbe` shape as `dispatch_attempt_with_run_id_tests`.
3399#[cfg(test)]
3400mod dispatch_attempt_with_step_ctx_tests {
3401 use super::*;
3402 use crate::worker::adapter::{SpawnError, SpawnerAdapter};
3403 use crate::worker::Worker;
3404 use std::sync::Mutex as StdMutex;
3405
3406 struct CtxProbe {
3407 seen: Arc<StdMutex<Option<Ctx>>>,
3408 }
3409
3410 #[async_trait::async_trait]
3411 impl SpawnerAdapter for CtxProbe {
3412 async fn spawn(
3413 &self,
3414 _engine: &Engine,
3415 ctx: &Ctx,
3416 _task_id: StepId,
3417 _attempt: u32,
3418 _token: CapToken,
3419 ) -> Result<Box<dyn Worker>, SpawnError> {
3420 *self.seen.lock().unwrap() = Some(ctx.clone());
3421 Err(SpawnError::Internal("probe stop".into()))
3422 }
3423 }
3424
3425 #[tokio::test]
3426 async fn step_ctx_lands_in_ctx_meta_runtime_on_attempt_1_and_2() {
3427 let engine = Engine::new(EngineCfg::default());
3428 let token = engine
3429 .attach("ut-op", Role::Operator, Duration::from_secs(30))
3430 .await
3431 .expect("attach");
3432 let tid = engine
3433 .start_task(
3434 &token,
3435 TaskSpec {
3436 agent: "probe".into(),
3437 initial_directive: "hi".into(),
3438 step_ctx: Some(serde_json::json!({ "work_dir": "/step" })),
3439 check_policy: None,
3440 },
3441 )
3442 .await
3443 .expect("start_task");
3444 let seen: Arc<StdMutex<Option<Ctx>>> = Arc::new(StdMutex::new(None));
3445 let spawner: Arc<dyn SpawnerAdapter> = Arc::new(CtxProbe { seen: seen.clone() });
3446
3447 // The probe always errors the spawn; only the ctx snapshot matters.
3448 let _ = engine
3449 .dispatch_attempt_with(&token, &tid, &spawner, None)
3450 .await;
3451 let first = seen
3452 .lock()
3453 .unwrap()
3454 .clone()
3455 .expect("attempt 1 ctx captured");
3456 assert_eq!(
3457 first.meta.runtime.get(STEP_CTX_KEY),
3458 Some(&serde_json::json!({ "work_dir": "/step" })),
3459 "attempt 1 must carry TaskSpec.step_ctx in ctx.meta.runtime[STEP_CTX_KEY]"
3460 );
3461
3462 let _ = engine
3463 .dispatch_attempt_with(&token, &tid, &spawner, None)
3464 .await;
3465 let second = seen
3466 .lock()
3467 .unwrap()
3468 .clone()
3469 .expect("attempt 2 ctx captured");
3470 assert_eq!(
3471 second.meta.runtime.get(STEP_CTX_KEY),
3472 Some(&serde_json::json!({ "work_dir": "/step" })),
3473 "attempt 2 (retry) must ALSO carry TaskSpec.step_ctx — prep re-reads the spec every attempt"
3474 );
3475 }
3476
3477 #[tokio::test]
3478 async fn step_ctx_key_absent_when_none() {
3479 let engine = Engine::new(EngineCfg::default());
3480 let token = engine
3481 .attach("ut-op", Role::Operator, Duration::from_secs(30))
3482 .await
3483 .expect("attach");
3484 let tid = engine
3485 .start_task(
3486 &token,
3487 TaskSpec {
3488 agent: "probe".into(),
3489 initial_directive: "hi".into(),
3490 step_ctx: None,
3491 check_policy: None,
3492 },
3493 )
3494 .await
3495 .expect("start_task");
3496 let seen: Arc<StdMutex<Option<Ctx>>> = Arc::new(StdMutex::new(None));
3497 let spawner: Arc<dyn SpawnerAdapter> = Arc::new(CtxProbe { seen: seen.clone() });
3498 let _ = engine
3499 .dispatch_attempt_with(&token, &tid, &spawner, None)
3500 .await;
3501 let observed = seen.lock().unwrap().clone().expect("ctx captured");
3502 assert!(
3503 !observed.meta.runtime.contains_key(STEP_CTX_KEY),
3504 "no step_ctx key must be injected when TaskSpec.step_ctx is None"
3505 );
3506 }
3507}
3508
3509// ─── issue #18: `TaskSpec.initial_directive` `Value` pass-through ──────────
3510#[cfg(test)]
3511mod initial_directive_value_passthrough_tests {
3512 use super::*;
3513
3514 async fn seeded_engine(initial_directive: Value) -> (Engine, CapToken, StepId) {
3515 let engine = Engine::new(EngineCfg::default());
3516 let op_token = engine
3517 .attach("ut-op", Role::Operator, Duration::from_secs(30))
3518 .await
3519 .expect("attach");
3520 let task_id = engine
3521 .start_task(
3522 &op_token,
3523 TaskSpec {
3524 agent: "planner".to_string(),
3525 initial_directive,
3526 step_ctx: None,
3527 check_policy: None,
3528 },
3529 )
3530 .await
3531 .expect("start_task");
3532 (engine, op_token, task_id)
3533 }
3534
3535 /// Mint + register a `Role::Worker` token the same way
3536 /// `dispatch_attempt_with` does — `fetch_prompt` is worker-verb-gated.
3537 async fn mint_worker_token(engine: &Engine, task_id: &StepId) -> CapToken {
3538 let worker_token = engine.signer().session(
3539 format!("worker-of-{task_id}"),
3540 Role::Worker,
3541 vec!["*".into()],
3542 Duration::from_secs(600),
3543 );
3544 let fp = worker_token.fingerprint();
3545 let record = CapTokenRecord::from_worker_token(worker_token.clone(), task_id.clone());
3546 engine
3547 .with_state("test.mint_worker", move |s| {
3548 s.tokens.insert(fp, record);
3549 })
3550 .await
3551 .expect("mint worker token");
3552 worker_token
3553 }
3554
3555 /// `EngineDispatcher::dispatch` no longer stringifies the evaluated
3556 /// `Step.in` value before seeding `TaskSpec.initial_directive` — an
3557 /// Object seed must round-trip through `start_task` /
3558 /// `read_task_state` byte-for-byte as the same `Value::Object`, not a
3559 /// JSON-stringified `Value::String`.
3560 #[tokio::test]
3561 async fn object_seed_passes_through_task_spec_unchanged() {
3562 let seed = serde_json::json!({"key": "value"});
3563 let (engine, token, task_id) = seeded_engine(seed.clone()).await;
3564 let state = engine
3565 .read_task_state(&token, &task_id)
3566 .await
3567 .expect("read_task_state");
3568 assert_eq!(
3569 state.spec.initial_directive, seed,
3570 "TaskSpec.initial_directive must equal the raw Object seed, not a stringified copy"
3571 );
3572 }
3573
3574 /// `Engine::fetch_prompt` returns the `Value` end-to-end (issue #18):
3575 /// an Object seed stays a `Value::Object` and is not stringified in
3576 /// the engine layer. The Worker HTTP boundary
3577 /// (`fetch_worker_payload*`) is what performs the render down to a
3578 /// JSON literal `String` for `WorkerPayload.prompt`.
3579 #[tokio::test]
3580 async fn object_seed_passes_through_fetch_prompt_as_value() {
3581 let seed = serde_json::json!({"key": "value"});
3582 let (engine, _token, task_id) = seeded_engine(seed.clone()).await;
3583 let worker_token = mint_worker_token(&engine, &task_id).await;
3584 let prompt = engine
3585 .fetch_prompt(&worker_token, &task_id)
3586 .await
3587 .expect("fetch_prompt");
3588 assert_eq!(
3589 prompt, seed,
3590 "fetch_prompt must return the raw Object Value, not a stringified copy"
3591 );
3592 }
3593
3594 /// The Worker HTTP boundary is the render point: `fetch_worker_payload*`
3595 /// coerces the stored `Value` down to `WorkerPayload.prompt: String`
3596 /// (JSON-literal shape for non-strings). Verifies the boundary render
3597 /// stays intact for an Object seed.
3598 #[tokio::test]
3599 async fn object_seed_renders_as_json_literal_at_worker_payload_boundary() {
3600 let seed = serde_json::json!({"key": "value"});
3601 let (engine, _token, task_id) = seeded_engine(seed).await;
3602 let worker_token = mint_worker_token(&engine, &task_id).await;
3603 let payload = engine
3604 .fetch_worker_payload(&worker_token, &task_id)
3605 .await
3606 .expect("fetch_worker_payload");
3607 assert_eq!(
3608 payload.prompt, r#"{"key":"value"}"#,
3609 "WorkerPayload.prompt must be the JSON literal String render of the Value seed"
3610 );
3611 }
3612
3613 /// A `String` seed is unaffected — still passes through verbatim, both
3614 /// as the `TaskSpec.initial_directive` `Value` and as the Worker
3615 /// `fetch_prompt` return (issue #18 Invariant 2).
3616 #[tokio::test]
3617 async fn string_seed_passes_through_unchanged() {
3618 let (engine, token, task_id) = seeded_engine(serde_json::json!("do the thing")).await;
3619 let state = engine
3620 .read_task_state(&token, &task_id)
3621 .await
3622 .expect("read_task_state");
3623 assert_eq!(
3624 state.spec.initial_directive,
3625 serde_json::json!("do the thing")
3626 );
3627 let worker_token = mint_worker_token(&engine, &task_id).await;
3628 let prompt = engine
3629 .fetch_prompt(&worker_token, &task_id)
3630 .await
3631 .expect("fetch_prompt");
3632 assert_eq!(prompt, serde_json::json!("do the thing"));
3633 }
3634}
3635
3636/// GH #31: `fetch_worker_payload{,_trusted}`'s size-threshold branch
3637/// between inline (`WorkerPayload.system`) and by-reference
3638/// (`WorkerPayload.system_ref`) delivery, plus the `bake_worker_system_prompt`
3639/// `agent_render_sizes` bookkeeping that feeds `agent_last_rendered_size`.
3640#[cfg(test)]
3641mod system_ref_threshold_tests {
3642 use super::*;
3643
3644 async fn seeded_engine_with_cfg(cfg: EngineCfg) -> (Engine, CapToken, StepId) {
3645 let engine = Engine::new(cfg);
3646 let op_token = engine
3647 .attach("ut-op", Role::Operator, Duration::from_secs(30))
3648 .await
3649 .expect("attach");
3650 let task_id = engine
3651 .start_task(
3652 &op_token,
3653 TaskSpec {
3654 agent: "planner".to_string(),
3655 initial_directive: serde_json::json!("do the thing"),
3656 step_ctx: None,
3657 check_policy: None,
3658 },
3659 )
3660 .await
3661 .expect("start_task");
3662 (engine, op_token, task_id)
3663 }
3664
3665 /// Same worker-token-minting fixture as
3666 /// `initial_directive_value_passthrough_tests::mint_worker_token`
3667 /// (kept local to this module — the two `mod`s do not share private
3668 /// helpers across `cfg(test)` boundaries).
3669 async fn mint_worker_token(engine: &Engine, task_id: &StepId) -> CapToken {
3670 let worker_token = engine.signer().session(
3671 format!("worker-of-{task_id}"),
3672 Role::Worker,
3673 vec!["*".into()],
3674 Duration::from_secs(600),
3675 );
3676 let fp = worker_token.fingerprint();
3677 let record = CapTokenRecord::from_worker_token(worker_token.clone(), task_id.clone());
3678 engine
3679 .with_state("test.mint_worker", move |s| {
3680 s.tokens.insert(fp, record);
3681 })
3682 .await
3683 .expect("mint worker token");
3684 worker_token
3685 }
3686
3687 /// Under-threshold: `system` stays inline, `system_ref` stays `None`.
3688 #[tokio::test]
3689 async fn under_threshold_stays_inline() {
3690 let (engine, _op_token, task_id) = seeded_engine_with_cfg(EngineCfg::default()).await;
3691 let worker_token = mint_worker_token(&engine, &task_id).await;
3692 let rendered = "a short system prompt".to_string();
3693 engine
3694 .bake_worker_system_prompt(&task_id, 1, Some(rendered.clone()))
3695 .await
3696 .expect("bake");
3697 let payload = engine
3698 .fetch_worker_payload(&worker_token, &task_id)
3699 .await
3700 .expect("fetch_worker_payload");
3701 assert_eq!(payload.system, Some(rendered));
3702 assert!(payload.system_ref.is_none());
3703 }
3704
3705 /// Over-threshold: `system` is cleared and `system_ref` is populated
3706 /// with a `sha256` matching the known input string. Exercises
3707 /// `fetch_worker_payload_trusted` (the `_trusted` sibling must be
3708 /// behaviorally identical to `fetch_worker_payload`).
3709 #[tokio::test]
3710 async fn over_threshold_switches_to_system_ref_with_matching_sha256() {
3711 let mut cfg = EngineCfg::default();
3712 cfg.system_ref.threshold_bytes = 16;
3713 cfg.system_ref.mode = crate::types::SystemRefMode::File;
3714 cfg.system_ref.store_dir =
3715 std::env::temp_dir().join(format!("mse-system-ref-test-{}", crate::types::now_unix()));
3716 let (engine, _op_token, task_id) = seeded_engine_with_cfg(cfg).await;
3717 let rendered =
3718 "this system prompt is deliberately longer than the 16 byte threshold".to_string();
3719 engine
3720 .bake_worker_system_prompt(&task_id, 1, Some(rendered.clone()))
3721 .await
3722 .expect("bake");
3723 let payload = engine
3724 .fetch_worker_payload_trusted(&task_id)
3725 .await
3726 .expect("fetch_worker_payload_trusted");
3727 assert!(
3728 payload.system.is_none(),
3729 "over-threshold response must not also inline `system`"
3730 );
3731 let system_ref = payload
3732 .system_ref
3733 .expect("over-threshold response must populate system_ref");
3734 assert_eq!(system_ref.size_bytes, rendered.len() as u64);
3735 assert_eq!(system_ref.mode, crate::types::SystemRefMode::File);
3736 use sha2::Digest;
3737 let expected_sha256 = hex::encode(sha2::Sha256::digest(rendered.as_bytes()));
3738 assert_eq!(system_ref.sha256, expected_sha256);
3739 assert!(system_ref.uri.starts_with("file://"));
3740 let written = tokio::fs::read_to_string(system_ref.uri.trim_start_matches("file://"))
3741 .await
3742 .expect("File mode must have written the referenced path");
3743 assert_eq!(written, rendered);
3744 }
3745
3746 /// `Http` mode never writes a file — `system_ref.uri` is the bare path
3747 /// the engine can construct on its own, scheme/host-free.
3748 #[tokio::test]
3749 async fn over_threshold_http_mode_constructs_path_only_uri() {
3750 let mut cfg = EngineCfg::default();
3751 cfg.system_ref.threshold_bytes = 16;
3752 cfg.system_ref.mode = crate::types::SystemRefMode::Http;
3753 let (engine, _op_token, task_id) = seeded_engine_with_cfg(cfg).await;
3754 let worker_token = mint_worker_token(&engine, &task_id).await;
3755 let rendered =
3756 "this system prompt is deliberately longer than the 16 byte threshold".to_string();
3757 engine
3758 .bake_worker_system_prompt(&task_id, 1, Some(rendered))
3759 .await
3760 .expect("bake");
3761 let payload = engine
3762 .fetch_worker_payload(&worker_token, &task_id)
3763 .await
3764 .expect("fetch_worker_payload");
3765 let system_ref = payload.system_ref.expect("system_ref must be populated");
3766 assert_eq!(system_ref.mode, crate::types::SystemRefMode::Http);
3767 assert_eq!(
3768 system_ref.uri,
3769 format!("/v1/worker/prompt/system?task_id={task_id}&attempt=1")
3770 );
3771 }
3772
3773 /// `bake_worker_system_prompt` records the render size keyed by agent
3774 /// name (last-write-wins), readable via `agent_last_rendered_size`.
3775 #[tokio::test]
3776 async fn bake_records_agent_render_size_last_write_wins() {
3777 let (engine, _op_token, task_id) = seeded_engine_with_cfg(EngineCfg::default()).await;
3778 assert_eq!(engine.agent_last_rendered_size("planner").await, None);
3779 engine
3780 .bake_worker_system_prompt(&task_id, 1, Some("a".repeat(10)))
3781 .await
3782 .expect("bake 1");
3783 assert_eq!(engine.agent_last_rendered_size("planner").await, Some(10));
3784 engine
3785 .bake_worker_system_prompt(&task_id, 2, Some("b".repeat(20)))
3786 .await
3787 .expect("bake 2");
3788 assert_eq!(
3789 engine.agent_last_rendered_size("planner").await,
3790 Some(20),
3791 "most-recently-observed size wins, not the largest"
3792 );
3793 }
3794}
3795
3796/// subtask-4 / ST2 rework: `submit_output` / `submit_worker_result_trusted`'s
3797/// submit-time projection sink (`Engine::materialize_final_submission`) —
3798/// the Data-plane `OutputStore` dual-write plus the
3799/// `FileProjectionAdapter`-backed file materialize, both fail-open. See
3800/// the subtask-4 Tests this module covers inline on each test.
3801#[cfg(test)]
3802mod submit_time_projection_sink_tests {
3803 use super::*;
3804 use crate::core::agent_context::AgentContextView;
3805 use crate::store::output::{ContentRef, InMemoryOutputStore, OutputEvent};
3806
3807 /// Starts a task under `agent`, returning `(engine, op_token, task_id,
3808 /// worker_token)` — same helper shape as the sibling test modules
3809 /// above (`initial_directive_value_passthrough_tests::seeded_engine` /
3810 /// `mint_worker_token`), duplicated locally per this file's
3811 /// established per-module convention.
3812 async fn seeded_task(agent: &str) -> (Engine, CapToken, StepId, CapToken) {
3813 let engine = Engine::new(EngineCfg::default());
3814 let op_token = engine
3815 .attach("ut-op", Role::Operator, Duration::from_secs(30))
3816 .await
3817 .expect("attach");
3818 let task_id = engine
3819 .start_task(
3820 &op_token,
3821 TaskSpec {
3822 agent: agent.to_string(),
3823 initial_directive: Value::String("go".into()),
3824 step_ctx: None,
3825 check_policy: None,
3826 },
3827 )
3828 .await
3829 .expect("start_task");
3830 let worker_token = engine.signer().session(
3831 format!("worker-of-{task_id}"),
3832 Role::Worker,
3833 vec!["*".into()],
3834 Duration::from_secs(600),
3835 );
3836 let fp = worker_token.fingerprint();
3837 let record = CapTokenRecord::from_worker_token(worker_token.clone(), task_id.clone());
3838 engine
3839 .with_state("test.mint_worker", move |s| {
3840 s.tokens.insert(fp, record);
3841 })
3842 .await
3843 .expect("mint worker token");
3844 (engine, op_token, task_id, worker_token)
3845 }
3846
3847 /// Sibling of [`seeded_task`] that lets a caller pin the engine's
3848 /// `EngineCfg.check_policy` before the engine is constructed — used
3849 /// by the `check_policy_*` regression tests below to exercise the
3850 /// three [`crate::core::config::CheckPolicy`] modes without touching
3851 /// the shared `seeded_task` helper (which every unrelated sink test
3852 /// depends on).
3853 async fn seeded_task_with_policy(
3854 agent: &str,
3855 policy: crate::core::config::CheckPolicy,
3856 ) -> (Engine, CapToken, StepId, CapToken) {
3857 let cfg = EngineCfg {
3858 check_policy: policy,
3859 ..EngineCfg::default()
3860 };
3861 let engine = Engine::new(cfg);
3862 let op_token = engine
3863 .attach("ut-op", Role::Operator, Duration::from_secs(30))
3864 .await
3865 .expect("attach");
3866 let task_id = engine
3867 .start_task(
3868 &op_token,
3869 TaskSpec {
3870 agent: agent.to_string(),
3871 initial_directive: Value::String("go".into()),
3872 step_ctx: None,
3873 check_policy: None,
3874 },
3875 )
3876 .await
3877 .expect("start_task");
3878 let worker_token = engine.signer().session(
3879 format!("worker-of-{task_id}"),
3880 Role::Worker,
3881 vec!["*".into()],
3882 Duration::from_secs(600),
3883 );
3884 let fp = worker_token.fingerprint();
3885 let record = CapTokenRecord::from_worker_token(worker_token.clone(), task_id.clone());
3886 engine
3887 .with_state("test.mint_worker", move |s| {
3888 s.tokens.insert(fp, record);
3889 })
3890 .await
3891 .expect("mint worker token");
3892 (engine, op_token, task_id, worker_token)
3893 }
3894
3895 /// Seeds `EngineState.agent_ctx[(task_id, attempt)].view` directly —
3896 /// the same snapshot `AgentContextMiddleware` writes at spawn time
3897 /// (see its module doc), stood up here without the full spawner
3898 /// stack so these tests can exercise `submit_output` in isolation.
3899 async fn seed_agent_context(engine: &Engine, task_id: &StepId, attempt: u32, work_dir: &str) {
3900 let task_id = task_id.clone();
3901 let work_dir = work_dir.to_string();
3902 engine
3903 .with_state("test.seed_agent_context", move |s| {
3904 s.agent_ctx.insert(
3905 (task_id, attempt),
3906 crate::core::state::AgentCtxEntry {
3907 view: AgentContextView {
3908 work_dir: Some(work_dir),
3909 ..Default::default()
3910 },
3911 policy: Default::default(),
3912 },
3913 );
3914 })
3915 .await
3916 .expect("seed agent_ctx");
3917 }
3918
3919 /// GH #27 (follow-up to #23): seeds `EngineState.agent_ctx` with an
3920 /// arbitrary `work_dir` / `project_root` pair (either may be `None`),
3921 /// unlike [`seed_agent_context`] (which only ever sets `work_dir`) —
3922 /// lets these tests exercise `ProjectionPlacement::resolve_root`'s
3923 /// fallback in both directions.
3924 async fn seed_agent_context_roots(
3925 engine: &Engine,
3926 task_id: &StepId,
3927 attempt: u32,
3928 work_dir: Option<&str>,
3929 project_root: Option<&str>,
3930 ) {
3931 let task_id = task_id.clone();
3932 let work_dir = work_dir.map(str::to_string);
3933 let project_root = project_root.map(str::to_string);
3934 engine
3935 .with_state("test.seed_agent_context_roots", move |s| {
3936 s.agent_ctx.insert(
3937 (task_id, attempt),
3938 crate::core::state::AgentCtxEntry {
3939 view: AgentContextView {
3940 work_dir,
3941 project_root,
3942 ..Default::default()
3943 },
3944 policy: Default::default(),
3945 },
3946 );
3947 })
3948 .await
3949 .expect("seed agent_ctx");
3950 }
3951
3952 /// GH #27 (follow-up to #23): seeds `EngineState.projection_placements`
3953 /// directly — the same snapshot `EngineDispatcher::dispatch` stashes
3954 /// at dispatch time (mirroring [`seed_step_naming`]'s contract) — so
3955 /// these tests can exercise a declared `ProjectionPlacement` without
3956 /// driving a real `Compiler::compile`.
3957 async fn seed_projection_placement(
3958 engine: &Engine,
3959 task_id: &StepId,
3960 placement: crate::core::projection_placement::ProjectionPlacement,
3961 ) {
3962 let task_id = task_id.clone();
3963 let placement = Arc::new(placement);
3964 engine
3965 .with_state("test.seed_projection_placement", move |s| {
3966 s.projection_placements.insert(task_id, placement);
3967 })
3968 .await
3969 .expect("seed projection_placements");
3970 }
3971
3972 /// GH #23 subtask-2: builds a fixture
3973 /// [`crate::core::step_naming::StepNaming`] table declaring `producer`
3974 /// → `canonical` (`AgentMeta.projection_name`), then seeds it into
3975 /// `EngineState.step_namings` for `task_id` — the same snapshot
3976 /// `EngineDispatcher::dispatch` stashes at dispatch time
3977 /// (`blueprint.rs`'s "construct once, read many" contract), stood up
3978 /// here without the full Blueprint-compile path so these tests can
3979 /// exercise the canonical-sink resolution in isolation.
3980 async fn seed_step_naming(engine: &Engine, task_id: &StepId, producer: &str, canonical: &str) {
3981 use crate::blueprint::{
3982 current_schema_version, AgentDef, AgentKind, AgentMeta, Blueprint, BlueprintMetadata,
3983 CompilerHints, CompilerStrategy,
3984 };
3985 use crate::core::step_naming::StepNaming;
3986 use mlua_flow_ir::{Expr, Node};
3987
3988 let flow = Node::Step {
3989 ref_: producer.to_string(),
3990 in_: Expr::Path {
3991 at: "$.in".parse().expect("literal test path: $.in"),
3992 },
3993 out: Expr::Path {
3994 at: format!("$.{producer}_out")
3995 .parse()
3996 .expect("literal test path"),
3997 },
3998 };
3999 let bp = Blueprint {
4000 schema_version: current_schema_version(),
4001 id: "sink-canonical-ut".into(),
4002 flow,
4003 agents: vec![AgentDef {
4004 name: producer.to_string(),
4005 kind: AgentKind::RustFn,
4006 spec: serde_json::json!({ "fn_id": producer }),
4007 profile: None,
4008 meta: Some(AgentMeta {
4009 projection_name: Some(canonical.to_string()),
4010 ..Default::default()
4011 }),
4012 runner: None,
4013 runner_ref: None,
4014 verdict: None,
4015 }],
4016 operators: vec![],
4017 metas: vec![],
4018 hints: CompilerHints::default(),
4019 strategy: CompilerStrategy::default(),
4020 metadata: BlueprintMetadata::default(),
4021 spawner_hints: Default::default(),
4022 default_agent_kind: AgentKind::Operator,
4023 default_operator_kind: None,
4024 default_init_ctx: None,
4025 default_agent_ctx: None,
4026 default_context_policy: None,
4027 projection_placement: None,
4028 audits: vec![],
4029 degradation_policy: None,
4030 runners: vec![],
4031 default_runner: None,
4032 check_policy: None,
4033 };
4034 let (naming, warnings) = StepNaming::from_blueprint(&bp).expect("no collision");
4035 assert!(warnings.is_empty(), "single-step fixture has no collisions");
4036 let naming = Arc::new(naming);
4037 let task_id = task_id.clone();
4038 engine
4039 .with_state("test.seed_step_naming", move |s| {
4040 s.step_namings.insert(task_id, naming);
4041 })
4042 .await
4043 .expect("seed step_namings");
4044 }
4045
4046 fn final_event(value: Value, ok: bool) -> crate::worker::output::OutputEvent {
4047 crate::worker::output::OutputEvent::Final {
4048 content: crate::worker::output::ContentRef::Inline { value },
4049 ok,
4050 }
4051 }
4052
4053 /// Subtask 4 Test #2: `submit_output`'s `Final` writes
4054 /// `<root>/workspace/tasks/<task_id>/ctx/<agent>.md`, content matching
4055 /// the submitted value.
4056 #[tokio::test]
4057 async fn submit_output_final_materializes_file_when_work_dir_resolved() {
4058 let dir = tempfile::TempDir::new().unwrap();
4059 let (engine, _op, task_id, worker_token) = seeded_task("planner").await;
4060 seed_agent_context(&engine, &task_id, 1, &dir.path().to_string_lossy()).await;
4061
4062 engine
4063 .submit_output(
4064 &worker_token,
4065 &task_id,
4066 1,
4067 final_event(serde_json::json!({"plan": "do it"}), true),
4068 )
4069 .await
4070 .expect("submit_output");
4071
4072 let expected_file = dir
4073 .path()
4074 .join("workspace/tasks")
4075 .join(task_id.as_str())
4076 .join("ctx/planner.md");
4077 assert!(
4078 expected_file.exists(),
4079 "materialized submission file missing at {expected_file:?}"
4080 );
4081 let body = std::fs::read_to_string(expected_file).unwrap();
4082 assert!(body.contains(r#""plan": "do it""#), "body: {body}");
4083 }
4084
4085 /// Subtask 4 Test #3: `work_dir` unresolved (no `agent_ctx`
4086 /// snapshot for this `(task_id, attempt)`) — submit still succeeds,
4087 /// fail-open, no file.
4088 #[tokio::test]
4089 async fn submit_output_final_skips_file_when_root_unresolved() {
4090 let (engine, _op, task_id, worker_token) = seeded_task("planner").await;
4091 // No seed_agent_context call — root is unresolved.
4092
4093 let result = engine
4094 .submit_output(
4095 &worker_token,
4096 &task_id,
4097 1,
4098 final_event(serde_json::json!("hi"), true),
4099 )
4100 .await;
4101 assert!(
4102 result.is_ok(),
4103 "submit must succeed even with no resolvable root (fail-open, Invariant 1)"
4104 );
4105 }
4106
4107 /// Regression for the check_policy cascade: the default
4108 /// [`crate::core::config::CheckPolicy::Warn`] preserves the
4109 /// pre-`CheckPolicy` fail-open semantics — a submit whose root is
4110 /// unresolved still succeeds. Byte-compat with
4111 /// `submit_output_final_skips_file_when_root_unresolved`; this test
4112 /// pins the mode explicitly so a future default change to
4113 /// `Strict` (silent breakage) is caught here.
4114 #[tokio::test]
4115 async fn submit_output_final_check_policy_warn_preserves_fail_open() {
4116 let (engine, _op, task_id, worker_token) =
4117 seeded_task_with_policy("planner", crate::core::config::CheckPolicy::Warn).await;
4118
4119 let result = engine
4120 .submit_output(
4121 &worker_token,
4122 &task_id,
4123 1,
4124 final_event(serde_json::json!("hi"), true),
4125 )
4126 .await;
4127 assert!(
4128 result.is_ok(),
4129 "Warn mode preserves fail-open: submit must succeed when root unresolved"
4130 );
4131 }
4132
4133 /// Regression for the check_policy cascade:
4134 /// [`crate::core::config::CheckPolicy::Strict`] surfaces the "no
4135 /// work_dir/project_root resolved" fail-open condition as an
4136 /// [`EngineError::CheckPolicyStrict`], letting a caller who has
4137 /// opted in fail fast instead of proceeding with a partially-
4138 /// realized submission. The error's `context` identifies the call
4139 /// site (`"file materialize"`), and `message` preserves the
4140 /// pre-`CheckPolicy` warn literal verbatim (log-parse compat).
4141 #[tokio::test]
4142 async fn submit_output_final_check_policy_strict_surfaces_error_when_root_unresolved() {
4143 let (engine, _op, task_id, worker_token) =
4144 seeded_task_with_policy("planner", crate::core::config::CheckPolicy::Strict).await;
4145
4146 let err = engine
4147 .submit_output(
4148 &worker_token,
4149 &task_id,
4150 1,
4151 final_event(serde_json::json!("hi"), true),
4152 )
4153 .await
4154 .expect_err("Strict mode must return an error when root unresolved");
4155 match err {
4156 EngineError::CheckPolicyStrict { context, message } => {
4157 assert!(
4158 context.contains("file materialize"),
4159 "context must identify the call site: {context}"
4160 );
4161 assert!(
4162 message.contains("no work_dir/project_root resolved"),
4163 "message must preserve the warn-log literal for log-parse compat: {message}"
4164 );
4165 }
4166 other => panic!(
4167 "expected EngineError::CheckPolicyStrict, got a different variant: {other:?}"
4168 ),
4169 }
4170 }
4171
4172 /// Regression for the check_policy cascade:
4173 /// [`crate::core::config::CheckPolicy::Silent`] returns `Ok(())` (
4174 /// like `Warn`) without surfacing an error. The log-suppression side
4175 /// of `Silent` (no `tracing::warn!`) is enforced at the call site
4176 /// via the `if !matches!(policy, Silent) { warn!(...) }` guard —
4177 /// verifying tracing output shape here would couple the test to a
4178 /// subscriber setup, so the assertion is limited to the error-
4179 /// return semantics (matches the helper unit tests in
4180 /// `check_policy_helper_tests`).
4181 #[tokio::test]
4182 async fn submit_output_final_check_policy_silent_returns_ok_when_root_unresolved() {
4183 let (engine, _op, task_id, worker_token) =
4184 seeded_task_with_policy("planner", crate::core::config::CheckPolicy::Silent).await;
4185
4186 let result = engine
4187 .submit_output(
4188 &worker_token,
4189 &task_id,
4190 1,
4191 final_event(serde_json::json!("hi"), true),
4192 )
4193 .await;
4194 assert!(
4195 result.is_ok(),
4196 "Silent mode returns Ok(()) at the error surface: submit must succeed"
4197 );
4198 }
4199
4200 /// Subtask 4 Test #4 (file half): re-submitting under the same
4201 /// `(task_id, agent)` overwrites the materialized file with the
4202 /// latest value.
4203 #[tokio::test]
4204 async fn resubmit_overwrites_materialized_file_with_latest() {
4205 let dir = tempfile::TempDir::new().unwrap();
4206 let (engine, _op, task_id, worker_token) = seeded_task("planner").await;
4207 seed_agent_context(&engine, &task_id, 1, &dir.path().to_string_lossy()).await;
4208
4209 engine
4210 .submit_output(
4211 &worker_token,
4212 &task_id,
4213 1,
4214 final_event(serde_json::json!("first"), true),
4215 )
4216 .await
4217 .expect("first submit");
4218 engine
4219 .submit_output(
4220 &worker_token,
4221 &task_id,
4222 1,
4223 final_event(serde_json::json!("second"), true),
4224 )
4225 .await
4226 .expect("second submit");
4227
4228 let expected_file = dir
4229 .path()
4230 .join("workspace/tasks")
4231 .join(task_id.as_str())
4232 .join("ctx/planner.md");
4233 let body = std::fs::read_to_string(expected_file).unwrap();
4234 assert!(body.contains("second"), "body must reflect latest: {body}");
4235 assert!(
4236 !body.contains("first"),
4237 "body must not carry the stale value: {body}"
4238 );
4239 }
4240
4241 /// GH #27 (follow-up to #23): the byte-compat default
4242 /// `ProjectionPlacement` (`root_preference = WorkDir`) falls back to
4243 /// `project_root` when `work_dir` is absent — the same fallback
4244 /// [`crate::core::projection_placement::ProjectionPlacement::resolve_root`]
4245 /// now performs for every one of the "3 path" call sites, this one
4246 /// exercised at the submit-sink layer.
4247 #[tokio::test]
4248 async fn submit_output_final_falls_back_to_project_root_when_work_dir_absent() {
4249 let dir = tempfile::TempDir::new().unwrap();
4250 let (engine, _op, task_id, worker_token) = seeded_task("planner").await;
4251 seed_agent_context_roots(
4252 &engine,
4253 &task_id,
4254 1,
4255 None,
4256 Some(&dir.path().to_string_lossy()),
4257 )
4258 .await;
4259
4260 engine
4261 .submit_output(
4262 &worker_token,
4263 &task_id,
4264 1,
4265 final_event(serde_json::json!({"plan": "via project_root"}), true),
4266 )
4267 .await
4268 .expect("submit_output");
4269
4270 let expected_file = dir
4271 .path()
4272 .join("workspace/tasks")
4273 .join(task_id.as_str())
4274 .join("ctx/planner.md");
4275 assert!(
4276 expected_file.exists(),
4277 "materialized submission file missing at {expected_file:?} \
4278 (work_dir absent must fall back to project_root)"
4279 );
4280 }
4281
4282 /// GH #27 (follow-up to #23): a declared `ProjectionPlacement`
4283 /// (`root_preference = ProjectRoot`, custom `dir_template`) changes
4284 /// BOTH which root is preferred (project_root wins even though
4285 /// work_dir is also present) AND the target directory layout — proof
4286 /// the submit sink consults the snapshotted resolver rather than a
4287 /// hardcoded layout.
4288 #[tokio::test]
4289 async fn submit_output_final_uses_declared_projection_placement() {
4290 let work_dir = tempfile::TempDir::new().unwrap();
4291 let project_root = tempfile::TempDir::new().unwrap();
4292 let (engine, _op, task_id, worker_token) = seeded_task("planner").await;
4293 seed_agent_context_roots(
4294 &engine,
4295 &task_id,
4296 1,
4297 Some(&work_dir.path().to_string_lossy()),
4298 Some(&project_root.path().to_string_lossy()),
4299 )
4300 .await;
4301 seed_projection_placement(
4302 &engine,
4303 &task_id,
4304 crate::core::projection_placement::ProjectionPlacement {
4305 root_preference: crate::core::projection_placement::RootPreference::ProjectRoot,
4306 dir_template: "custom/{task_id}/out".to_string(),
4307 },
4308 )
4309 .await;
4310
4311 engine
4312 .submit_output(
4313 &worker_token,
4314 &task_id,
4315 1,
4316 final_event(serde_json::json!({"plan": "via custom placement"}), true),
4317 )
4318 .await
4319 .expect("submit_output");
4320
4321 let expected_file = project_root
4322 .path()
4323 .join("custom")
4324 .join(task_id.as_str())
4325 .join("out/planner.md");
4326 assert!(
4327 expected_file.exists(),
4328 "materialized submission file missing at custom placement target {expected_file:?}"
4329 );
4330 let unexpected_file = work_dir
4331 .path()
4332 .join("workspace/tasks")
4333 .join(task_id.as_str())
4334 .join("ctx/planner.md");
4335 assert!(
4336 !unexpected_file.exists(),
4337 "declared root_preference=ProjectRoot must not fall back to work_dir: {unexpected_file:?}"
4338 );
4339 }
4340
4341 /// Subtask 4 Invariant 3 / crux requirement #3: when
4342 /// [`Engine::set_output_store`] wires a Data-plane [`crate::store::output::OutputStore`],
4343 /// `submit_output`'s `Final` dual-writes into it under
4344 /// `producer_agent = TaskState.spec.agent` — the store becomes
4345 /// queryable via `get_latest_by_name`, independent of whether a root
4346 /// resolved for the file half.
4347 #[tokio::test]
4348 async fn submit_output_final_dual_writes_into_configured_output_store() {
4349 let (engine, _op, task_id, worker_token) = seeded_task("reviewer").await;
4350 let data_store: Arc<dyn crate::store::output::OutputStore> =
4351 Arc::new(InMemoryOutputStore::new());
4352 engine.set_output_store(data_store.clone());
4353
4354 engine
4355 .submit_output(
4356 &worker_token,
4357 &task_id,
4358 1,
4359 final_event(serde_json::json!({"verdict": "pass"}), true),
4360 )
4361 .await
4362 .expect("submit_output");
4363
4364 let record = data_store
4365 .get_latest_by_name("reviewer")
4366 .await
4367 .expect("dual-written record");
4368 match record.event {
4369 OutputEvent::Final { content, ok } => {
4370 assert!(ok);
4371 match content {
4372 ContentRef::Inline { value } => {
4373 assert_eq!(value, serde_json::json!({"verdict": "pass"}));
4374 }
4375 other => panic!("expected Inline content, got {other:?}"),
4376 }
4377 }
4378 other => panic!("expected Final event, got {other:?}"),
4379 }
4380 }
4381
4382 /// GH #34 subtask-3 gap fix: an `Artifact` event submitted via
4383 /// `submit_output` dual-writes into a wired Data-plane `OutputStore`
4384 /// under its OWN `name`, verbatim — mirrors
4385 /// `submit_output_final_dual_writes_into_configured_output_store`
4386 /// above, but for the `Artifact` variant.
4387 #[tokio::test]
4388 async fn submit_output_artifact_dual_writes_into_configured_output_store() {
4389 let (engine, _op, task_id, worker_token) = seeded_task("echo").await;
4390 let data_store: Arc<dyn crate::store::output::OutputStore> =
4391 Arc::new(InMemoryOutputStore::new());
4392 engine.set_output_store(data_store.clone());
4393
4394 engine
4395 .submit_output(
4396 &worker_token,
4397 &task_id,
4398 1,
4399 OutputEvent::Artifact {
4400 name: "audit:echo".to_string(),
4401 content: ContentRef::Inline {
4402 value: serde_json::json!({"finding": "clean"}),
4403 },
4404 },
4405 )
4406 .await
4407 .expect("submit_output");
4408
4409 let record = data_store
4410 .get_latest_by_name("audit:echo")
4411 .await
4412 .expect("dual-written artifact record");
4413 match record.event {
4414 OutputEvent::Artifact { name, content } => {
4415 assert_eq!(name, "audit:echo");
4416 match content {
4417 ContentRef::Inline { value } => {
4418 assert_eq!(value, serde_json::json!({"finding": "clean"}));
4419 }
4420 other => panic!("expected Inline content, got {other:?}"),
4421 }
4422 }
4423 other => panic!("expected Artifact event, got {other:?}"),
4424 }
4425 // The `Artifact` dual-write must never collide with / overwrite
4426 // the producing step's own `Final` name — `submit_output` never
4427 // materialized a `Final` here, so `"echo"` must stay unresolved.
4428 assert!(
4429 data_store.get_latest_by_name("echo").await.is_err(),
4430 "artifact write must not fabricate a record under the raw producer_agent name"
4431 );
4432 }
4433
4434 /// Invariant 1 (fail-open) for `Artifact`, mirroring
4435 /// `submit_output_final_skips_file_when_root_unresolved`'s Final-side
4436 /// coverage: no `OutputStore` wired at all — submit still succeeds.
4437 #[tokio::test]
4438 async fn submit_output_artifact_is_fail_open_when_no_output_store_configured() {
4439 let (engine, _op, task_id, worker_token) = seeded_task("echo").await;
4440
4441 let result = engine
4442 .submit_output(
4443 &worker_token,
4444 &task_id,
4445 1,
4446 OutputEvent::Artifact {
4447 name: "audit:echo".to_string(),
4448 content: ContentRef::Inline {
4449 value: serde_json::json!("finding"),
4450 },
4451 },
4452 )
4453 .await;
4454 assert!(
4455 result.is_ok(),
4456 "submit must succeed even with no OutputStore wired (fail-open, Invariant 1)"
4457 );
4458 }
4459
4460 /// `submit_worker_result_trusted` (the `/v1/worker/submit` short-handle
4461 /// path) triggers the exact same sink as `submit_output` — parity
4462 /// across both worker-submit entry points.
4463 #[tokio::test]
4464 async fn submit_worker_result_trusted_also_triggers_projection_sink() {
4465 let dir = tempfile::TempDir::new().unwrap();
4466 let (engine, _op, task_id, _worker_token) = seeded_task("planner").await;
4467 seed_agent_context(&engine, &task_id, 1, &dir.path().to_string_lossy()).await;
4468 let data_store: Arc<dyn crate::store::output::OutputStore> =
4469 Arc::new(InMemoryOutputStore::new());
4470 engine.set_output_store(data_store.clone());
4471
4472 engine
4473 .submit_worker_result_trusted(&task_id, 1, serde_json::json!("trusted-value"), true)
4474 .await
4475 .expect("submit_worker_result_trusted");
4476
4477 let expected_file = dir
4478 .path()
4479 .join("workspace/tasks")
4480 .join(task_id.as_str())
4481 .join("ctx/planner.md");
4482 assert!(expected_file.exists());
4483 let record = data_store
4484 .get_latest_by_name("planner")
4485 .await
4486 .expect("dual-written record");
4487 assert!(matches!(record.event, OutputEvent::Final { ok: true, .. }));
4488 }
4489
4490 /// GH #23 subtask-2 (canonical sink): a declared `projection_name`
4491 /// (`AgentMeta.projection_name`, surfaced via `StepNaming`) redirects
4492 /// `submit_output`'s Final canonical sink — both the Data-plane
4493 /// dual-write name and the materialized file stem resolve to the
4494 /// canonical name, not the raw `producer_agent`.
4495 #[tokio::test]
4496 async fn submit_output_final_uses_canonical_name_when_step_naming_declares_one() {
4497 let dir = tempfile::TempDir::new().unwrap();
4498 let (engine, _op, task_id, worker_token) = seeded_task("reviewer").await;
4499 seed_agent_context(&engine, &task_id, 1, &dir.path().to_string_lossy()).await;
4500 seed_step_naming(&engine, &task_id, "reviewer", "verdict-final").await;
4501 let data_store: Arc<dyn crate::store::output::OutputStore> =
4502 Arc::new(InMemoryOutputStore::new());
4503 engine.set_output_store(data_store.clone());
4504
4505 engine
4506 .submit_output(
4507 &worker_token,
4508 &task_id,
4509 1,
4510 final_event(serde_json::json!({"verdict": "pass"}), true),
4511 )
4512 .await
4513 .expect("submit_output");
4514
4515 let record = data_store
4516 .get_latest_by_name("verdict-final")
4517 .await
4518 .expect("dual-written record under canonical name");
4519 assert!(matches!(record.event, OutputEvent::Final { ok: true, .. }));
4520 assert!(
4521 data_store.get_latest_by_name("reviewer").await.is_err(),
4522 "raw producer_agent name must not be written once canonical resolves"
4523 );
4524
4525 let expected_file = dir
4526 .path()
4527 .join("workspace/tasks")
4528 .join(task_id.as_str())
4529 .join("ctx/verdict-final.md");
4530 assert!(
4531 expected_file.exists(),
4532 "materialized file stem must be canonical at {expected_file:?}"
4533 );
4534 }
4535
4536 /// GH #23 subtask-2: no `StepNaming` table snapshotted for this
4537 /// `task_id` (the pre-GH-#23 / no-`with_step_naming` path) is a
4538 /// defensive fail-open — the canonical sink falls back to the raw
4539 /// `producer_agent`, byte-identical to
4540 /// `submit_output_final_dual_writes_into_configured_output_store`
4541 /// above (which never calls `seed_step_naming`).
4542 #[tokio::test]
4543 async fn submit_output_final_falls_back_to_producer_agent_when_no_step_naming_table() {
4544 let (engine, _op, task_id, worker_token) = seeded_task("reviewer").await;
4545 let data_store: Arc<dyn crate::store::output::OutputStore> =
4546 Arc::new(InMemoryOutputStore::new());
4547 engine.set_output_store(data_store.clone());
4548
4549 engine
4550 .submit_output(
4551 &worker_token,
4552 &task_id,
4553 1,
4554 final_event(serde_json::json!({"verdict": "pass"}), true),
4555 )
4556 .await
4557 .expect("submit_output");
4558
4559 let record = data_store
4560 .get_latest_by_name("reviewer")
4561 .await
4562 .expect("fail-open dual-write under raw producer_agent name");
4563 assert!(matches!(record.event, OutputEvent::Final { ok: true, .. }));
4564 }
4565
4566 /// GH #23 subtask-2 (Layer 2): `OutputStore::get_latest_by_name_in_run`
4567 /// resolves the value `submit_output` dual-wrote for this exact
4568 /// `(task_id, attempt)` run, independent of `get_latest_by_name`'s
4569 /// cross-Run race (two Runs sharing a producer name never bleed into
4570 /// each other through the Run-scoped accessor).
4571 #[tokio::test]
4572 async fn submit_output_final_is_resolvable_via_run_scoped_lookup() {
4573 let (engine, _op, task_id, worker_token) = seeded_task("reviewer").await;
4574 let data_store: Arc<dyn crate::store::output::OutputStore> =
4575 Arc::new(InMemoryOutputStore::new());
4576 engine.set_output_store(data_store.clone());
4577
4578 engine
4579 .submit_output(
4580 &worker_token,
4581 &task_id,
4582 1,
4583 final_event(serde_json::json!({"verdict": "pass"}), true),
4584 )
4585 .await
4586 .expect("submit_output");
4587
4588 let record = data_store
4589 .get_latest_by_name_in_run(task_id.as_str(), 1, "reviewer")
4590 .await
4591 .expect("run-scoped lookup resolves the dual-written record");
4592 assert!(matches!(record.event, OutputEvent::Final { ok: true, .. }));
4593
4594 // A different attempt of the same task must not resolve — the
4595 // Run-scoped lookup does not fall back across attempts.
4596 assert!(
4597 data_store
4598 .get_latest_by_name_in_run(task_id.as_str(), 2, "reviewer")
4599 .await
4600 .is_err(),
4601 "a different attempt must not resolve the same-named record"
4602 );
4603 }
4604
4605 // ─── staged part file materialize ───
4606
4607 /// Staging a part with a resolved `work_dir` writes
4608 /// `<work_dir>/workspace/tasks/<task_id>/ctx/<name>` with the part's
4609 /// content RAW (no front matter / fenced wrapper).
4610 #[tokio::test]
4611 async fn stage_artifact_materializes_part_file_when_work_dir_resolved() {
4612 let dir = tempfile::TempDir::new().unwrap();
4613 let (engine, _op, task_id, _worker_token) = seeded_task("planner").await;
4614 seed_agent_context(&engine, &task_id, 1, &dir.path().to_string_lossy()).await;
4615
4616 engine
4617 .stage_worker_artifact_trusted(
4618 &task_id,
4619 1,
4620 "plan.md".to_string(),
4621 serde_json::json!("# Plan\n\nstep one\n"),
4622 )
4623 .await
4624 .expect("stage artifact");
4625
4626 let expected_file = dir
4627 .path()
4628 .join("workspace/tasks")
4629 .join(task_id.as_str())
4630 .join("ctx/plan.md");
4631 assert!(
4632 expected_file.exists(),
4633 "materialized part file missing at {expected_file:?}"
4634 );
4635 let body = std::fs::read_to_string(expected_file).unwrap();
4636 // Raw — no YAML front matter / fenced-json wrapper.
4637 assert_eq!(body, "# Plan\n\nstep one\n");
4638 }
4639
4640 /// No resolvable root + `Warn` — staging still
4641 /// succeeds (fail-open), and no part file is written.
4642 #[tokio::test]
4643 async fn stage_artifact_check_policy_warn_skips_part_file_when_root_unresolved() {
4644 let dir = tempfile::TempDir::new().unwrap();
4645 let (engine, _op, task_id, _worker_token) =
4646 seeded_task_with_policy("planner", crate::core::config::CheckPolicy::Warn).await;
4647 // No seed_agent_context — root unresolved.
4648
4649 let result = engine
4650 .stage_worker_artifact_trusted(
4651 &task_id,
4652 1,
4653 "plan.md".to_string(),
4654 serde_json::json!("x"),
4655 )
4656 .await;
4657 assert!(
4658 result.is_ok(),
4659 "Warn mode preserves fail-open: stage must succeed when root unresolved"
4660 );
4661 assert!(
4662 !dir.path().join("workspace").exists(),
4663 "no part file may be materialized when root is unresolved"
4664 );
4665 }
4666
4667 /// No resolvable root + `Strict` — staging surfaces
4668 /// the fail-open condition as an [`EngineError::CheckPolicyStrict`],
4669 /// its message identifying the "part file materialize" call site.
4670 #[tokio::test]
4671 async fn stage_artifact_check_policy_strict_surfaces_error_when_root_unresolved() {
4672 let (engine, _op, task_id, _worker_token) =
4673 seeded_task_with_policy("planner", crate::core::config::CheckPolicy::Strict).await;
4674
4675 let err = engine
4676 .stage_worker_artifact_trusted(
4677 &task_id,
4678 1,
4679 "plan.md".to_string(),
4680 serde_json::json!("x"),
4681 )
4682 .await
4683 .expect_err("Strict mode must return an error when root unresolved");
4684 match err {
4685 EngineError::CheckPolicyStrict { context, message } => {
4686 assert!(
4687 context.contains("part file materialize"),
4688 "context must identify the call site: {context}"
4689 );
4690 assert!(
4691 message.contains("part file materialize"),
4692 "message must identify the part-file sink: {message}"
4693 );
4694 assert!(
4695 message.contains("no work_dir/project_root resolved"),
4696 "message must preserve the warn-log literal: {message}"
4697 );
4698 }
4699 other => panic!(
4700 "expected EngineError::CheckPolicyStrict, got a different variant: {other:?}"
4701 ),
4702 }
4703 }
4704
4705 /// A path-traversal `name` (`../evil.md`) with a
4706 /// resolved root — the name guard fails the write, but fail-open keeps
4707 /// the stage succeeding, and nothing is written outside the ctx dir.
4708 #[tokio::test]
4709 async fn stage_artifact_traversal_name_is_fail_open_and_writes_nothing() {
4710 let dir = tempfile::TempDir::new().unwrap();
4711 let (engine, _op, task_id, _worker_token) = seeded_task("planner").await;
4712 seed_agent_context(&engine, &task_id, 1, &dir.path().to_string_lossy()).await;
4713
4714 let result = engine
4715 .stage_worker_artifact_trusted(
4716 &task_id,
4717 1,
4718 "../evil.md".to_string(),
4719 serde_json::json!("pwned"),
4720 )
4721 .await;
4722 assert!(
4723 result.is_ok(),
4724 "default (Warn) policy is fail-open even on a rejected part name"
4725 );
4726 // The escaped target (ctx dir's parent) must not have been written.
4727 let escaped = dir
4728 .path()
4729 .join("workspace/tasks")
4730 .join(task_id.as_str())
4731 .join("evil.md");
4732 assert!(
4733 !escaped.exists(),
4734 "a traversal name must never write outside the ctx dir: {escaped:?}"
4735 );
4736 }
4737}
4738
4739/// GH #36 ST1: named multi-part worker output. Covers (a) the pure
4740/// `fold_final_and_parts` assembly `dispatch_attempt_with`'s Final-pull
4741/// delegates to, (b) `stage_worker_artifact_trusted`'s per-attempt
4742/// isolation on `EngineState.output_store` / `.worker_artifact_names` (the
4743/// same `HashMap<(StepId, u32), _>` key shape `submit_worker_result_trusted`
4744/// uses — a fresh attempt is a fresh key, so nothing to explicitly "clean
4745/// up"), and (c) the allowlist behavior that keeps a non-opt-in `Artifact`
4746/// producer (e.g. `AfterRunAuditMiddleware`) from being folded in.
4747#[cfg(test)]
4748mod named_multi_part_worker_output_tests {
4749 use super::*;
4750 use crate::worker::output::{ContentRef, OutputEvent};
4751
4752 fn artifact(name: &str, value: Value) -> OutputEvent {
4753 OutputEvent::Artifact {
4754 name: name.to_string(),
4755 content: ContentRef::Inline { value },
4756 }
4757 }
4758
4759 fn final_ev(value: Value, ok: bool) -> OutputEvent {
4760 OutputEvent::Final {
4761 content: ContentRef::Inline { value },
4762 ok,
4763 }
4764 }
4765
4766 fn names(list: &[&str]) -> Vec<String> {
4767 list.iter().map(|s| s.to_string()).collect()
4768 }
4769
4770 /// Two staged parts (both in `staged_names`) + a `Final` fold into
4771 /// `{"out", "parts"}`, each value carried through verbatim.
4772 #[test]
4773 fn fold_final_and_parts_assembles_out_and_parts_shape() {
4774 let tail = vec![
4775 artifact("summary", serde_json::json!("the summary")),
4776 artifact("diff", serde_json::json!({"lines": 3})),
4777 final_ev(serde_json::json!("final text"), true),
4778 ];
4779 let staged = names(&["summary", "diff"]);
4780 let (value, ok) = fold_final_and_parts(&tail, &staged).expect("Final present");
4781 assert!(ok);
4782 assert_eq!(
4783 value,
4784 serde_json::json!({
4785 "out": "final text",
4786 "parts": {
4787 "summary": "the summary",
4788 "diff": {"lines": 3},
4789 }
4790 })
4791 );
4792 }
4793
4794 /// Zero staged parts: the value is exactly the plain `Final` value — no
4795 /// `{"out", "parts"}` wrapping. This is the back-compat guarantee (GH
4796 /// #36 must not change the shape for a worker that never POSTs to
4797 /// `/v1/worker/artifact`).
4798 #[test]
4799 fn fold_final_and_parts_with_no_parts_returns_plain_final_value() {
4800 let tail = vec![final_ev(serde_json::json!("plain value"), true)];
4801 let (value, ok) = fold_final_and_parts(&tail, &[]).expect("Final present");
4802 assert!(ok);
4803 assert_eq!(value, serde_json::json!("plain value"));
4804 }
4805
4806 /// The same staged part `name` appearing twice in one attempt: the
4807 /// LATER (tail-order) value wins — `parts` is a `Map`, not an
4808 /// accumulating list.
4809 #[test]
4810 fn fold_final_and_parts_same_name_twice_last_write_wins() {
4811 let tail = vec![
4812 artifact("a", serde_json::json!("first")),
4813 artifact("a", serde_json::json!("second")),
4814 final_ev(serde_json::json!("f"), true),
4815 ];
4816 let staged = names(&["a"]);
4817 let (value, _ok) = fold_final_and_parts(&tail, &staged).expect("Final present");
4818 assert_eq!(
4819 value,
4820 serde_json::json!({"out": "f", "parts": {"a": "second"}})
4821 );
4822 }
4823
4824 /// No `Final` anywhere in the tail (only staged parts, e.g. the worker
4825 /// crashed before submitting) — `None`, the caller's pre-existing "no
4826 /// Final in output_tail" error path.
4827 #[test]
4828 fn fold_final_and_parts_returns_none_when_no_final_present() {
4829 let tail = vec![artifact("a", serde_json::json!("v"))];
4830 let staged = names(&["a"]);
4831 assert!(fold_final_and_parts(&tail, &staged).is_none());
4832 }
4833
4834 /// An `Artifact` on the tail whose name is NOT in `staged_names` (e.g.
4835 /// `AfterRunAuditMiddleware`'s `"audit:<step_ref>"` sidecar finding on
4836 /// an audited step's own tail) must NOT be folded into `"parts"` — the
4837 /// value stays the plain `Final` value, exactly the pre-GH-#36
4838 /// behavior for every producer that isn't the worker's own
4839 /// `/v1/worker/artifact` staging. This is the regression this fold was
4840 /// almost shipped without (see `dispatch_attempt_with`'s doc).
4841 #[test]
4842 fn fold_final_and_parts_ignores_artifacts_outside_the_staged_allowlist() {
4843 let tail = vec![
4844 final_ev(serde_json::json!({"echoed": "hi"}), true),
4845 artifact("audit:echo", serde_json::json!({"finding": "clean"})),
4846 ];
4847 // `staged_names` empty: the worker itself never staged anything —
4848 // the audit sidecar Artifact must be ignored.
4849 let (value, ok) = fold_final_and_parts(&tail, &[]).expect("Final present");
4850 assert!(ok);
4851 assert_eq!(value, serde_json::json!({"echoed": "hi"}));
4852 }
4853
4854 /// Mixed tail: one staged (allowlisted) part and one non-staged
4855 /// (audit-style) `Artifact` — only the staged one is folded in.
4856 #[test]
4857 fn fold_final_and_parts_folds_only_the_staged_subset_of_a_mixed_tail() {
4858 let tail = vec![
4859 artifact("summary", serde_json::json!("s")),
4860 artifact("audit:echo", serde_json::json!({"finding": "clean"})),
4861 final_ev(serde_json::json!("f"), true),
4862 ];
4863 let staged = names(&["summary"]);
4864 let (value, _ok) = fold_final_and_parts(&tail, &staged).expect("Final present");
4865 assert_eq!(
4866 value,
4867 serde_json::json!({"out": "f", "parts": {"summary": "s"}})
4868 );
4869 }
4870
4871 /// `stage_worker_artifact_trusted` writes onto the `(task_id, attempt)`
4872 /// key exactly like `submit_worker_result_trusted` does — a part staged
4873 /// under attempt N is invisible to an `output_tail` / allowlist read of
4874 /// attempt N+1 (a fresh attempt starts empty; nothing carries over).
4875 #[tokio::test]
4876 async fn stage_worker_artifact_trusted_is_isolated_per_attempt() {
4877 let engine = Engine::new(EngineCfg::default());
4878 let task_id = StepId::new();
4879
4880 engine
4881 .stage_worker_artifact_trusted(&task_id, 1, "a".to_string(), serde_json::json!("v1"))
4882 .await
4883 .expect("stage attempt 1");
4884
4885 let attempt_1_tail = engine.output_tail(&task_id, 1).await;
4886 assert_eq!(attempt_1_tail.len(), 1);
4887 assert!(matches!(
4888 &attempt_1_tail[0],
4889 OutputEvent::Artifact { name, .. } if name == "a"
4890 ));
4891 assert_eq!(
4892 engine.worker_artifact_names_for(&task_id, 1).await,
4893 vec!["a".to_string()]
4894 );
4895
4896 let attempt_2_tail = engine.output_tail(&task_id, 2).await;
4897 assert!(
4898 attempt_2_tail.is_empty(),
4899 "attempt 2 must not see attempt 1's staged part"
4900 );
4901 assert!(
4902 engine
4903 .worker_artifact_names_for(&task_id, 2)
4904 .await
4905 .is_empty(),
4906 "attempt 2's allowlist must not see attempt 1's staged name"
4907 );
4908 }
4909}
4910
4911// ─── GH #50 (Subtask 2): `Engine::register_verdict_contracts` /
4912// `Engine::verdict_contract_for_task` ────────────────────────────────────
4913#[cfg(test)]
4914mod verdict_contract_registry_tests {
4915 use super::*;
4916
4917 async fn seeded_engine(agent: &str) -> (Engine, StepId) {
4918 let engine = Engine::new(EngineCfg::default());
4919 let op_token = engine
4920 .attach("ut-op", Role::Operator, Duration::from_secs(30))
4921 .await
4922 .expect("attach");
4923 let task_id = engine
4924 .start_task(
4925 &op_token,
4926 TaskSpec {
4927 agent: agent.to_string(),
4928 initial_directive: serde_json::json!("x"),
4929 step_ctx: None,
4930 check_policy: None,
4931 },
4932 )
4933 .await
4934 .expect("start_task");
4935 (engine, task_id)
4936 }
4937
4938 /// An agent with no registered contract at all → `None` (the opt-in
4939 /// default; every pre-GH-#50 `Engine`).
4940 #[tokio::test]
4941 async fn returns_none_when_no_contract_registered_for_the_agent() {
4942 let (engine, task_id) = seeded_engine("gate").await;
4943 assert_eq!(engine.verdict_contract_for_task(&task_id).await, None);
4944 }
4945
4946 /// A registered contract for the running task's agent is returned
4947 /// verbatim.
4948 #[tokio::test]
4949 async fn returns_the_registered_contract_for_the_running_agent() {
4950 let (engine, task_id) = seeded_engine("gate").await;
4951 let contract = mlua_swarm_schema::VerdictContract {
4952 channel: mlua_swarm_schema::VerdictChannel::Body,
4953 values: vec!["PASS".to_string(), "BLOCKED".to_string()],
4954 };
4955 engine.register_verdict_contracts(HashMap::from([("gate".to_string(), contract.clone())]));
4956 assert_eq!(
4957 engine.verdict_contract_for_task(&task_id).await,
4958 Some(contract)
4959 );
4960 }
4961
4962 /// A registered contract for a DIFFERENT agent name never leaks onto
4963 /// an unrelated task.
4964 #[tokio::test]
4965 async fn does_not_leak_a_contract_registered_for_a_different_agent() {
4966 let (engine, task_id) = seeded_engine("gate").await;
4967 engine.register_verdict_contracts(HashMap::from([(
4968 "other-agent".to_string(),
4969 mlua_swarm_schema::VerdictContract {
4970 channel: mlua_swarm_schema::VerdictChannel::Body,
4971 values: vec!["PASS".to_string()],
4972 },
4973 )]));
4974 assert_eq!(engine.verdict_contract_for_task(&task_id).await, None);
4975 }
4976
4977 /// An unknown `task_id` → `None`, not a panic / error.
4978 #[tokio::test]
4979 async fn returns_none_for_an_unknown_task_id() {
4980 let engine = Engine::new(EngineCfg::default());
4981 let unknown = StepId::new();
4982 assert_eq!(engine.verdict_contract_for_task(&unknown).await, None);
4983 }
4984
4985 /// `register_verdict_contracts` is additive (`HashMap::extend`): a
4986 /// second call registering a DIFFERENT agent does not clobber the
4987 /// first call's entry.
4988 #[tokio::test]
4989 async fn register_verdict_contracts_is_additive_across_calls() {
4990 let (engine, task_id) = seeded_engine("gate").await;
4991 let contract = mlua_swarm_schema::VerdictContract {
4992 channel: mlua_swarm_schema::VerdictChannel::Part,
4993 values: vec!["ALLOW".to_string()],
4994 };
4995 engine.register_verdict_contracts(HashMap::from([("gate".to_string(), contract.clone())]));
4996 engine.register_verdict_contracts(HashMap::from([(
4997 "unrelated-agent".to_string(),
4998 mlua_swarm_schema::VerdictContract {
4999 channel: mlua_swarm_schema::VerdictChannel::Body,
5000 values: vec!["X".to_string()],
5001 },
5002 )]));
5003 assert_eq!(
5004 engine.verdict_contract_for_task(&task_id).await,
5005 Some(contract)
5006 );
5007 }
5008}
5009
5010// ─── GH #51: completion-time verdict-contract enforcement — the shared
5011// `Engine::verdict_contract_completion_check` choke point embedded inside
5012// `submit_worker_result_trusted` / `submit_output`, exercised here at the
5013// `submit_output` level (the WS Operator fallback route's own unit-test
5014// coverage — see `crates/mlua-swarm-server/tests/verdict_contract.rs` for
5015// the HTTP-round-trip coverage of the other 2 routes) ───────────────────
5016#[cfg(test)]
5017mod verdict_contract_completion_tests {
5018 use super::*;
5019
5020 /// Seeds a `Pending` task bound to `agent` and mints a bound
5021 /// `Role::Worker` token for it — the same mint-and-register pattern
5022 /// `initial_directive_value_passthrough_tests::mint_worker_token`
5023 /// uses (duplicated here: that helper is private to its own sibling
5024 /// `#[cfg(test)]` module, not reachable via `super::*` from this one).
5025 async fn seeded_task_with_worker_token(agent: &str) -> (Engine, CapToken, StepId) {
5026 let engine = Engine::new(EngineCfg::default());
5027 let op_token = engine
5028 .attach("ut-op", Role::Operator, Duration::from_secs(30))
5029 .await
5030 .expect("attach");
5031 let task_id = engine
5032 .start_task(
5033 &op_token,
5034 TaskSpec {
5035 agent: agent.to_string(),
5036 initial_directive: serde_json::json!("x"),
5037 step_ctx: None,
5038 check_policy: None,
5039 },
5040 )
5041 .await
5042 .expect("start_task");
5043 let worker_token = engine.signer().session(
5044 format!("worker-of-{task_id}"),
5045 Role::Worker,
5046 vec!["*".into()],
5047 Duration::from_secs(600),
5048 );
5049 let fp = worker_token.fingerprint();
5050 let record = CapTokenRecord::from_worker_token(worker_token.clone(), task_id.clone());
5051 engine
5052 .with_state("test.mint_worker", move |s| {
5053 s.tokens.insert(fp, record);
5054 })
5055 .await
5056 .expect("mint worker token");
5057 (engine, worker_token, task_id)
5058 }
5059
5060 fn body_contract(values: &[&str]) -> mlua_swarm_schema::VerdictContract {
5061 mlua_swarm_schema::VerdictContract {
5062 channel: mlua_swarm_schema::VerdictChannel::Body,
5063 values: values.iter().map(|v| v.to_string()).collect(),
5064 }
5065 }
5066
5067 fn part_contract(values: &[&str]) -> mlua_swarm_schema::VerdictContract {
5068 mlua_swarm_schema::VerdictContract {
5069 channel: mlua_swarm_schema::VerdictChannel::Part,
5070 values: values.iter().map(|v| v.to_string()).collect(),
5071 }
5072 }
5073
5074 fn final_event(value: Value, ok: bool) -> crate::worker::output::OutputEvent {
5075 crate::worker::output::OutputEvent::Final {
5076 content: crate::worker::output::ContentRef::Inline { value },
5077 ok,
5078 }
5079 }
5080
5081 /// Route 3 (WS Operator fallback, `submit_output` level) — a
5082 /// `channel: "part"` contract's attempt completes via a plain
5083 /// `Final` without ever staging a `"verdict"` artifact: rejected
5084 /// with `EngineError::VerdictPartMissing`, and nothing lands on
5085 /// `output_tail` — the rejected value never reaches the flow ctx.
5086 #[tokio::test]
5087 async fn submit_output_rejects_missing_verdict_part() {
5088 let (engine, token, task_id) = seeded_task_with_worker_token("gate").await;
5089 engine.register_verdict_contracts(HashMap::from([(
5090 "gate".to_string(),
5091 part_contract(&["PASS", "BLOCKED"]),
5092 )]));
5093
5094 let err = engine
5095 .submit_output(
5096 &token,
5097 &task_id,
5098 1,
5099 final_event(serde_json::json!("anything"), true),
5100 )
5101 .await
5102 .expect_err("missing staged verdict part must be rejected");
5103 assert!(
5104 matches!(err, EngineError::VerdictPartMissing { .. }),
5105 "unexpected error variant: {err:?}"
5106 );
5107
5108 let tail = engine.output_tail(&task_id, 1).await;
5109 assert!(
5110 !tail
5111 .iter()
5112 .any(|ev| matches!(ev, crate::worker::output::OutputEvent::Final { .. })),
5113 "a rejected completion must not write a Final onto output_tail"
5114 );
5115 }
5116
5117 /// Route 3 — a `channel: "part"` contract completes normally when the
5118 /// worker DID stage a matching `"verdict"` artifact first (defense in
5119 /// depth: presence AND membership both hold).
5120 #[tokio::test]
5121 async fn submit_output_accepts_when_verdict_part_is_staged_and_a_member() {
5122 let (engine, token, task_id) = seeded_task_with_worker_token("gate").await;
5123 engine.register_verdict_contracts(HashMap::from([(
5124 "gate".to_string(),
5125 part_contract(&["PASS", "BLOCKED"]),
5126 )]));
5127 engine
5128 .stage_worker_artifact_trusted(
5129 &task_id,
5130 1,
5131 "verdict".to_string(),
5132 serde_json::json!("PASS"),
5133 )
5134 .await
5135 .expect("stage verdict part");
5136
5137 engine
5138 .submit_output(
5139 &token,
5140 &task_id,
5141 1,
5142 final_event(serde_json::json!("full report"), true),
5143 )
5144 .await
5145 .expect("staged + member verdict part must be accepted");
5146
5147 let tail = engine.output_tail(&task_id, 1).await;
5148 assert!(
5149 tail.iter()
5150 .any(|ev| matches!(ev, crate::worker::output::OutputEvent::Final { .. })),
5151 "an accepted completion must write its Final onto output_tail"
5152 );
5153 }
5154
5155 /// Route 3 — a `channel: "body"` contract's completing value is NOT a
5156 /// member of `values`: rejected with
5157 /// `EngineError::VerdictValueRejected`, no `Final` written.
5158 #[tokio::test]
5159 async fn submit_output_rejects_body_value_outside_contract() {
5160 let (engine, token, task_id) = seeded_task_with_worker_token("gate").await;
5161 engine.register_verdict_contracts(HashMap::from([(
5162 "gate".to_string(),
5163 body_contract(&["PASS", "BLOCKED"]),
5164 )]));
5165
5166 let err = engine
5167 .submit_output(
5168 &token,
5169 &task_id,
5170 1,
5171 final_event(serde_json::json!("UNKNOWN"), true),
5172 )
5173 .await
5174 .expect_err("out-of-contract body value must be rejected");
5175 match err {
5176 EngineError::VerdictValueRejected { value, allowed } => {
5177 assert_eq!(value, "UNKNOWN");
5178 assert_eq!(allowed, vec!["PASS".to_string(), "BLOCKED".to_string()]);
5179 }
5180 other => panic!("unexpected error variant: {other:?}"),
5181 }
5182
5183 let tail = engine.output_tail(&task_id, 1).await;
5184 assert!(
5185 !tail
5186 .iter()
5187 .any(|ev| matches!(ev, crate::worker::output::OutputEvent::Final { .. })),
5188 "a rejected completion must not write a Final onto output_tail"
5189 );
5190 }
5191
5192 /// `ok=false` bypasses the completion-time check entirely, regardless
5193 /// of channel or membership — the exemption acceptance criterion,
5194 /// exercised at the `submit_output` choke point.
5195 #[tokio::test]
5196 async fn submit_output_ok_false_bypasses_the_check() {
5197 let (engine, token, task_id) = seeded_task_with_worker_token("gate").await;
5198 engine.register_verdict_contracts(HashMap::from([(
5199 "gate".to_string(),
5200 body_contract(&["PASS", "BLOCKED"]),
5201 )]));
5202
5203 engine
5204 .submit_output(
5205 &token,
5206 &task_id,
5207 1,
5208 final_event(serde_json::json!("UNKNOWN"), false),
5209 )
5210 .await
5211 .expect("ok=false must bypass the verdict contract check entirely");
5212
5213 let tail = engine.output_tail(&task_id, 1).await;
5214 assert!(
5215 tail.iter()
5216 .any(|ev| matches!(ev, crate::worker::output::OutputEvent::Final { .. })),
5217 "an ok=false completion is exempt, not rejected — its Final must still land"
5218 );
5219 }
5220
5221 /// `staged_verdict_value_for` mirrors `fold_final_and_parts`'s
5222 /// last-write-wins semantics: staging `"verdict"` twice within the
5223 /// same attempt returns the LAST value, not the first.
5224 #[tokio::test]
5225 async fn staged_verdict_value_for_is_last_write_wins() {
5226 let (engine, _token, task_id) = seeded_task_with_worker_token("gate").await;
5227 engine
5228 .stage_worker_artifact_trusted(
5229 &task_id,
5230 1,
5231 "verdict".to_string(),
5232 serde_json::json!("PASS"),
5233 )
5234 .await
5235 .expect("stage first verdict part");
5236 engine
5237 .stage_worker_artifact_trusted(
5238 &task_id,
5239 1,
5240 "verdict".to_string(),
5241 serde_json::json!("BLOCKED"),
5242 )
5243 .await
5244 .expect("stage second verdict part");
5245
5246 assert_eq!(
5247 engine.staged_verdict_value_for(&task_id, 1).await,
5248 Some("BLOCKED".to_string())
5249 );
5250 }
5251
5252 /// `staged_verdict_value_for` ignores artifacts staged under any name
5253 /// OTHER than the literal `"verdict"` — mirrors `channel: "part"`
5254 /// contracts only ever addressing that one part.
5255 #[tokio::test]
5256 async fn staged_verdict_value_for_ignores_other_artifact_names() {
5257 let (engine, _token, task_id) = seeded_task_with_worker_token("gate").await;
5258 engine
5259 .stage_worker_artifact_trusted(
5260 &task_id,
5261 1,
5262 "notes".to_string(),
5263 serde_json::json!("irrelevant"),
5264 )
5265 .await
5266 .expect("stage unrelated part");
5267
5268 assert_eq!(engine.staged_verdict_value_for(&task_id, 1).await, None);
5269 }
5270
5271 /// `staged_verdict_value_for` → `None` when nothing was ever staged —
5272 /// the normal case the completion check turns into
5273 /// `EngineError::VerdictPartMissing`.
5274 #[tokio::test]
5275 async fn staged_verdict_value_for_returns_none_when_nothing_staged() {
5276 let (engine, _token, task_id) = seeded_task_with_worker_token("gate").await;
5277 assert_eq!(engine.staged_verdict_value_for(&task_id, 1).await, None);
5278 }
5279}