mlua_swarm/core/engine.rs
1//! `Engine` — the long-running stateful runtime plus the `with_state`
2//! helper (R1-R4 discipline).
3//!
4//! The engine owns the Domain side of the Data / Domain split:
5//! flow control (dispatch / verdict), state (`EngineState`), and the
6//! `submit_output` / `output_tail` surface that feeds it. Data-plane
7//! traffic (Big Response bodies) is delegated to the `output_store` module
8//! plus its paired `SpawnerLayer`s and passes through here without the
9//! engine core needing to grow.
10
11use crate::core::agent_context::{RUN_ID_KEY, STEP_CTX_KEY};
12use crate::core::config::EngineCfg;
13use crate::core::ctx::{Ctx, OperatorInfo, OperatorKind, SeniorBridge, SpawnHook};
14use crate::core::errors::EngineError;
15use crate::core::state::{
16 CapTokenRecord, DispatchOutcome, EngineState, Event, EventStream, OperatorSession, ResumeKey,
17 ResumePending, TaskSpec, TaskState, TaskStatus,
18};
19use crate::store::replay::{hash_input_value, ReplayEntry};
20use crate::store::run::RunContext;
21use crate::types::{
22 default_role_verb_table, now_unix, CapToken, Role, RoleVerbGate, RunId, SessionId, StepId,
23 TokenSigner, Verb,
24};
25use crate::worker::adapter::SpawnerAdapter;
26use serde_json::Value;
27use std::collections::HashMap;
28use std::sync::Arc;
29use std::time::{Duration, Instant};
30use tokio::sync::{broadcast, Mutex};
31
32/// Process-wide long-running runtime. Cheap to `clone()` — an `Arc`
33/// lives inside.
34#[derive(Clone)]
35pub struct Engine {
36 inner: Arc<EngineInner>,
37}
38
39struct EngineInner {
40 state: Mutex<EngineState>,
41 cfg: EngineCfg,
42 signer: TokenSigner,
43 gate: RoleVerbGate,
44 event_tx: broadcast::Sender<Event>,
45 /// ID-keyed bridge registry (register-by-ID design). `SeniorBridge`
46 /// and `SpawnHook` are registered by ID; sessions bind to those IDs
47 /// only. Persistence stores just the ID, and on reattach the caller
48 /// re-registers under the same ID to restore presence.
49 senior_bridges: tokio::sync::RwLock<HashMap<String, Arc<dyn SeniorBridge>>>,
50 spawn_hooks: tokio::sync::RwLock<HashMap<String, Arc<dyn SpawnHook>>>,
51 /// ID registry for full-spawn Operator backends (backends that take the
52 /// entire spawn via `execute`). Sibling to `senior_bridges` /
53 /// `spawn_hooks`. `OperatorDelegateMiddleware` looks these up via
54 /// `ctx` and, when `kind = MainAi` / `Composite`, bypasses
55 /// `inner.spawn` and calls `operator.execute` instead.
56 operators: tokio::sync::RwLock<HashMap<String, Arc<dyn crate::operator::Operator>>>,
57 /// Base and hint layer factories for the `SpawnerStack`. At
58 /// `service::linker::link` time, `compiled.router` is wrapped with
59 /// the base factories plus the hint factories resolved from
60 /// `blueprint.spawner_hints.layers`. This is the engine-side
61 /// counterpart to the discipline "Flow / Blueprint doesn't spell out
62 /// middleware implementations — it declares the capabilities it needs
63 /// as hint keys".
64 layer_registry: crate::middleware::LayerRegistry,
65 /// Optional Data-plane `OutputStore` backend (subtask-4 / ST2 rework —
66 /// see `submit_output`'s doc). `None` (the default) preserves
67 /// pre-subtask-4 behavior exactly: `submit_output` /
68 /// `submit_worker_result_trusted` only touch the Domain-plane
69 /// `EngineState.output_store` HashMap, same as before this was added.
70 /// `Some` additionally dual-writes every `Final` event into this store
71 /// via [`crate::store::output::OutputStore::append`], making it
72 /// queryable (e.g. by `mlua-swarm-server`'s `GET /v1/tasks/:id/ctx`)
73 /// even for an in-flight run. A plain `std::sync::RwLock` (not
74 /// `tokio::sync::RwLock`) — set once at boot via [`Engine::set_output_store`]
75 /// from a synchronous call site (`mlua-swarm-server`'s router builder),
76 /// then only ever briefly read (clone the `Option<Arc<..>>`, never held
77 /// across an `.await`) from the async submit path.
78 data_store: std::sync::RwLock<Option<Arc<dyn crate::store::output::OutputStore>>>,
79 /// GH #50 (Subtask 2 — runtime plumbing): agent name → declared
80 /// [`mlua_swarm_schema::VerdictContract`], the Engine-side registry
81 /// [`Self::verdict_contract_for_task`] resolves against. Populated via
82 /// [`Self::register_verdict_contracts`] — same sync-`RwLock`,
83 /// set-outside-the-lock idiom as `data_store` above. Empty by default
84 /// (every pre-GH-#50 `Engine`), which is exactly the opt-in "no
85 /// contract declared" state `verdict_contract_for_task` treats as
86 /// `None`. Populated from a live `Compiler::compile`'s
87 /// `CompiledAgentTable.verdict_contracts` output by
88 /// `TaskLaunchService::launch`, immediately after `compiler.compile`
89 /// succeeds — see [`Self::register_verdict_contracts`]'s doc for the
90 /// overwrite semantics of that merge.
91 verdict_contracts: std::sync::RwLock<HashMap<String, mlua_swarm_schema::VerdictContract>>,
92}
93
94/// Renders a `TaskSpec.initial_directive` / `EngineState.prompts`
95/// `Value` down to the `String` shape that string-consuming boundaries
96/// require (issue #18). Strings pass through verbatim; anything else
97/// (Object / Array / Number / Bool / Null) is serde-stringified. This
98/// is the single canonical rendering — the coercion that used to sit
99/// inside `EngineDispatcher::dispatch` moved here and is invoked only
100/// at consumer boundaries: `WorkerPayload.prompt` (HTTP
101/// `/v1/worker/prompt`), `WorkerInvocation.prompt` (in-process
102/// spawners), the subprocess spawner's directive arg/stdin, and the
103/// WS Spawn frame text render (`operator_ws::session`). Everything
104/// upstream (Blueprint dispatch → engine state → `fetch_prompt` →
105/// `Operator::execute`) keeps the `Value` end-to-end.
106pub(crate) fn render_directive_to_string(v: &Value) -> String {
107 match v {
108 Value::String(s) => s.clone(),
109 other => other.to_string(),
110 }
111}
112
113/// Renders a [`crate::worker::output::ContentRef`] down to the `Value` shape
114/// the BP-chain / `DispatchOutcome` consume. `Inline` passes its `value`
115/// through verbatim; `FileRef` is stringified into the same
116/// `{"file_ref", "mime", "size_hint"}` shape `materialize_final_submission`
117/// uses for its own file-materialize projection — one canonical
118/// stringification, not two independently-maintained copies (GH #36 ST1:
119/// shared by both the `Final`-pull and the `Artifact`-parts fold in
120/// [`Engine::dispatch_attempt_with`]'s doc).
121fn content_ref_to_value(content: crate::worker::output::ContentRef) -> Value {
122 match content {
123 crate::worker::output::ContentRef::Inline { value } => value,
124 crate::worker::output::ContentRef::FileRef {
125 path,
126 mime,
127 size_hint,
128 } => serde_json::json!({
129 "file_ref": path.to_string_lossy(),
130 "mime": mime,
131 "size_hint": size_hint,
132 }),
133 }
134}
135
136/// GH #51 — reduces a [`content_ref_to_value`] result down to the `String`
137/// shape the completion-time verdict-contract check compares against a
138/// declared `VerdictContract.values` token set. A `Value::String` unwraps
139/// to its raw contents (no surrounding JSON quotes) — this mirrors the
140/// pre-GH-#51 `check_verdict_contract` (`mlua-swarm-server`'s
141/// `worker.rs`), which always compared the raw submitted body string
142/// directly, never a JSON-stringified copy. Any OTHER `Value` shape
143/// (`Number` / `Object` / `Array` / `Bool` / `Null` — i.e. a `channel:
144/// "body"` contract whose completing value is not a string at all, or a
145/// `FileRef` content whose `content_ref_to_value` projection is an
146/// object) falls back to `Value::to_string()`'s JSON-encoded form: it can
147/// never collide with a plain declared token like `"PASS"`, so it
148/// naturally fails membership — consistent with the "non-string values
149/// under a body contract are violations" rule (issue #51's Proposal).
150fn content_ref_to_comparable_string(content: crate::worker::output::ContentRef) -> String {
151 let value = content_ref_to_value(content);
152 match value {
153 Value::String(s) => s,
154 other => other.to_string(),
155 }
156}
157
158/// [`Engine::dispatch_attempt_with`]'s Final-pull assembly (GH #36 ST1:
159/// named multi-part worker output), factored out as a pure function of the
160/// output-event tail so it is unit-testable without a live `Engine` /
161/// spawner.
162///
163/// Finds the LAST `Final` event in `tail` (mirrors the pre-GH-#36 pull:
164/// "last Final wins" if more than one was ever appended) and folds every
165/// `Artifact` event in the SAME tail WHOSE NAME APPEARS IN `staged_names`
166/// into a `"parts"` object keyed by `Artifact.name` — walked in tail (=
167/// event-append) order, so a name staged more than once within the attempt
168/// is last-write-wins (`Map` insert semantics, not an accumulating list;
169/// `Engine::stage_worker_artifact_trusted`'s doc). `staged_names` is the
170/// WORKER's own opt-in allowlist (`EngineState.worker_artifact_names`'s
171/// doc) — an `Artifact` on the tail whose name is NOT in `staged_names`
172/// (e.g. `AfterRunAuditMiddleware`'s `"audit:<step_ref>"` sidecar finding)
173/// is left alone, exactly as before GH #36; this is what keeps an audited
174/// step's BP-chain value byte-identical when the worker itself never
175/// staged a part.
176///
177/// At least one matching part: the returned value is `{"out": <final
178/// value>, "parts": {<name>: <value>, ...}}`. Zero matching parts: the
179/// returned value is the plain final value, unchanged from the pre-GH-#36
180/// shape — this is the back-compat guarantee, not an incidental default.
181///
182/// `None` when `tail` carries no `Final` at all (the caller's pre-existing
183/// "no Final in output_tail" error path).
184fn fold_final_and_parts(
185 tail: &[crate::worker::output::OutputEvent],
186 staged_names: &[String],
187) -> Option<(Value, bool)> {
188 let (final_content, ok) = tail.iter().rev().find_map(|ev| match ev {
189 crate::worker::output::OutputEvent::Final { content, ok } => Some((content.clone(), *ok)),
190 _ => None,
191 })?;
192 let final_value = content_ref_to_value(final_content);
193
194 let mut parts = serde_json::Map::new();
195 for ev in tail {
196 if let crate::worker::output::OutputEvent::Artifact { name, content } = ev {
197 if staged_names.iter().any(|staged| staged == name) {
198 parts.insert(name.clone(), content_ref_to_value(content.clone()));
199 }
200 }
201 }
202
203 let value = if parts.is_empty() {
204 final_value
205 } else {
206 serde_json::json!({ "out": final_value, "parts": Value::Object(parts) })
207 };
208 Some((value, ok))
209}
210
211impl Engine {
212 /// Backwards-compatible constructor that starts the engine without a
213 /// layer registry, preserving the signature already used by ~88
214 /// existing call sites. Use this when automatic middleware wrapping
215 /// at bind time is not needed. Callers such as `mlua-swarm-server` go through
216 /// `new_with_layers(cfg, registry)` to enable the hint-resolution path.
217 pub fn new(cfg: EngineCfg) -> Self {
218 Self::new_with_layers(cfg, crate::middleware::LayerRegistry::new())
219 }
220
221 /// Construct an `Engine` with an explicit `LayerRegistry`, enabling
222 /// hint-resolution: `spawner_hints.layers` declared on a `Blueprint`
223 /// are resolved against this registry when the spawner stack is bound
224 /// at `service::linker::link` time.
225 pub fn new_with_layers(
226 cfg: EngineCfg,
227 layer_registry: crate::middleware::LayerRegistry,
228 ) -> Self {
229 let (event_tx, _) = broadcast::channel(256);
230 let signer = TokenSigner::new(&cfg.token_secret);
231 Self {
232 inner: Arc::new(EngineInner {
233 state: Mutex::new(EngineState::new()),
234 cfg,
235 signer,
236 gate: default_role_verb_table(),
237 event_tx,
238 senior_bridges: tokio::sync::RwLock::new(HashMap::new()),
239 spawn_hooks: tokio::sync::RwLock::new(HashMap::new()),
240 operators: tokio::sync::RwLock::new(HashMap::new()),
241 layer_registry,
242 data_store: std::sync::RwLock::new(None),
243 verdict_contracts: std::sync::RwLock::new(HashMap::new()),
244 }),
245 }
246 }
247
248 /// Rebuild this `Engine` with a different `RoleVerbGate`. The gate is
249 /// treated as fixed-at-build-time, so this constructs a fresh
250 /// `EngineInner` (fresh empty `EngineState`) rather than mutating in
251 /// place — mainly a testing convenience for swapping gate rules.
252 pub fn with_gate(self, gate: RoleVerbGate) -> Self {
253 // The gate is fixed at build time — the intent is to build a fresh
254 // instance rather than mutating in place. As a testing convenience we
255 // do allow swapping the inner Arc. Simpler form: just rebuild
256 // Arc<EngineInner>.
257 let inner = Arc::new(EngineInner {
258 state: Mutex::new(EngineState::new()),
259 cfg: self.inner.cfg.clone(),
260 signer: self.inner.signer.clone(),
261 gate,
262 event_tx: self.inner.event_tx.clone(),
263 senior_bridges: tokio::sync::RwLock::new(HashMap::new()),
264 spawn_hooks: tokio::sync::RwLock::new(HashMap::new()),
265 operators: tokio::sync::RwLock::new(HashMap::new()),
266 layer_registry: self.inner.layer_registry.clone(),
267 data_store: std::sync::RwLock::new(None),
268 verdict_contracts: std::sync::RwLock::new(HashMap::new()),
269 });
270 Self { inner }
271 }
272
273 // ═══════════════════════════════════════════════════════════════════════
274 // Accessors. Production code drives execution through compile +
275 // `service::linker::link` + `dispatch_attempt_with(spawner)` inside
276 // `TaskLaunchService`; `Engine` itself is a pure execution surface — it
277 // does not own a BlueprintStore / EnhanceAdapter / Compiler, nor a
278 // global spawner (the spawner is carried per-request, never stashed on
279 // the engine).
280 // ═══════════════════════════════════════════════════════════════════════
281
282 /// Access the `EngineCfg` this engine was built with.
283 pub fn cfg(&self) -> &EngineCfg {
284 &self.inner.cfg
285 }
286
287 /// Expose the internal `LayerRegistry` — used when deriving a
288 /// sub-engine that needs the same registry re-injected. The
289 /// per-request sub-engine in `mlua-swarm-server` reads the parent engine's
290 /// registry through this accessor and passes it to
291 /// `Engine::new_with_layers(cfg, parent.layer_registry().clone())`.
292 pub fn layer_registry(&self) -> &crate::middleware::LayerRegistry {
293 &self.inner.layer_registry
294 }
295
296 /// Access the `TokenSigner` used to mint/verify `CapToken`s.
297 pub fn signer(&self) -> &TokenSigner {
298 &self.inner.signer
299 }
300
301 /// Clone a handle to the process-wide `Event` broadcast sender. Prefer
302 /// `subscribe` for a ready-to-use receiver.
303 pub fn event_tx(&self) -> broadcast::Sender<Event> {
304 self.inner.event_tx.clone()
305 }
306
307 /// Subscribe to the engine's `Event` broadcast stream.
308 pub fn subscribe(&self) -> EventStream {
309 self.inner.event_tx.subscribe()
310 }
311
312 /// Wires the Data-plane [`crate::store::output::OutputStore`] backend
313 /// used by `submit_output` / `submit_worker_result_trusted`'s
314 /// submit-time projection sink (subtask-4 / ST2 rework — see
315 /// `submit_output`'s doc). Synchronous (a plain `std::sync::RwLock`
316 /// write) so a caller can wire it up at boot from a non-`async`
317 /// context (`mlua-swarm-server`'s router builder passes the same
318 /// `Arc` it hands to its `AppState.data_store`, so `POST
319 /// /v1/data/emit` and every worker's ordinary `/v1/worker/submit` land
320 /// in the one store). Calling this more than once replaces the
321 /// previous backend; not calling it at all (the default) preserves
322 /// pre-subtask-4 behavior exactly — `submit_output` only touches the
323 /// Domain-plane `EngineState.output_store` HashMap.
324 pub fn set_output_store(&self, store: Arc<dyn crate::store::output::OutputStore>) {
325 let mut guard = self
326 .inner
327 .data_store
328 .write()
329 .unwrap_or_else(|poisoned| poisoned.into_inner());
330 *guard = Some(store);
331 }
332
333 /// Clones the currently-wired Data-plane store handle, if any. Kept
334 /// private and side-effect-free (no lock held past this call) —
335 /// callers (`materialize_final_submission`) do their actual `.append`
336 /// work outside of any lock.
337 fn output_store_backend(&self) -> Option<Arc<dyn crate::store::output::OutputStore>> {
338 self.inner
339 .data_store
340 .read()
341 .unwrap_or_else(|poisoned| poisoned.into_inner())
342 .clone()
343 }
344
345 /// GH #50 (Subtask 2): merges `contracts` (agent name → declared
346 /// [`mlua_swarm_schema::VerdictContract`]) into the engine's runtime
347 /// verdict-contract registry, later resolved per-task by
348 /// [`Self::verdict_contract_for_task`]. Same sync-write idiom as
349 /// [`Self::set_output_store`] — a plain `std::sync::RwLock` write, so
350 /// this can be called from a non-`async` context. Production call
351 /// site: `TaskLaunchService::launch`, immediately after a successful
352 /// `Compiler::compile`, passing `compiled.router.verdict_contracts.clone()`.
353 ///
354 /// # Overwrite semantics (explicit — read before adding a second call site)
355 ///
356 /// The registry is a single flat `HashMap` **keyed by agent name only**
357 /// (`String`), with process-wide (not per-task, not per-Blueprint,
358 /// not per-launch) scope. Registration is additive via
359 /// `HashMap::extend`: an entry for an agent name NOT already present is
360 /// added; an entry for an agent name ALREADY present is REPLACED
361 /// (last write wins) by the incoming one. Concretely: launching a
362 /// second Blueprint that also declares a `verdict` contract for an
363 /// agent named `"gate"` OVERWRITES whatever contract a first, still
364 /// in-flight, launch registered for an agent of that same name — even
365 /// if the two Blueprints intend it as two semantically different
366 /// agents that merely share a name, and even while the first launch's
367 /// tasks are still running. This is a **known limitation** of the v1
368 /// design; a per-task (or per-`RunId` / per-Blueprint) scoped registry
369 /// is a possible follow-up if two concurrently in-flight Blueprints
370 /// declaring conflicting contracts under the same agent name turns out
371 /// to matter in practice. Calling this with an empty map (or not at
372 /// all — the default) is a no-op, preserving pre-GH-#50 behavior
373 /// exactly (opt-in).
374 pub fn register_verdict_contracts(
375 &self,
376 contracts: HashMap<String, mlua_swarm_schema::VerdictContract>,
377 ) {
378 let mut guard = self
379 .inner
380 .verdict_contracts
381 .write()
382 .unwrap_or_else(|poisoned| poisoned.into_inner());
383 guard.extend(contracts);
384 }
385
386 /// GH #50 (Subtask 2): the declared
387 /// [`mlua_swarm_schema::VerdictContract`] for the agent currently
388 /// running `task_id`, if any. Resolves `task_id` → `TaskState.spec.agent`
389 /// (via `EngineState.tasks`, the same lookup [`Self::task_attempt`]
390 /// performs) and looks that agent name up in the registry
391 /// [`Self::register_verdict_contracts`] populates.
392 ///
393 /// `None` in both of these cases — deliberately collapsed to the same
394 /// value, mirroring [`Self::agent_context_for`]'s `Result`-into-`Option`
395 /// pattern (`.ok().flatten()`; a lookup failure here is never itself an
396 /// error worth surfacing to a caller):
397 /// - `task_id` is unknown (no `TaskState` for it).
398 /// - `task_id` resolves to a known agent, but that agent declared no
399 /// `verdict` contract (the opt-in default).
400 ///
401 /// Callers (`mlua-swarm-server`'s `worker_submit` / `worker_artifact`)
402 /// treat every `None` identically: skip the submit-time verdict gate
403 /// entirely, preserving pre-GH-#50 behavior byte-for-byte.
404 pub async fn verdict_contract_for_task(
405 &self,
406 task_id: &StepId,
407 ) -> Option<mlua_swarm_schema::VerdictContract> {
408 let tid = task_id.clone();
409 let agent = self
410 .with_state("verdict_contract_for_task", move |s| {
411 s.tasks.get(&tid).map(|t| t.spec.agent.clone())
412 })
413 .await
414 .ok()
415 .flatten()?;
416 self.inner
417 .verdict_contracts
418 .read()
419 .unwrap_or_else(|poisoned| poisoned.into_inner())
420 .get(&agent)
421 .cloned()
422 }
423
424 /// GH #51 — the value of the LAST staged `"verdict"` `Artifact` for
425 /// `(task_id, attempt)`, if any. Mirrors [`fold_final_and_parts`]'s
426 /// reverse-scan-of-`output_tail` pattern (last-write-wins per name,
427 /// same as that fold and [`Self::stage_worker_artifact_trusted`]'s
428 /// doc), narrowed to the single literal artifact name
429 /// `channel: "part"` contracts address (Pattern B — see
430 /// `blueprint-authoring.md`'s "Returning verdicts to drive BP flow").
431 ///
432 /// Infallible accessor: `None` is the normal "nothing staged yet"
433 /// case, not an error — the caller
434 /// ([`Self::verdict_contract_completion_check`]) is what converts
435 /// `None` into `Err(EngineError::VerdictPartMissing)`.
436 pub(crate) async fn staged_verdict_value_for(
437 &self,
438 task_id: &StepId,
439 attempt: u32,
440 ) -> Option<String> {
441 let tail = self.output_tail(task_id, attempt).await;
442 tail.iter().rev().find_map(|ev| match ev {
443 crate::worker::output::OutputEvent::Artifact { name, content } if name == "verdict" => {
444 Some(content_ref_to_comparable_string(content.clone()))
445 }
446 _ => None,
447 })
448 }
449
450 /// GH #51 — the single completion-time verdict-contract choke point,
451 /// embedded inside BOTH [`Self::submit_worker_result_trusted`] and
452 /// [`Self::submit_output`] (the two engine-side writes every HTTP/WS
453 /// completion route ultimately passes through). Not duplicated per
454 /// route handler — a future 4th completion route is gated for free
455 /// as long as it funnels through one of those two functions.
456 ///
457 /// `ok=false` is exempt on every route (this single early-return IS
458 /// the exemption, reused identically by both embedding sites — see
459 /// issue #51's "ok=false completions are exempt" acceptance
460 /// criterion). An agent with no declared contract, or a contract for
461 /// the OTHER channel, is untouched (`Ok(())`) — same opt-in,
462 /// byte-for-byte-preserving posture as
463 /// [`Self::verdict_contract_for_task`]'s doc.
464 ///
465 /// - `channel: "body"` — `value` (the completing `Final`'s content,
466 /// already reduced to a comparable string by the caller via
467 /// [`content_ref_to_comparable_string`]) must be a member of
468 /// `contract.values`.
469 /// - `channel: "part"` — [`Self::staged_verdict_value_for`] must find
470 /// a staged `"verdict"` artifact for this attempt (presence,
471 /// defense in depth over the staging-time membership check) AND its
472 /// value must be a member of `contract.values`.
473 async fn verdict_contract_completion_check(
474 &self,
475 task_id: &StepId,
476 attempt: u32,
477 ok: bool,
478 value: &str,
479 ) -> Result<(), EngineError> {
480 if !ok {
481 return Ok(());
482 }
483 let Some(contract) = self.verdict_contract_for_task(task_id).await else {
484 return Ok(());
485 };
486 match contract.channel {
487 mlua_swarm_schema::VerdictChannel::Body => {
488 if contract.values.iter().any(|v| v == value) {
489 Ok(())
490 } else {
491 Err(EngineError::VerdictValueRejected {
492 value: value.to_string(),
493 allowed: contract.values.clone(),
494 })
495 }
496 }
497 mlua_swarm_schema::VerdictChannel::Part => {
498 match self.staged_verdict_value_for(task_id, attempt).await {
499 None => Err(EngineError::VerdictPartMissing {
500 allowed: contract.values.clone(),
501 }),
502 Some(staged) if contract.values.iter().any(|v| v == &staged) => Ok(()),
503 Some(staged) => Err(EngineError::VerdictValueRejected {
504 value: staged,
505 allowed: contract.values.clone(),
506 }),
507 }
508 }
509 }
510 }
511
512 // ═══════════════════════════════════════════════════════════════════════
513 // §7 with_state — single Mutex + R1-R4 (try_lock + bounded retry + max-hold panic)
514 // ═══════════════════════════════════════════════════════════════════════
515
516 /// The closure is a **sync** `FnOnce` — you cannot pass an async
517 /// closure, which enforces R3 at the type level. Exceeding `max_hold`
518 /// panics so that R4 violations surface immediately.
519 pub async fn with_state<F, R>(&self, op: &'static str, f: F) -> Result<R, EngineError>
520 where
521 F: FnOnce(&mut EngineState) -> R,
522 {
523 let cfg = &self.inner.cfg;
524
525 // R2: try_lock + bounded retry
526 let mut guard_opt = None;
527 for attempt in 0..=cfg.max_retry {
528 match self.inner.state.try_lock() {
529 Ok(g) => {
530 guard_opt = Some(g);
531 break;
532 }
533 Err(_) if cfg.try_only => return Err(EngineError::LockBusy(op)),
534 Err(_) => {
535 let backoff = cfg.backoff_ms_step * (attempt as u64 + 1);
536 tokio::time::sleep(Duration::from_millis(backoff)).await;
537 }
538 }
539 }
540 let mut guard = guard_opt.ok_or(EngineError::LockBusyAfterRetry(op))?;
541
542 // R4: max_hold guard
543 let start = Instant::now();
544 let result = f(&mut guard);
545 let elapsed_ms = start.elapsed().as_millis();
546 drop(guard);
547
548 if elapsed_ms > cfg.max_hold_ms {
549 panic!(
550 "Engine.with_state('{op}') held {elapsed_ms}ms > max {}ms — suspected R3 violation (long op inside lock)",
551 cfg.max_hold_ms
552 );
553 }
554 Ok(result)
555 }
556
557 // ═══════════════════════════════════════════════════════════════════════
558 // Token verify (= sig + expire + gate + uses_left)
559 // ═══════════════════════════════════════════════════════════════════════
560
561 /// Four steps: (1) signature verify, (2) expiry check, (3) role × verb
562 /// gate, (4) `uses_left` consume.
563 pub async fn verify_token(&self, token: &CapToken, verb: Verb) -> Result<(), EngineError> {
564 // (1) sig
565 if !self.inner.signer.verify_sig(token) {
566 return Err(EngineError::BadSignature);
567 }
568 // (2) expire
569 if token.is_expired(now_unix()) {
570 return Err(EngineError::TokenExpired);
571 }
572 // (3) role × verb gate
573 if !self.inner.gate.is_allowed(token.role, verb) {
574 return Err(EngineError::RoleViolation {
575 role: token.role,
576 verb,
577 });
578 }
579 // (4) server-side uses_left consume
580 let fp = token.fingerprint();
581 self.with_state("token.consume", move |s| {
582 let rec = s
583 .tokens
584 .get_mut(&fp)
585 .ok_or_else(|| EngineError::TokenNotFound(fp.clone()))?;
586 rec.consume()
587 .map_err(|_: crate::core::state::CapTokenConsumeError| {
588 EngineError::TokenUsesExhausted
589 })?;
590 Ok::<(), EngineError>(())
591 })
592 .await??;
593 Ok(())
594 }
595
596 /// `verify_token` plus the **task-ownership gate**.
597 ///
598 /// When a Worker-role token calls a state-touch verb (`fetch_prompt` /
599 /// `post_result` / `read_task_state` / `cancel_task` / `poll_task`),
600 /// the gate checks that `CapTokenRecord.task_id` matches the argument
601 /// `task_id`; a mismatch returns `EngineError::TokenTaskMismatch`.
602 /// Operator / Senior / Observer tokens are outside the ownership gate
603 /// and may touch any task.
604 ///
605 /// **Verbs exempt from the gate.** `start_task` and `dispatch_attempt`
606 /// stay outside so recursive swarming keeps working; depth is capped
607 /// by `max_spawn_depth`.
608 pub async fn verify_token_for_task(
609 &self,
610 token: &CapToken,
611 verb: Verb,
612 task_id: &StepId,
613 ) -> Result<(), EngineError> {
614 self.verify_token(token, verb).await?;
615 if token.role != Role::Worker {
616 return Ok(());
617 }
618 let fp = token.fingerprint();
619 let arg_tid = task_id.clone();
620 self.with_state("token.ownership_gate", move |s| {
621 let bound = s.tokens.get(&fp).and_then(|r| r.task_id.as_ref()).cloned();
622 match bound {
623 Some(t) if t == arg_tid => Ok(()),
624 Some(t) => Err(EngineError::TokenTaskMismatch {
625 bound: t.into_string(),
626 arg: arg_tid.into_string(),
627 }),
628 None => Err(EngineError::TokenNotFound(fp.clone())),
629 }
630 })
631 .await??;
632 Ok(())
633 }
634
635 /// Resolve the bound `task_id` from a Worker-role token. Used on the
636 /// simple `/v1/worker/submit` endpoint, where the worker POSTs with a
637 /// token but no `task_id`. Returns `Err` if the token role is not
638 /// Worker, or if no bound task is set.
639 pub async fn task_id_from_token(&self, token: &CapToken) -> Result<StepId, EngineError> {
640 if token.role != Role::Worker {
641 return Err(EngineError::RoleViolation {
642 role: token.role,
643 verb: Verb::PostResult,
644 });
645 }
646 let fp = token.fingerprint();
647 self.with_state("task_id_from_token", move |s| {
648 s.tokens
649 .get(&fp)
650 .and_then(|r| r.task_id.as_ref())
651 .cloned()
652 .ok_or_else(|| EngineError::TokenNotFound(fp.clone()))
653 })
654 .await?
655 }
656
657 /// Resolve a short worker handle (`wh-XXXXXXXX`) to the bound
658 /// `task_id`. Used on `/v1/worker/submit` when the Bearer is a short
659 /// handle string rather than a full `CapToken` JSON. A missing entry
660 /// returns `TokenNotFound`, i.e. "the handle is not in the store".
661 pub async fn task_id_from_handle(&self, handle: &str) -> Result<StepId, EngineError> {
662 let h = handle.to_string();
663 self.with_state("task_id_from_handle", move |s| {
664 let fp = s
665 .worker_handles
666 .get(&h)
667 .cloned()
668 .ok_or_else(|| EngineError::TokenNotFound(format!("handle={h}")))?;
669 s.tokens
670 .get(&fp)
671 .and_then(|r| r.task_id.as_ref())
672 .cloned()
673 .ok_or_else(|| EngineError::TokenNotFound(format!("fp={fp}")))
674 })
675 .await?
676 }
677
678 /// Submit a worker result via a short handle. Skips token verification
679 /// and updates `output_tail` `Final` + `task.last_result` directly in
680 /// a thin path. The caller is expected to have already resolved
681 /// `task_id` via `task_id_from_handle` — the handle's presence in
682 /// `worker_handles` means it was minted server-side and is therefore
683 /// trusted.
684 pub async fn submit_worker_result_trusted(
685 &self,
686 task_id: &StepId,
687 attempt: u32,
688 value: Value,
689 ok: bool,
690 ) -> Result<(), EngineError> {
691 // GH #51 — completion-time verdict-contract enforcement, embedded
692 // choke point 1 of 2 (see `Self::verdict_contract_completion_check`'s
693 // doc). This path always submits a `Final` by construction (there
694 // is no other event kind on `/v1/worker/submit`), so the check
695 // always applies — unlike `submit_output` below, no `if let
696 // OutputEvent::Final { .. }` guard is needed here since there is
697 // no other `OutputEvent` variant this function could be asked to
698 // write. Runs BEFORE the `output_tail` write immediately below:
699 // on `Err`, this returns immediately and neither `with_state` call
700 // in this function executes.
701 let comparable_value =
702 content_ref_to_comparable_string(crate::worker::output::ContentRef::Inline {
703 value: value.clone(),
704 });
705 self.verdict_contract_completion_check(task_id, attempt, ok, &comparable_value)
706 .await?;
707 let task_id_for_apply = task_id.clone();
708 let value_for_event = value.clone();
709 self.with_state("submit_worker_result_trusted.output", move |s| {
710 let ev = crate::worker::output::OutputEvent::Final {
711 content: crate::worker::output::ContentRef::Inline {
712 value: value_for_event,
713 },
714 ok,
715 };
716 s.output_store
717 .entry((task_id_for_apply.clone(), attempt))
718 .or_default()
719 .push(ev.clone());
720 s.push_event(crate::core::state::Event::WorkerOutput {
721 task_id: task_id_for_apply,
722 attempt,
723 event: ev,
724 });
725 })
726 .await?;
727 let task_id_for_result = task_id.clone();
728 let value_for_result = value.clone();
729 self.with_state("submit_worker_result_trusted.last_result", move |s| {
730 if let Some(t) = s.tasks.get_mut(&task_id_for_result) {
731 t.last_result = Some(value_for_result);
732 t.updated_at = now_unix();
733 }
734 })
735 .await?;
736 // subtask-4 / ST2 rework: this path always submits a `Final` (there
737 // is no other event kind on `/v1/worker/submit`), so the
738 // submit-time projection sink always fires — see
739 // `materialize_final_submission`'s doc and `submit_output`'s
740 // Invariants (fail-open, never turns a would-have-succeeded submit
741 // into a failure).
742 let content = crate::worker::output::ContentRef::Inline { value };
743 self.materialize_final_submission(task_id, attempt, &content, ok)
744 .await?;
745 Ok(())
746 }
747
748 /// Stage a named `Artifact` from a worker via a short handle (GH #36
749 /// ST1: named multi-part worker output). Trusted analog of
750 /// [`Self::submit_worker_result_trusted`] for `OutputEvent::Artifact`:
751 /// skips token verification for the same reason (the caller already
752 /// resolved `task_id` via `task_id_from_handle`, so the handle's
753 /// presence in `worker_handles` is itself the trust boundary).
754 ///
755 /// Appends to the same per-`(task_id, attempt)` `output_store` tail
756 /// [`Self::dispatch_attempt_with`]'s Final-pull later folds into
757 /// `{"out": <final>, "parts": {<name>: <value>, ...}}` (see that
758 /// method's doc for the fold semantics — event order, last-write-wins
759 /// per name), AND records `name` in `EngineState.worker_artifact_names`
760 /// — the fold's allowlist of the WORKER's own staged parts, as opposed
761 /// to every `Artifact` that happens to land on the shared tail (e.g. an
762 /// audit sidecar finding; see that field's doc). Also dual-writes to
763 /// the Data-plane `OutputStore` the same way [`Self::submit_output`]'s
764 /// `Artifact` arm does, via [`Self::materialize_artifact_submission`]
765 /// (the artifact's own `name` is its Data-plane key, no
766 /// canonicalization — see that method's doc).
767 pub async fn stage_worker_artifact_trusted(
768 &self,
769 task_id: &StepId,
770 attempt: u32,
771 name: String,
772 value: Value,
773 ) -> Result<(), EngineError> {
774 let content = crate::worker::output::ContentRef::Inline { value };
775 let task_id_for_apply = task_id.clone();
776 let name_for_apply = name.clone();
777 let content_for_apply = content.clone();
778 self.with_state("stage_worker_artifact_trusted.output", move |s| {
779 let ev = crate::worker::output::OutputEvent::Artifact {
780 name: name_for_apply.clone(),
781 content: content_for_apply,
782 };
783 s.output_store
784 .entry((task_id_for_apply.clone(), attempt))
785 .or_default()
786 .push(ev.clone());
787 s.worker_artifact_names
788 .entry((task_id_for_apply.clone(), attempt))
789 .or_default()
790 .push(name_for_apply);
791 s.push_event(crate::core::state::Event::WorkerOutput {
792 task_id: task_id_for_apply,
793 attempt,
794 event: ev,
795 });
796 })
797 .await?;
798 self.materialize_artifact_submission(task_id, attempt, &name, &content)
799 .await?;
800 Ok(())
801 }
802
803 /// GH #36 ST1: the set of `Artifact` names staged for `(task_id,
804 /// attempt)` via [`Self::stage_worker_artifact_trusted`] — see
805 /// `EngineState.worker_artifact_names`'s doc. Used by
806 /// [`Self::dispatch_attempt_with`]'s Final-pull to distinguish a
807 /// worker's own named parts from any other `Artifact` producer on the
808 /// same tail.
809 async fn worker_artifact_names_for(&self, task_id: &StepId, attempt: u32) -> Vec<String> {
810 let key = (task_id.clone(), attempt);
811 self.with_state("worker_artifact_names_for", move |s| {
812 s.worker_artifact_names
813 .get(&key)
814 .cloned()
815 .unwrap_or_default()
816 })
817 .await
818 .unwrap_or_default()
819 }
820
821 /// Mint a short handle and register it in the `worker_handles` map.
822 /// Called immediately after the worker-token mint inside
823 /// `dispatch_attempt_with`, and issues a handle bound to the same
824 /// token fingerprint. Format is `wh-<8 hex chars>` (11 chars total),
825 /// designed to remove the base64 copy-paste failure mode.
826 async fn mint_worker_handle(&self, worker_fp: String) -> Result<String, EngineError> {
827 // The handle is a sole bearer secret on the `/v1/worker/submit`
828 // short-handle path (`submit_worker_result_trusted` skips token
829 // verification), so it must be unguessable — OS RNG, not the
830 // predictable uid counter. 8 hex chars (~4B entropy) keeps the
831 // documented `wh-<8 hex>` wire shape; collision between live
832 // handles is negligible at in-process handle counts.
833 let short = crate::types::secure_hex(4);
834 let handle = format!("wh-{short}");
835 let h = handle.clone();
836 self.with_state("mint_worker_handle", move |s| {
837 s.worker_handles.insert(h, worker_fp);
838 })
839 .await?;
840 Ok(handle)
841 }
842
843 // ═══════════════════════════════════════════════════════════════════════
844 // Session API
845 // ═══════════════════════════════════════════════════════════════════════
846
847 /// Attach a new session with default `OperatorInfo` (`Automate`, no
848 /// bridges/hooks). Shorthand for `attach_with(.., OperatorInfo::default())`.
849 pub async fn attach(
850 &self,
851 operator_id: impl Into<String>,
852 role: Role,
853 ttl: Duration,
854 ) -> Result<CapToken, EngineError> {
855 self.attach_with(
856 operator_id,
857 role,
858 ttl,
859 crate::core::ctx::OperatorInfo::default(),
860 )
861 .await
862 }
863
864 // ═══════════════════════════════════════════════════════════════════════
865 // BridgeRegistry API.
866 // ═══════════════════════════════════════════════════════════════════════
867
868 /// Register a `SeniorBridge` under a name. An existing entry with the
869 /// same name is overwritten. On the persisted-session reattach path,
870 /// the caller re-registers under the same ID beforehand and the
871 /// bridge becomes effective again.
872 pub async fn register_senior_bridge(
873 &self,
874 id: impl Into<String>,
875 bridge: Arc<dyn SeniorBridge>,
876 ) {
877 self.inner
878 .senior_bridges
879 .write()
880 .await
881 .insert(id.into(), bridge);
882 }
883
884 /// Register a `SpawnHook` under a name. An existing entry with the
885 /// same name is overwritten.
886 pub async fn register_spawn_hook(&self, id: impl Into<String>, hook: Arc<dyn SpawnHook>) {
887 self.inner.spawn_hooks.write().await.insert(id.into(), hook);
888 }
889
890 /// Register an `Operator` (a spawn-body backend) under a name. An
891 /// existing entry with the same name is overwritten.
892 /// `OperatorDelegateMiddleware` looks this up via `ctx` and, when
893 /// `kind = MainAi` / `Composite`, bypasses `inner.spawn` and calls
894 /// `operator.execute` instead.
895 pub async fn register_operator(
896 &self,
897 id: impl Into<String>,
898 operator: Arc<dyn crate::operator::Operator>,
899 ) {
900 self.inner
901 .operators
902 .write()
903 .await
904 .insert(id.into(), operator);
905 }
906
907 /// Unregister a `SeniorBridge` by name (e.g. on WebSocket disconnect
908 /// or explicit teardown). A missing ID is a no-op.
909 pub async fn unregister_senior_bridge(&self, id: &str) {
910 self.inner.senior_bridges.write().await.remove(id);
911 }
912
913 /// Unregister a `SpawnHook` by name. A missing ID is a no-op.
914 pub async fn unregister_spawn_hook(&self, id: &str) {
915 self.inner.spawn_hooks.write().await.remove(id);
916 }
917
918 /// Unregister an `Operator` backend by name. A missing ID is a no-op.
919 pub async fn unregister_operator(&self, id: &str) {
920 self.inner.operators.write().await.remove(id);
921 }
922
923 /// Snapshot the list of registered `SpawnHook` IDs (for test
924 /// observation and debugging).
925 pub async fn list_spawn_hook_ids(&self) -> Vec<String> {
926 self.inner
927 .spawn_hooks
928 .read()
929 .await
930 .keys()
931 .cloned()
932 .collect()
933 }
934
935 /// Snapshot the list of registered `SeniorBridge` IDs.
936 pub async fn list_senior_bridge_ids(&self) -> Vec<String> {
937 self.inner
938 .senior_bridges
939 .read()
940 .await
941 .keys()
942 .cloned()
943 .collect()
944 }
945
946 /// Snapshot the list of registered `Operator` IDs.
947 pub async fn list_operator_ids(&self) -> Vec<String> {
948 self.inner.operators.read().await.keys().cloned().collect()
949 }
950
951 /// Attach specifying IDs directly. The caller is expected to have
952 /// pre-registered them via `register_senior_bridge` /
953 /// `register_spawn_hook` / `register_operator`. This is the canonical
954 /// path when persistence is in play.
955 ///
956 /// `kind` is the "Runtime Global" tier of the `OperatorKind` cascade
957 /// (stored verbatim on `OperatorSession.operator_kind`): `Some(_)` is
958 /// an explicit request (including `Some(OperatorKind::Automate)`) that
959 /// outranks the BP-level tiers; `None` leaves it unspecified so the
960 /// BP-level tiers / final default decide. See
961 /// `crate::core::ctx::collapse_operator_kind`.
962 #[allow(clippy::too_many_arguments)]
963 pub async fn attach_with_ids(
964 &self,
965 operator_id: impl Into<String>,
966 role: Role,
967 ttl: Duration,
968 kind: Option<OperatorKind>,
969 bridge_id: Option<String>,
970 hook_id: Option<String>,
971 operator_backend_id: Option<String>,
972 operator_kind_overrides: HashMap<String, OperatorKind>,
973 bp_agent_kinds: HashMap<String, OperatorKind>,
974 bp_global_kind: Option<OperatorKind>,
975 ) -> Result<CapToken, EngineError> {
976 let operator_id = operator_id.into();
977 let token = self
978 .inner
979 .signer
980 .session(operator_id.clone(), role, vec!["*".into()], ttl);
981 let session_id = SessionId::new();
982 let fp = token.fingerprint();
983 let now = now_unix();
984 let token_for_store = token.clone();
985
986 self.with_state("attach_with_ids", |s| {
987 s.tokens
988 .insert(fp.clone(), CapTokenRecord::from_token(token_for_store));
989 s.sessions.insert(
990 session_id.clone(),
991 OperatorSession {
992 id: session_id.clone(),
993 operator_id: operator_id.clone(),
994 role,
995 attached_at: now,
996 last_seen: now,
997 attached: true,
998 owned_task_ids: Vec::new(),
999 token_fp: fp.clone(),
1000 operator_kind: kind,
1001 runtime_agent_kinds: operator_kind_overrides,
1002 bp_agent_kinds,
1003 bp_global_kind,
1004 bridge_id,
1005 hook_id,
1006 operator_backend_id,
1007 },
1008 );
1009 s.push_event(Event::SessionAttached {
1010 session_id: session_id.clone(),
1011 role,
1012 });
1013 })
1014 .await?;
1015
1016 let _ = self
1017 .inner
1018 .event_tx
1019 .send(Event::SessionAttached { session_id, role });
1020 Ok(token)
1021 }
1022
1023 /// Build an `OperatorInfo` by looking up the session's registered IDs
1024 /// on the `BridgeRegistry`, plus resolving the 4-tier `OperatorKind`
1025 /// cascade for `agent_name` via `crate::core::ctx::collapse_operator_kind`.
1026 /// Used when `dispatch_attempt` injects `Ctx`. An unresolved ID
1027 /// (nothing registered) is silently `None` — the bridge / hook simply
1028 /// does not fire and the default behaviour applies.
1029 async fn resolve_operator_info(
1030 &self,
1031 session: &OperatorSession,
1032 agent_name: &str,
1033 ) -> OperatorInfo {
1034 let senior_bridge = if let Some(id) = &session.bridge_id {
1035 self.inner.senior_bridges.read().await.get(id).cloned()
1036 } else {
1037 None
1038 };
1039 let spawn_hook = if let Some(id) = &session.hook_id {
1040 self.inner.spawn_hooks.read().await.get(id).cloned()
1041 } else {
1042 None
1043 };
1044 let operator = if let Some(id) = &session.operator_backend_id {
1045 self.inner.operators.read().await.get(id).cloned()
1046 } else {
1047 None
1048 };
1049 let runtime_agent = session.runtime_agent_kinds.get(agent_name).copied();
1050 // "Runtime Global" tier: `Some(_)` is always an explicit request
1051 // (see the field doc on `OperatorSession.operator_kind`).
1052 let runtime_global = session.operator_kind;
1053 let bp_agent = session.bp_agent_kinds.get(agent_name).copied();
1054 let bp_global = session.bp_global_kind;
1055 let kind = crate::core::ctx::collapse_operator_kind(
1056 runtime_agent,
1057 runtime_global,
1058 bp_agent,
1059 bp_global,
1060 );
1061 OperatorInfo {
1062 kind,
1063 id: session.operator_id.clone(),
1064 senior_bridge,
1065 spawn_hook,
1066 operator,
1067 }
1068 }
1069
1070 /// Convenience attach that takes an `OperatorInfo` (three
1071 /// `Arc<dyn ...>` fields plus `kind`) **inline**.
1072 ///
1073 /// # Pipeline
1074 ///
1075 /// Each `Arc<dyn ...>` is auto-registered on the engine's registry
1076 /// under a synthetic ID (`br-<hex>` / `hk-<hex>` / `ob-<hex>`), and
1077 /// the session stores that synthetic ID. Subsequent `dispatch_attempt`
1078 /// calls rebuild the `Arc`s from those IDs via
1079 /// `resolve_operator_info`, and the three middlewares fire as usual.
1080 ///
1081 /// # ⚠ Non-persisted sessions only
1082 ///
1083 /// Because this API takes inline `Arc`s, the reattach path after
1084 /// session persistence cannot rebuild them — the synthetic IDs are
1085 /// not present in a freshly started process's registry. If you need
1086 /// persistence, use [`Self::attach_with_ids`] with `register_*` calls
1087 /// beforehand to go through **named IDs** instead.
1088 ///
1089 /// Handy for tests and short-lived in-process sessions. Production
1090 /// WebSocket callbacks and the like should prefer `attach_with_ids`
1091 /// as the canonical path.
1092 pub async fn attach_with(
1093 &self,
1094 operator_id: impl Into<String>,
1095 role: Role,
1096 ttl: Duration,
1097 operator_info: crate::core::ctx::OperatorInfo,
1098 ) -> Result<CapToken, EngineError> {
1099 let operator_id = operator_id.into();
1100 // The caller always hands in a fully-formed `OperatorInfo`
1101 // (including its `kind`), so it is stored as an explicit "Runtime
1102 // Global" tier request (`Some(kind)`) — this path never persists
1103 // BP-level tiers (both stay empty below), so `Some(kind)` resolves
1104 // to the same `kind` at dispatch either way; see
1105 // `OperatorSession.operator_kind` doc.
1106 let kind = operator_info.kind;
1107 // BridgeRegistry auto-register: when the caller hands in an
1108 // `Arc<dyn>` directly, register it under a synthesised ID (the inline
1109 // path aware of persistence). Callers who want to pre-register with a
1110 // named ID should use `register_senior_bridge` / `register_spawn_hook`
1111 // + `attach_with_ids`.
1112 let bridge_id = if let Some(bridge) = operator_info.senior_bridge.clone() {
1113 let id = format!("br-{}", crate::types::uid_hex(8));
1114 self.inner
1115 .senior_bridges
1116 .write()
1117 .await
1118 .insert(id.clone(), bridge);
1119 Some(id)
1120 } else {
1121 None
1122 };
1123 let hook_id = if let Some(hook) = operator_info.spawn_hook.clone() {
1124 let id = format!("hk-{}", crate::types::uid_hex(8));
1125 self.inner
1126 .spawn_hooks
1127 .write()
1128 .await
1129 .insert(id.clone(), hook);
1130 Some(id)
1131 } else {
1132 None
1133 };
1134 let operator_backend_id = if let Some(operator) = operator_info.operator.clone() {
1135 // `ob-` = operator-backend registry id. Renamed from `op-` in the
1136 // issue #11 prefix reconciliation: `op-` used to collide with the
1137 // WS operator sid shape (now unified into `S-<hex>` anyway), and a
1138 // shared prefix across two unrelated registries made log filtering
1139 // by prefix silently ambiguous.
1140 let id = format!("ob-{}", crate::types::uid_hex(8));
1141 self.inner
1142 .operators
1143 .write()
1144 .await
1145 .insert(id.clone(), operator);
1146 Some(id)
1147 } else {
1148 None
1149 };
1150
1151 let token = self
1152 .inner
1153 .signer
1154 .session(operator_id.clone(), role, vec!["*".into()], ttl);
1155 let session_id = SessionId::new();
1156 let fp = token.fingerprint();
1157 let now = now_unix();
1158 let token_for_store = token.clone();
1159
1160 self.with_state("attach_with", |s| {
1161 s.tokens
1162 .insert(fp.clone(), CapTokenRecord::from_token(token_for_store));
1163 s.sessions.insert(
1164 session_id.clone(),
1165 OperatorSession {
1166 id: session_id.clone(),
1167 operator_id,
1168 role,
1169 attached_at: now,
1170 last_seen: now,
1171 attached: true,
1172 owned_task_ids: Vec::new(),
1173 token_fp: fp.clone(),
1174 operator_kind: Some(kind),
1175 runtime_agent_kinds: HashMap::new(),
1176 bp_agent_kinds: HashMap::new(),
1177 bp_global_kind: None,
1178 bridge_id,
1179 hook_id,
1180 operator_backend_id,
1181 },
1182 );
1183 s.push_event(Event::SessionAttached {
1184 session_id: session_id.clone(),
1185 role,
1186 });
1187 })
1188 .await?;
1189
1190 let _ = self
1191 .inner
1192 .event_tx
1193 .send(Event::SessionAttached { session_id, role });
1194 Ok(token)
1195 }
1196
1197 /// Mark the session bound to `token` as detached (`attached = false`).
1198 /// Tasks are left in place — a later `attach`/`attach_with_ids` call
1199 /// carrying the same registered bridge/hook IDs can pick them back up.
1200 pub async fn detach(&self, token: &CapToken) -> Result<(), EngineError> {
1201 self.verify_token(token, Verb::DetachSession).await?;
1202 let fp = token.fingerprint();
1203 self.with_state("detach", move |s| {
1204 let sid = s
1205 .sessions
1206 .iter()
1207 .find(|(_, sess)| sess.token_fp == fp)
1208 .map(|(id, _)| id.clone());
1209 if let Some(sid) = sid {
1210 if let Some(sess) = s.sessions.get_mut(&sid) {
1211 sess.attached = false;
1212 }
1213 s.push_event(Event::SessionDetached {
1214 session_id: sid.clone(),
1215 });
1216 let _ = sid;
1217 }
1218 })
1219 .await?;
1220 Ok(())
1221 }
1222
1223 /// Refresh the session's `last_seen` timestamp and mark it `attached`.
1224 /// Called periodically by an attached client to avoid being flipped to
1225 /// detached by `start_detach_loop`.
1226 pub async fn heartbeat(&self, token: &CapToken) -> Result<(), EngineError> {
1227 self.verify_token(token, Verb::Heartbeat).await?;
1228 let now = now_unix();
1229 let fp = token.fingerprint();
1230 self.with_state("heartbeat", move |s| {
1231 if let Some(sess) = s.sessions.values_mut().find(|sess| sess.token_fp == fp) {
1232 sess.last_seen = now;
1233 sess.attached = true;
1234 }
1235 })
1236 .await?;
1237 Ok(())
1238 }
1239
1240 // ═══════════════════════════════════════════════════════════════════════
1241 // Task lifecycle
1242 // ═══════════════════════════════════════════════════════════════════════
1243
1244 /// Create a new `TaskState` from `spec` and register its initial
1245 /// prompt. When the calling token is a Worker (i.e. this is a
1246 /// recursive spawn), the new task inherits `parent.spawn_depth + 1`
1247 /// and is rejected with `SpawnDepthExceeded` once `max_spawn_depth` is
1248 /// hit; an Operator-issued call starts at depth 0.
1249 pub async fn start_task(
1250 &self,
1251 token: &CapToken,
1252 spec: TaskSpec,
1253 ) -> Result<StepId, EngineError> {
1254 self.verify_token(token, Verb::StartTask).await?;
1255 let task_id = StepId::new();
1256 let initial_directive = spec.initial_directive.clone();
1257 let task_id_clone = task_id.clone();
1258 let fp = token.fingerprint();
1259 let max_depth = self.inner.cfg.max_spawn_depth;
1260 self.with_state("start_task", move |s| {
1261 // Recursive swarm depth gate (recursion guard):
1262 // Worker tokens carry CapTokenRecord.parent_task_id. Give the
1263 // child parent's spawn_depth + 1; if it exceeds `max`, raise an
1264 // error. Operator tokens (parent_task_id=None) start at depth 0.
1265 let parent_depth_opt = s
1266 .tokens
1267 .get(&fp)
1268 .and_then(|rec| rec.task_id.as_ref())
1269 .and_then(|tid| s.tasks.get(tid))
1270 .map(|t| t.spawn_depth);
1271 let depth = match parent_depth_opt {
1272 Some(d) => {
1273 if d + 1 >= max_depth {
1274 return Err(EngineError::SpawnDepthExceeded {
1275 current: d + 1,
1276 max: max_depth,
1277 });
1278 }
1279 d + 1
1280 }
1281 None => 0,
1282 };
1283
1284 let mut task = TaskState::new(task_id_clone.clone(), spec);
1285 task.spawn_depth = depth;
1286 s.tasks.insert(task_id_clone.clone(), task);
1287 s.prompts
1288 .insert((task_id_clone.clone(), 1), initial_directive);
1289 // Link to the owner session (only Operator tokens match; Worker tokens have no session).
1290 if let Some(sess) = s.sessions.values_mut().find(|sess| sess.token_fp == fp) {
1291 sess.owned_task_ids.push(task_id_clone.clone());
1292 }
1293 s.push_event(Event::TaskCreated {
1294 task_id: task_id_clone.clone(),
1295 });
1296 Ok::<(), EngineError>(())
1297 })
1298 .await??;
1299 let _ = self.inner.event_tx.send(Event::TaskCreated {
1300 task_id: task_id.clone(),
1301 });
1302 Ok(task_id)
1303 }
1304
1305 /// Fetch a snapshot of `TaskState` for `task_id`, subject to the
1306 /// task-ownership gate (see `verify_token_for_task`).
1307 pub async fn read_task_state(
1308 &self,
1309 token: &CapToken,
1310 task_id: &StepId,
1311 ) -> Result<TaskState, EngineError> {
1312 self.verify_token_for_task(token, Verb::ReadTaskState, task_id)
1313 .await?;
1314 let task_id = task_id.clone();
1315 self.with_state("read_task_state", move |s| {
1316 s.tasks
1317 .get(&task_id)
1318 .cloned()
1319 .ok_or_else(|| EngineError::TaskNotFound(task_id.to_string()))
1320 })
1321 .await?
1322 }
1323
1324 /// Mark `task_id` as `Cancelled` and wake any caller blocked in
1325 /// `poll_task` for it.
1326 pub async fn cancel_task(&self, token: &CapToken, task_id: &StepId) -> Result<(), EngineError> {
1327 self.verify_token_for_task(token, Verb::CancelTask, task_id)
1328 .await?;
1329 let tid = task_id.clone();
1330 self.with_state("cancel_task", move |s| {
1331 let task = s
1332 .tasks
1333 .get_mut(&tid)
1334 .ok_or_else(|| EngineError::TaskNotFound(tid.to_string()))?;
1335 task.status = TaskStatus::Cancelled;
1336 task.updated_at = now_unix();
1337 s.push_event(Event::TaskCancelled {
1338 task_id: tid.clone(),
1339 });
1340 Ok::<(), EngineError>(())
1341 })
1342 .await??;
1343 self.wake_task(task_id).await?;
1344 Ok(())
1345 }
1346
1347 /// Dispatch a single attempt through the given `spawner`.
1348 ///
1349 /// The lock is only held for snapshot capture; the actual spawn and
1350 /// completion await happen outside the lock (R3 discipline).
1351 ///
1352 /// Sits on the Domain side of the Data / Domain split. The dispatch
1353 /// path itself does not touch big response bodies — those flow through
1354 /// the Data plane (`output_store` module + sink / input_inject
1355 /// `SpawnerLayer`s) around this method.
1356 ///
1357 /// The caller does the compile plus `service::linker::link` and
1358 /// carries the same stack through each dispatch. Because the spawner
1359 /// is passed per-request rather than looked up from engine-global
1360 /// state, parallel requests against a single `Engine` instance
1361 /// (different Blueprints, different spawners) do not race.
1362 ///
1363 /// `run_id`, when `Some` (issue #13 run_id propagation —
1364 /// `EngineDispatcher` threads it in from its `RunContext`), is
1365 /// inserted into `Ctx.meta.runtime["run_id"]` (a plain JSON string)
1366 /// alongside `worker_handle`, so `Operator::execute` implementations
1367 /// (e.g. `WSOperatorSession`) can read it back and surface it to the
1368 /// worker (Spawn directive / prompt). `None` (every pre-existing
1369 /// caller / test) omits the key entirely — unchanged behavior.
1370 pub async fn dispatch_attempt_with(
1371 &self,
1372 token: &CapToken,
1373 task_id: &StepId,
1374 spawner: &Arc<dyn SpawnerAdapter>,
1375 run_id: Option<&RunId>,
1376 ) -> Result<DispatchOutcome, EngineError> {
1377 self.verify_token(token, Verb::DispatchAttempt).await?;
1378 let task_id = task_id.clone();
1379
1380 // 1) Under the lock: increment the attempt number, mark Running, snapshot the
1381 // prompt, and pull `operator_info` from the session so we can inject it into Ctx.
1382 let fp = token.fingerprint();
1383 let tid_for_prep = task_id.clone();
1384 let (attempt, agent, session_snapshot, step_ctx) = self
1385 .with_state("dispatch.prep", move |s| {
1386 let task = s
1387 .tasks
1388 .get_mut(&tid_for_prep)
1389 .ok_or_else(|| EngineError::TaskNotFound(tid_for_prep.to_string()))?;
1390 task.attempt += 1;
1391 task.status = TaskStatus::Running;
1392 task.updated_at = now_unix();
1393 // The spawner pulls the prompt via engine.fetch_prompt. In prep,
1394 // if the prompts table has no entry for this attempt yet,
1395 // fall back and insert `initial_directive` so the subsequent
1396 // fetch_prompt succeeds.
1397 let attempt = task.attempt;
1398 let initial = task.spec.initial_directive.clone();
1399 s.prompts
1400 .entry((tid_for_prep.clone(), attempt))
1401 .or_insert(initial);
1402 let task = s
1403 .tasks
1404 .get(&tid_for_prep)
1405 .ok_or_else(|| EngineError::TaskNotFound(tid_for_prep.to_string()))?;
1406 let agent = task.spec.agent.clone();
1407 // GH #21 Phase 2: re-read `TaskSpec.step_ctx` on EVERY
1408 // attempt (not cached once at start_task) so retries and
1409 // Run-rekicks all carry the Step tier through to Ctx —
1410 // see TaskSpec.step_ctx's doc.
1411 let step_ctx = task.spec.step_ctx.clone();
1412 // Session snapshot (looked up by token nonce). When no session
1413 // exists (worker token invoked directly / test injection), fall
1414 // back to None → default OperatorInfo.
1415 let sess_clone = s
1416 .sessions
1417 .values()
1418 .find(|sess| sess.token_fp == fp)
1419 .cloned();
1420 Ok::<_, EngineError>((attempt, agent, sess_clone, step_ctx))
1421 })
1422 .await??;
1423 // BridgeRegistry lookup + per-agent OperatorKind cascade.
1424 let operator_info = match session_snapshot {
1425 Some(sess) => self.resolve_operator_info(&sess, &agent).await,
1426 None => OperatorInfo::default(),
1427 };
1428
1429 // 2) Outside the lock: worker token mint + spawn.
1430 //
1431 // Session-style mint (max_uses=None). Within one attempt the worker is
1432 // expected to hit `verify_token + fetch_prompt + fetch_data + post_result`
1433 // multiple times in order, so `one_time` would exhaust the token on the
1434 // very first verb. Capability is guarded by (a) the role × verb gate and
1435 // (b) the short TTL (1800s).
1436 let worker_token = self.inner.signer.session(
1437 format!("worker-of-{task_id}"),
1438 Role::Worker,
1439 vec!["*".into()],
1440 Duration::from_secs(1800),
1441 );
1442 let worker_fp = worker_token.fingerprint();
1443 let task_id_for_worker = task_id.clone();
1444 let worker_token_for_store = worker_token.clone();
1445 self.with_state("dispatch.mint_worker", move |s| {
1446 s.tokens.insert(
1447 worker_fp,
1448 CapTokenRecord::from_worker_token(worker_token_for_store, task_id_for_worker),
1449 );
1450 })
1451 .await?;
1452
1453 // Mint a short handle (`wh-XXXXXXXX`) and register it in worker_handles.
1454 // Used by the simplified Bearer path for SubAgents (short-handle form
1455 // avoids base64 copy-paste incidents).
1456 let worker_handle = self.mint_worker_handle(worker_token.fingerprint()).await?;
1457
1458 let mut ctx = Ctx::new(task_id.clone(), attempt, agent.clone());
1459 ctx.operator = operator_info; // activates MainAIMiddleware / Senior bridge
1460 ctx.meta
1461 .runtime
1462 .insert("worker_handle".to_string(), Value::String(worker_handle));
1463 if let Some(rid) = run_id {
1464 ctx.meta
1465 .runtime
1466 .insert(RUN_ID_KEY.to_string(), Value::String(rid.to_string()));
1467 }
1468 // GH #21 Phase 2: the Step tier's resolved context bundle (from
1469 // `TaskSpec.step_ctx`, re-read every attempt above) — consumed by
1470 // `AgentContextMiddleware`, which unpacks its keys ahead of the
1471 // Agent / BP-global tiers.
1472 if let Some(step_ctx) = step_ctx {
1473 ctx.meta.runtime.insert(STEP_CTX_KEY.to_string(), step_ctx);
1474 }
1475
1476 let worker = spawner
1477 .spawn(self, &ctx, task_id.clone(), attempt, worker_token)
1478 .await
1479 .map_err(|e| EngineError::DispatchFailed(e.to_string()))?;
1480
1481 // 3) Outside the lock: await worker.join() (signal-only). WorkerError is
1482 // stringified. The value is fetched via output_tail (sink path).
1483 let signal_result: Result<(), String> = worker.join().await.map_err(|e| e.to_string());
1484
1485 // Pull the last Final from output_tail and use it as the value. GH
1486 // #36 ST1 (named multi-part worker output): also fold every
1487 // `Artifact` the WORKER ITSELF staged on the same tail (via
1488 // `stage_worker_artifact_trusted` / `POST /v1/worker/artifact`)
1489 // into a `"parts"` object keyed by name — event order,
1490 // last-write-wins per name (a name staged twice overwrites,
1491 // mirroring `HashMap`/`Map` insert semantics, not an accumulating
1492 // list). `worker_artifact_names_for` is the allowlist that scopes
1493 // this to the worker's own opt-in parts — an `Artifact` some OTHER
1494 // producer appended to this same tail (e.g.
1495 // `AfterRunAuditMiddleware`'s `"audit:<step_ref>"` sidecar finding)
1496 // is left untouched (see `fold_final_and_parts`'s doc). When at
1497 // least one part was staged, the BP-chain value becomes `{"out":
1498 // <final value>, "parts": {...}}`; zero parts staged (the
1499 // pre-GH-#36 case, and every non-opt-in step) leaves the value
1500 // exactly the plain `Final` value, byte-identical to before this
1501 // change.
1502 let value_ok: Result<(Value, bool), String> = match signal_result {
1503 Ok(()) => {
1504 let tail = self.output_tail(&task_id, attempt).await;
1505 let staged_names = self.worker_artifact_names_for(&task_id, attempt).await;
1506 fold_final_and_parts(&tail, &staged_names)
1507 .ok_or_else(|| "no Final in output_tail".to_string())
1508 }
1509 Err(msg) => Err(msg),
1510 };
1511
1512 // 4) Under the lock: apply (split the borrow scope so push_event and task mut can co-exist).
1513 let outcome = self
1514 .with_state("dispatch.apply", |s| {
1515 if !s.tasks.contains_key(&task_id) {
1516 return Err(EngineError::TaskNotFound(task_id.to_string()));
1517 }
1518 match value_ok {
1519 Ok((value, ok)) => {
1520 let pass = ok;
1521 {
1522 let task = s.tasks.get_mut(&task_id).unwrap();
1523 task.last_result = Some(value.clone());
1524 task.updated_at = now_unix();
1525 task.status = if pass {
1526 TaskStatus::Pass
1527 } else {
1528 TaskStatus::Blocked
1529 };
1530 }
1531 s.push_event(Event::TaskAttemptCompleted {
1532 task_id: task_id.clone(),
1533 attempt,
1534 result: value.clone(),
1535 });
1536 if pass {
1537 s.push_event(Event::TaskPass {
1538 task_id: task_id.clone(),
1539 result: value.clone(),
1540 });
1541 Ok::<_, EngineError>(DispatchOutcome::Pass(value))
1542 } else {
1543 s.push_event(Event::TaskBlocked {
1544 task_id: task_id.clone(),
1545 result: value.clone(),
1546 });
1547 Ok(DispatchOutcome::Blocked(value))
1548 }
1549 }
1550 Err(msg) => {
1551 let task = s.tasks.get_mut(&task_id).unwrap();
1552 task.status = TaskStatus::Blocked;
1553 task.updated_at = now_unix();
1554 Err(EngineError::DispatchFailed(msg))
1555 }
1556 }
1557 })
1558 .await??;
1559
1560 // event broadcast (outside the lock — push_event feeds the in-memory tail; broadcast is a separate path).
1561 let _ = self.inner.event_tx.send(Event::TaskAttemptCompleted {
1562 task_id: task_id.clone(),
1563 attempt,
1564 result: match &outcome {
1565 DispatchOutcome::Pass(v) | DispatchOutcome::Blocked(v) => v.clone(),
1566 _ => Value::Null,
1567 },
1568 });
1569
1570 // Wake any callers waiting in poll_task.
1571 self.wake_task(&task_id).await?;
1572
1573 Ok(outcome)
1574 }
1575
1576 /// Dispatch a single attempt, opt-in to the replay-log Core primitive
1577 /// ([`crate::store::replay`]) via `run_ctx`.
1578 ///
1579 /// This is the [`Self::dispatch_attempt_with`] sibling used by callers
1580 /// that carry a `RunContext` with `replay_store` / `replay_cursor`
1581 /// populated. Behavior versus the plain `dispatch_attempt_with`:
1582 ///
1583 /// - **`run_ctx.replay_cursor` is `Some` AND the cursor has a matching
1584 /// `(step_ref, input_hash, occurrence)` row** — the stored value is
1585 /// returned verbatim as `DispatchOutcome::Pass(v)`; the `Adapter`
1586 /// (spawner + worker) is never touched. The task's `attempt` is
1587 /// still bumped and `TaskStatus` set to `Pass`, so downstream state
1588 /// (`task.last_result`, `TaskAttemptCompleted` / `TaskPass` events,
1589 /// `wake_task`) fires the same way an ordinary Pass would.
1590 /// - **Miss (or `replay_cursor: None`)** — the ordinary spawn path
1591 /// runs. When `run_ctx.replay_store` is `Some` AND the outcome is
1592 /// `Pass`, one `ReplayEntry` is appended carrying the whole `Ctx`
1593 /// snapshot (with `operator` dropped by `#[serde(skip)]`) plus the
1594 /// `step_output` value. `Blocked` / `Err` outcomes are never
1595 /// logged — a partial-failure row would poison the replay path
1596 /// after a subsequent successful retry.
1597 ///
1598 /// `run_ctx: None` collapses to the same behavior as
1599 /// `dispatch_attempt_with(token, task_id, spawner, None)` — no run
1600 /// tracing, no replay.
1601 pub async fn dispatch_attempt_with_run_ctx(
1602 &self,
1603 token: &CapToken,
1604 task_id: &StepId,
1605 spawner: &Arc<dyn SpawnerAdapter>,
1606 run_ctx: Option<&RunContext>,
1607 ) -> Result<DispatchOutcome, EngineError> {
1608 self.verify_token(token, Verb::DispatchAttempt).await?;
1609 let task_id = task_id.clone();
1610
1611 // 1) Under the lock: prep (bump attempt, snapshot agent/directive).
1612 let fp = token.fingerprint();
1613 let tid_for_prep = task_id.clone();
1614 let (attempt, agent, session_snapshot, step_ctx, initial_directive) = self
1615 .with_state("dispatch_run_ctx.prep", move |s| {
1616 let task = s
1617 .tasks
1618 .get_mut(&tid_for_prep)
1619 .ok_or_else(|| EngineError::TaskNotFound(tid_for_prep.to_string()))?;
1620 task.attempt += 1;
1621 task.status = TaskStatus::Running;
1622 task.updated_at = now_unix();
1623 let attempt = task.attempt;
1624 let initial = task.spec.initial_directive.clone();
1625 s.prompts
1626 .entry((tid_for_prep.clone(), attempt))
1627 .or_insert(initial.clone());
1628 let task = s
1629 .tasks
1630 .get(&tid_for_prep)
1631 .ok_or_else(|| EngineError::TaskNotFound(tid_for_prep.to_string()))?;
1632 let agent = task.spec.agent.clone();
1633 let step_ctx = task.spec.step_ctx.clone();
1634 let sess_clone = s
1635 .sessions
1636 .values()
1637 .find(|sess| sess.token_fp == fp)
1638 .cloned();
1639 Ok::<_, EngineError>((attempt, agent, sess_clone, step_ctx, initial))
1640 })
1641 .await??;
1642
1643 let operator_info = match session_snapshot {
1644 Some(sess) => self.resolve_operator_info(&sess, &agent).await,
1645 None => OperatorInfo::default(),
1646 };
1647
1648 // 2) Compute the replay key from step_ref (= agent) + hashed input.
1649 // Occurrence comes from the cursor's per-key counter (bumped
1650 // once per dispatch, so a loop that re-visits the same step
1651 // with the same input gets 0, 1, 2, … distinct rows).
1652 let step_ref = agent.clone();
1653 let input_hash = match run_ctx.and_then(|rc| rc.binding_digests.get(&step_ref)) {
1654 Some(binding_digest) => hash_input_value(&serde_json::json!({
1655 "input": initial_directive,
1656 "binding_digest": binding_digest,
1657 })),
1658 None => hash_input_value(&initial_directive),
1659 };
1660 let (replay_hit_value, occurrence) = if let Some(rc) = run_ctx {
1661 if let Some(cursor) = &rc.replay_cursor {
1662 let mut guard = cursor.lock().expect("replay cursor mutex poisoned");
1663 let occ = guard.next_occurrence(&step_ref, &input_hash);
1664 let hit = guard.find(&step_ref, &input_hash, occ);
1665 (hit, occ)
1666 } else {
1667 (None, 0)
1668 }
1669 } else {
1670 (None, 0)
1671 };
1672
1673 // 3) Build the Ctx that (a) either the spawner will see on a miss,
1674 // or (b) we log alongside the replay row.
1675 let mut ctx = Ctx::new(task_id.clone(), attempt, agent.clone());
1676 ctx.operator = operator_info;
1677 if let Some(rc) = run_ctx {
1678 ctx.meta
1679 .runtime
1680 .insert(RUN_ID_KEY.to_string(), Value::String(rc.run_id.to_string()));
1681 }
1682 if let Some(step_ctx) = step_ctx {
1683 ctx.meta.runtime.insert(STEP_CTX_KEY.to_string(), step_ctx);
1684 }
1685
1686 // 4) Replay-hit shortcut: skip the spawn+join, return stored value.
1687 let was_replay_hit = replay_hit_value.is_some();
1688 let value_ok: Result<(Value, bool), String> = if let Some(stored) = replay_hit_value {
1689 tracing::info!(
1690 task_id = %task_id,
1691 step_ref = %step_ref,
1692 occurrence = occurrence,
1693 "replayed from log; worker dispatch skipped"
1694 );
1695 Ok((stored, true))
1696 } else {
1697 // 5) Ordinary spawn path — mint a worker token+handle, run the
1698 // spawner, join, and pull the last Final from output_tail.
1699 let worker_token = self.inner.signer.session(
1700 format!("worker-of-{task_id}"),
1701 Role::Worker,
1702 vec!["*".into()],
1703 Duration::from_secs(1800),
1704 );
1705 let worker_fp = worker_token.fingerprint();
1706 let task_id_for_worker = task_id.clone();
1707 let worker_token_for_store = worker_token.clone();
1708 self.with_state("dispatch_run_ctx.mint_worker", move |s| {
1709 s.tokens.insert(
1710 worker_fp,
1711 CapTokenRecord::from_worker_token(worker_token_for_store, task_id_for_worker),
1712 );
1713 })
1714 .await?;
1715 let worker_handle = self.mint_worker_handle(worker_token.fingerprint()).await?;
1716 ctx.meta
1717 .runtime
1718 .insert("worker_handle".to_string(), Value::String(worker_handle));
1719
1720 let worker = spawner
1721 .spawn(self, &ctx, task_id.clone(), attempt, worker_token)
1722 .await
1723 .map_err(|e| EngineError::DispatchFailed(e.to_string()))?;
1724 let signal_result: Result<(), String> = worker.join().await.map_err(|e| e.to_string());
1725 match signal_result {
1726 Ok(()) => {
1727 let tail = self.output_tail(&task_id, attempt).await;
1728 let staged_names = self.worker_artifact_names_for(&task_id, attempt).await;
1729 fold_final_and_parts(&tail, &staged_names)
1730 .ok_or_else(|| "no Final in output_tail".to_string())
1731 }
1732 Err(msg) => Err(msg),
1733 }
1734 };
1735
1736 // 6) Apply — mirrors `dispatch_attempt_with`'s apply arm exactly
1737 // (task.last_result / status update + TaskAttemptCompleted /
1738 // TaskPass / TaskBlocked events).
1739 let outcome = self
1740 .with_state("dispatch_run_ctx.apply", |s| {
1741 if !s.tasks.contains_key(&task_id) {
1742 return Err(EngineError::TaskNotFound(task_id.to_string()));
1743 }
1744 match value_ok {
1745 Ok((value, ok)) => {
1746 let pass = ok;
1747 {
1748 let task = s.tasks.get_mut(&task_id).unwrap();
1749 task.last_result = Some(value.clone());
1750 task.updated_at = now_unix();
1751 task.status = if pass {
1752 TaskStatus::Pass
1753 } else {
1754 TaskStatus::Blocked
1755 };
1756 }
1757 s.push_event(Event::TaskAttemptCompleted {
1758 task_id: task_id.clone(),
1759 attempt,
1760 result: value.clone(),
1761 });
1762 if pass {
1763 s.push_event(Event::TaskPass {
1764 task_id: task_id.clone(),
1765 result: value.clone(),
1766 });
1767 Ok::<_, EngineError>(DispatchOutcome::Pass(value))
1768 } else {
1769 s.push_event(Event::TaskBlocked {
1770 task_id: task_id.clone(),
1771 result: value.clone(),
1772 });
1773 Ok(DispatchOutcome::Blocked(value))
1774 }
1775 }
1776 Err(msg) => {
1777 let task = s.tasks.get_mut(&task_id).unwrap();
1778 task.status = TaskStatus::Blocked;
1779 task.updated_at = now_unix();
1780 Err(EngineError::DispatchFailed(msg))
1781 }
1782 }
1783 })
1784 .await??;
1785
1786 // 7) On MISS + Pass + replay_store present, append a replay row.
1787 // Replay-HIT rows are already logged from the original run and
1788 // must never be double-logged (Core primitive contract). A
1789 // secondary-persistence failure here (`tracing::warn!` +
1790 // swallow) matches the `run_ctx.run_store.append_step_entry`
1791 // convention in `EngineDispatcher::dispatch`: it must not mask
1792 // the primary dispatch outcome the caller already has in hand.
1793 if !was_replay_hit {
1794 if let (Some(rc), DispatchOutcome::Pass(v)) = (run_ctx, &outcome) {
1795 if let Some(store) = &rc.replay_store {
1796 match ReplayEntry::from_completion(
1797 rc.run_id.clone(),
1798 step_ref.clone(),
1799 input_hash.clone(),
1800 occurrence,
1801 &ctx,
1802 v,
1803 ) {
1804 Ok(entry) => {
1805 if let Err(e) = store.append(entry).await {
1806 tracing::warn!(
1807 run_id = %rc.run_id,
1808 step_ref = %step_ref,
1809 occurrence = occurrence,
1810 error = %e,
1811 "dispatch_attempt_with_run_ctx: replay_store.append failed"
1812 );
1813 }
1814 }
1815 Err(e) => {
1816 tracing::warn!(
1817 run_id = %rc.run_id,
1818 step_ref = %step_ref,
1819 occurrence = occurrence,
1820 error = %e,
1821 "dispatch_attempt_with_run_ctx: ReplayEntry encode failed"
1822 );
1823 }
1824 }
1825 }
1826 }
1827 }
1828
1829 let _ = self.inner.event_tx.send(Event::TaskAttemptCompleted {
1830 task_id: task_id.clone(),
1831 attempt,
1832 result: match &outcome {
1833 DispatchOutcome::Pass(v) | DispatchOutcome::Blocked(v) => v.clone(),
1834 _ => Value::Null,
1835 },
1836 });
1837
1838 self.wake_task(&task_id).await?;
1839
1840 Ok(outcome)
1841 }
1842
1843 // ═══════════════════════════════════════════════════════════════════════
1844 // Worker-side API (= prompt / data fetch + result post)
1845 // ═══════════════════════════════════════════════════════════════════════
1846
1847 /// Fetch the directive/prompt `Value` for `task_id`'s current attempt.
1848 /// Falls back to `initial_directive` when no prompt has been recorded
1849 /// yet for that attempt. Returns the `Value` end-to-end (issue #18);
1850 /// the render down to `String` happens only at the two consumer
1851 /// boundaries — the Worker HTTP path (`fetch_worker_payload*` →
1852 /// `WorkerPayload.prompt: String`) and the WS Spawn frame text
1853 /// render (`operator_ws::session`).
1854 pub async fn fetch_prompt(
1855 &self,
1856 token: &CapToken,
1857 task_id: &StepId,
1858 ) -> Result<Value, EngineError> {
1859 self.verify_token_for_task(token, Verb::FetchPrompt, task_id)
1860 .await?;
1861 let task_id = task_id.clone();
1862 self.with_state("fetch_prompt", move |s| {
1863 let task = s
1864 .tasks
1865 .get(&task_id)
1866 .ok_or_else(|| EngineError::TaskNotFound(task_id.to_string()))?;
1867 s.prompts
1868 .get(&(task_id.clone(), task.attempt.max(1)))
1869 .cloned()
1870 .ok_or_else(|| {
1871 EngineError::ResourceNotFound(format!(
1872 "prompt({}, attempt={})",
1873 task_id, task.attempt
1874 ))
1875 })
1876 })
1877 .await?
1878 }
1879
1880 /// Combined fetch for `HTTP /v1/worker/prompt`: returns `prompt` +
1881 /// (optional) `system` + `agent` + `attempt` in a single round trip.
1882 /// The verb gate reuses `FetchPrompt` — same semantics as "the worker
1883 /// pulls its task input".
1884 ///
1885 /// `system` is the value written by `OperatorSpawner::spawn` through
1886 /// `bake_worker_system_prompt` when it ran; otherwise `None` (no
1887 /// profile present, or the bake never happened).
1888 pub async fn fetch_worker_payload(
1889 &self,
1890 token: &CapToken,
1891 task_id: &StepId,
1892 ) -> Result<crate::types::WorkerPayload, EngineError> {
1893 self.verify_token_for_task(token, Verb::FetchPrompt, task_id)
1894 .await?;
1895 let task_id_clone = task_id.clone();
1896 let mut payload = self
1897 .with_state("fetch_worker_payload", move |s| {
1898 let task = s
1899 .tasks
1900 .get(&task_id_clone)
1901 .ok_or_else(|| EngineError::TaskNotFound(task_id_clone.to_string()))?;
1902 let attempt = task.attempt.max(1);
1903 let prompt = s
1904 .prompts
1905 .get(&(task_id_clone.clone(), attempt))
1906 .cloned()
1907 .ok_or_else(|| {
1908 EngineError::ResourceNotFound(format!(
1909 "prompt({}, attempt={})",
1910 task_id_clone, attempt
1911 ))
1912 })?;
1913 let system = s
1914 .systems
1915 .get(&(task_id_clone.clone(), attempt))
1916 .cloned()
1917 .unwrap_or(None);
1918 let agent = task.spec.agent.clone();
1919 let context = s
1920 .agent_ctx
1921 .get(&(task_id_clone.clone(), attempt))
1922 .map(|e| e.view.clone());
1923 Ok::<_, EngineError>(crate::types::WorkerPayload {
1924 task_id: task_id_clone.clone(),
1925 attempt,
1926 agent,
1927 prompt: render_directive_to_string(&prompt),
1928 system,
1929 context,
1930 system_ref: None,
1931 })
1932 })
1933 .await??;
1934 self.apply_system_ref_threshold(&mut payload).await?;
1935 Ok(payload)
1936 }
1937
1938 /// Fetch a worker payload via a short handle. Skips token verification
1939 /// and returns `prompt` + `system` + `agent` + `attempt` in a thin
1940 /// path. The caller is expected to have already resolved `task_id`
1941 /// via `task_id_from_handle` — the handle's presence in
1942 /// `worker_handles` means it was minted server-side and is therefore
1943 /// trusted.
1944 pub async fn fetch_worker_payload_trusted(
1945 &self,
1946 task_id: &StepId,
1947 ) -> Result<crate::types::WorkerPayload, EngineError> {
1948 let task_id_clone = task_id.clone();
1949 let mut payload = self
1950 .with_state("fetch_worker_payload_trusted", move |s| {
1951 let task = s
1952 .tasks
1953 .get(&task_id_clone)
1954 .ok_or_else(|| EngineError::TaskNotFound(task_id_clone.to_string()))?;
1955 let attempt = task.attempt.max(1);
1956 let prompt = s
1957 .prompts
1958 .get(&(task_id_clone.clone(), attempt))
1959 .cloned()
1960 .ok_or_else(|| {
1961 EngineError::ResourceNotFound(format!(
1962 "prompt({}, attempt={})",
1963 task_id_clone, attempt
1964 ))
1965 })?;
1966 let system = s
1967 .systems
1968 .get(&(task_id_clone.clone(), attempt))
1969 .cloned()
1970 .unwrap_or(None);
1971 let agent = task.spec.agent.clone();
1972 let context = s
1973 .agent_ctx
1974 .get(&(task_id_clone.clone(), attempt))
1975 .map(|e| e.view.clone());
1976 Ok::<_, EngineError>(crate::types::WorkerPayload {
1977 task_id: task_id_clone.clone(),
1978 attempt,
1979 agent,
1980 prompt: render_directive_to_string(&prompt),
1981 system,
1982 context,
1983 system_ref: None,
1984 })
1985 })
1986 .await??;
1987 self.apply_system_ref_threshold(&mut payload).await?;
1988 Ok(payload)
1989 }
1990
1991 /// GH #31: shared threshold-branch tail for
1992 /// [`Self::fetch_worker_payload`] / [`Self::fetch_worker_payload_trusted`].
1993 /// Both build a raw `WorkerPayload` inside `with_state` with `system`
1994 /// populated as before and `system_ref: None`; this runs *outside* any
1995 /// lock (R3 — `SystemRefMode::File`'s `tokio::fs` write is a genuine
1996 /// `.await`, which `with_state`'s sync-closure contract forbids inside
1997 /// the lock) and rewrites `payload.system` / `payload.system_ref` in
1998 /// place per `SystemRefConfig.threshold_bytes`: over-threshold clears
1999 /// `system` and populates `system_ref`; at-or-under-threshold leaves
2000 /// `system` as-is and `system_ref` stays `None`. A no-op when
2001 /// `payload.system` is already `None` (no `system_prompt` was baked).
2002 async fn apply_system_ref_threshold(
2003 &self,
2004 payload: &mut crate::types::WorkerPayload,
2005 ) -> Result<(), EngineError> {
2006 let Some(rendered) = payload.system.take() else {
2007 return Ok(());
2008 };
2009 let cfg = self.cfg().system_ref.clone();
2010 if rendered.len() <= cfg.threshold_bytes {
2011 payload.system = Some(rendered);
2012 return Ok(());
2013 }
2014 use sha2::Digest;
2015 let size_bytes = rendered.len() as u64;
2016 let sha256 = hex::encode(sha2::Sha256::digest(rendered.as_bytes()));
2017 let task_id = &payload.task_id;
2018 let attempt = payload.attempt;
2019 let system_ref = match cfg.mode {
2020 crate::types::SystemRefMode::Http => crate::types::SystemRef {
2021 // The engine has no knowledge of scheme/host here — see
2022 // `SystemRefMode::Http`'s doc for who fills that in.
2023 uri: format!("/v1/worker/prompt/system?task_id={task_id}&attempt={attempt}"),
2024 sha256,
2025 size_bytes,
2026 mode: crate::types::SystemRefMode::Http,
2027 },
2028 crate::types::SystemRefMode::File => {
2029 tokio::fs::create_dir_all(&cfg.store_dir).await?;
2030 let path = cfg.store_dir.join(format!("{task_id}-{attempt}.md"));
2031 tokio::fs::write(&path, rendered.as_bytes()).await?;
2032 crate::types::SystemRef {
2033 uri: format!("file://{}", path.display()),
2034 sha256,
2035 size_bytes,
2036 mode: crate::types::SystemRefMode::File,
2037 }
2038 }
2039 };
2040 payload.system = None;
2041 payload.system_ref = Some(system_ref);
2042 Ok(())
2043 }
2044
2045 /// Returns the effective [`mlua_swarm_schema::ContextPolicy`]
2046 /// `AgentContextMiddleware` resolved and snapshotted for `(task_id,
2047 /// attempt)` at spawn time (the same policy already applied to that
2048 /// key's `EngineState.agent_ctx` entry's `.view`, GH #23 fold).
2049 /// Pass-all (`ContextPolicy::default()`) when no entry exists — either
2050 /// a pre-ST5 spawn, or a spawner stack that never layered
2051 /// `AgentContextMiddleware` (fail-open, mirroring [`Self::output_tail`]'s
2052 /// "no entry = empty default" convention).
2053 ///
2054 /// `crates/mlua-swarm-server/src/worker.rs`'s `GET /v1/worker/prompt`
2055 /// handler reads this back to filter `WorkerPayload.context.steps` via
2056 /// `ContextPolicy::allows_step`, without re-deriving the policy from
2057 /// the Blueprint at fetch time (`projection-adapter` ST5).
2058 pub async fn context_policy_for(
2059 &self,
2060 task_id: &StepId,
2061 attempt: u32,
2062 ) -> mlua_swarm_schema::ContextPolicy {
2063 let key = (task_id.clone(), attempt);
2064 self.with_state("context_policy_for", move |s| {
2065 s.agent_ctx
2066 .get(&key)
2067 .map(|e| e.policy.clone())
2068 .unwrap_or_default()
2069 })
2070 .await
2071 .unwrap_or_default()
2072 }
2073
2074 /// GH #23: returns the Blueprint-wide
2075 /// [`crate::core::step_naming::StepNaming`] table snapshotted for
2076 /// `task_id` (the same `Arc` `crate::blueprint::EngineDispatcher::dispatch`
2077 /// stashed into `EngineState.step_namings` at dispatch time —
2078 /// `Self::start_task`'s `StepId`, not the `TaskId` work item). `None`
2079 /// when no entry exists — either the dispatcher was never given a
2080 /// `StepNaming` (`EngineDispatcher::with_step_naming` not called) or
2081 /// the lock could not be acquired; callers are expected to fall back
2082 /// to the pre-GH-#23 runtime union rule in that case (subtask-2/3
2083 /// consumers).
2084 pub async fn step_naming_for(
2085 &self,
2086 task_id: &StepId,
2087 ) -> Option<Arc<crate::core::step_naming::StepNaming>> {
2088 let key = task_id.clone();
2089 self.with_state("step_naming_for", move |s| {
2090 s.step_namings.get(&key).cloned()
2091 })
2092 .await
2093 .ok()
2094 .flatten()
2095 }
2096
2097 /// GH #27 (follow-up to #23): returns the Blueprint-wide
2098 /// [`crate::core::projection_placement::ProjectionPlacement`] resolver
2099 /// snapshotted for `task_id` (the same `Arc`
2100 /// `crate::blueprint::EngineDispatcher::dispatch` stashed into
2101 /// `EngineState.projection_placements` at dispatch time — mirroring
2102 /// [`Self::step_naming_for`]'s contract exactly). `None` when no entry
2103 /// exists — either the dispatcher was never given a
2104 /// `ProjectionPlacement` (`EngineDispatcher::with_projection_placement`
2105 /// not called) or the lock could not be acquired; callers are expected
2106 /// to fall back to `ProjectionPlacement::default()` (byte-compat with
2107 /// the pre-#27 hardcoded layout) in that case.
2108 pub async fn projection_placement_for(
2109 &self,
2110 task_id: &StepId,
2111 ) -> Option<Arc<crate::core::projection_placement::ProjectionPlacement>> {
2112 let key = task_id.clone();
2113 self.with_state("projection_placement_for", move |s| {
2114 s.projection_placements.get(&key).cloned()
2115 })
2116 .await
2117 .ok()
2118 .flatten()
2119 }
2120
2121 /// Returns the [`crate::core::agent_context::AgentContextView`]
2122 /// snapshotted for `(task_id, attempt)`, if `AgentContextMiddleware`
2123 /// stashed one — the same lookup [`Self::fetch_worker_payload`] /
2124 /// [`Self::fetch_worker_payload_trusted`] perform inline, exposed
2125 /// standalone for callers that only need the view (not a full
2126 /// `WorkerPayload`) — e.g. the HTTP debug-plane `GET
2127 /// /v1/tasks/:id/runs/:run/steps*` handlers resolving a
2128 /// materialized-file root for a step *other than* the one currently
2129 /// fetching its own prompt (`projection-adapter` ST5).
2130 pub async fn agent_context_for(
2131 &self,
2132 task_id: &StepId,
2133 attempt: u32,
2134 ) -> Option<crate::core::agent_context::AgentContextView> {
2135 let key = (task_id.clone(), attempt);
2136 self.with_state("agent_context_for", move |s| {
2137 s.agent_ctx.get(&key).map(|e| e.view.clone())
2138 })
2139 .await
2140 .ok()
2141 .flatten()
2142 }
2143
2144 /// Read the current attempt number for a task (server-side lookup, no
2145 /// token verification). Used on `HTTP /v1/worker/result` when the
2146 /// worker omits `attempt` and the server has to fill it in.
2147 pub async fn task_attempt(&self, task_id: &StepId) -> Result<u32, EngineError> {
2148 let task_id = task_id.clone();
2149 self.with_state("task_attempt", move |s| {
2150 s.tasks
2151 .get(&task_id)
2152 .map(|t| t.attempt)
2153 .ok_or_else(|| EngineError::TaskNotFound(task_id.to_string()))
2154 })
2155 .await?
2156 }
2157
2158 /// Server-side admin API that lets `OperatorSpawner::spawn` bake the
2159 /// rendered `system_prompt` into engine state. There is no verb gate
2160 /// — the only expected caller is inside the spawner. SubAgents fetch
2161 /// this alongside the prompt on the `/v1/worker/prompt` path.
2162 pub async fn bake_worker_system_prompt(
2163 &self,
2164 task_id: &StepId,
2165 attempt: u32,
2166 system: Option<String>,
2167 ) -> Result<(), EngineError> {
2168 let task_id = task_id.clone();
2169 self.with_state("bake_worker_system_prompt", move |s| {
2170 // GH #31: record this agent's most-recently-baked render size
2171 // before `system` is moved into `s.systems.insert` below. Same
2172 // `s.tasks.get(&task_id)` → `.spec.agent` lookup pattern
2173 // `fetch_worker_payload` uses (see its doc for why this keying
2174 // is load-bearing for a later `bp_doctor` route).
2175 if let Some(rendered) = system.as_ref() {
2176 if let Some(agent) = s.tasks.get(&task_id).map(|t| t.spec.agent.clone()) {
2177 s.agent_render_sizes.insert(agent, rendered.len());
2178 }
2179 }
2180 s.systems.insert((task_id, attempt), system);
2181 })
2182 .await?;
2183 Ok(())
2184 }
2185
2186 /// GH #31: the most-recently-baked `system_prompt` render size (in
2187 /// bytes) observed for `agent_name`, if `bake_worker_system_prompt` has
2188 /// ever recorded one — last-write-wins across every `(task_id,
2189 /// attempt)` dispatch of that agent. `None` when no `system_prompt`
2190 /// has ever been baked for this agent name. Read by the `bp_doctor`
2191 /// route this subtask's follow-up adds.
2192 pub async fn agent_last_rendered_size(&self, agent_name: &str) -> Option<usize> {
2193 let agent_name = agent_name.to_string();
2194 self.with_state("agent_last_rendered_size", move |s| {
2195 s.agent_render_sizes.get(&agent_name).copied()
2196 })
2197 .await
2198 .ok()
2199 .flatten()
2200 }
2201
2202 /// GH #31: plain read-through of the baked `system` string for
2203 /// `(task_id, attempt)` from `EngineState.systems`, with no threshold
2204 /// branching. Backs `GET /v1/worker/prompt/system` (the `Http`-mode
2205 /// fetch target `system_ref.uri` points at) — that route needs the
2206 /// exact raw bytes to serve as the response body for the client's
2207 /// sha256 verification, not a `WorkerPayload`-wrapped value.
2208 ///
2209 /// Distinct from `apply_system_ref_threshold` (private, mutates an
2210 /// already-built `WorkerPayload` in place after full construction):
2211 /// this accessor has no threshold logic and is `pub` so
2212 /// `mlua-swarm-server`'s `worker` module can call it directly.
2213 ///
2214 /// Returns `Ok(None)` if no baked system exists for that `(task_id,
2215 /// attempt)` (either the task/attempt has no entry in `s.systems`, or
2216 /// the entry is present but stores `None`) — the caller maps this to
2217 /// a 404.
2218 pub async fn raw_system_prompt(
2219 &self,
2220 task_id: &StepId,
2221 attempt: u32,
2222 ) -> Result<Option<String>, EngineError> {
2223 let task_id = task_id.clone();
2224 self.with_state("raw_system_prompt", move |s| {
2225 s.systems.get(&(task_id, attempt)).cloned().unwrap_or(None)
2226 })
2227 .await
2228 }
2229
2230 /// Fetch an arbitrary named resource previously stored via
2231 /// `set_resource`. Not task-scoped — any valid token with the
2232 /// `FetchData` verb may read any key.
2233 pub async fn fetch_data(&self, token: &CapToken, key: &str) -> Result<Value, EngineError> {
2234 self.verify_token(token, Verb::FetchData).await?;
2235 let key = key.to_string();
2236 self.with_state("fetch_data", move |s| {
2237 s.resources
2238 .get(&key)
2239 .cloned()
2240 .ok_or(EngineError::ResourceNotFound(key))
2241 })
2242 .await?
2243 }
2244
2245 // ───────────────────────────────────────────────────────────────────────
2246 // Output path.
2247 // ───────────────────────────────────────────────────────────────────────
2248
2249 /// Send one output event from inside a `SpawnerAdapter` or worker.
2250 /// Structuring is assumed to be complete by the time we cross the
2251 /// `SpawnerAdapter` boundary; this API just appends to the
2252 /// `OutputStore`, pushes to the `EventLog`, and (for `Final`) emits
2253 /// the `TaskAttemptCompleted` event.
2254 ///
2255 /// This is Domain-side plumbing: it feeds the engine's verdict flow,
2256 /// not the Data-plane store in the `output_store` module. It also
2257 /// does not wake the dispatch path — that is done through the
2258 /// spawner's completion oneshot when the worker terminates.
2259 ///
2260 /// # Submit-time projection sink (subtask-4 / ST2 rework)
2261 ///
2262 /// A `Final` event additionally fans out to the submit-time projection
2263 /// sink ([`Self::materialize_final_submission`]): (a) when
2264 /// [`Self::set_output_store`] has wired a Data-plane
2265 /// [`crate::store::output::OutputStore`], the event is dual-written
2266 /// there (`producer_agent` = `TaskState.spec.agent`, resolved to its
2267 /// GH #23 canonical projection name — see below), and (b) when this
2268 /// task's spawn ran through `AgentContextMiddleware` (so
2269 /// `EngineState.agent_ctx` has a `.view.work_dir` / `.view.project_root`
2270 /// for it), the value is additionally materialized to the
2271 /// [`crate::core::projection_placement::ProjectionPlacement`]
2272 /// resolver's target (byte-compat default layout
2273 /// `<root>/workspace/tasks/<task_id>/ctx/<canonical_agent>.md`) — see
2274 /// `crate::core::projection`'s module doc.
2275 ///
2276 /// **GH #23 subtask-2 (canonical sink):** both writes above key off the
2277 /// canonical name — `Engine::step_naming_for(task_id)`'s
2278 /// `StepNaming::canonical_of_producer(producer_agent)` when a table was
2279 /// snapshotted for this task (`EngineDispatcher::with_step_naming`),
2280 /// else `producer_agent` unchanged (fail-open, byte-identical to
2281 /// pre-GH-#23 behavior — see [`crate::core::step_naming`]'s module
2282 /// doc).
2283 ///
2284 /// **Invariants** (Subtask 4): (1) this sink is fail-open — an
2285 /// unresolved root, an unconfigured `OutputStore`, or either one
2286 /// erroring, only logs a `tracing::warn!` and never turns this
2287 /// `Ok(())` into an `Err`; (2) the wired `OutputStore` stays the single
2288 /// source of truth for cross-step queries — the materialized file is a
2289 /// projection of it, not a second store; (3) core does not depend on
2290 /// `mlua-swarm-server` — everything this sink touches
2291 /// (`crate::store::output` / `crate::core::projection`) already lives
2292 /// in this crate.
2293 ///
2294 /// # `Artifact` dual-write (GH #34 subtask-3 gap fix)
2295 ///
2296 /// An `Artifact` event ALSO fans out to the Data-plane, via
2297 /// [`Self::materialize_artifact_submission`] — general-form: every
2298 /// `Artifact` submitted through this API dual-writes, no name-prefix
2299 /// gate. Unlike `Final`, the dual-write key is the artifact's own
2300 /// `name` field, verbatim — NOT resolved through the GH #23 canonical
2301 /// `StepNaming` table. An artifact's `name` IS its identity (mirrors
2302 /// [`crate::store::output::OutputStore::get_latest_by_name`]'s doc),
2303 /// so no canonicalization applies. Same fail-open discipline as
2304 /// `Final` (Invariant 1 above), but `Artifact` does NOT drive the
2305 /// file-materialize half (b) — artifact findings (e.g.
2306 /// `AfterRunAuditMiddleware`'s `"audit:<step_ref>"`) are observational
2307 /// sidecar data, not a step's own submission a work_dir/project_root
2308 /// projection needs to track. `Progress` / `Partial` events are
2309 /// unaffected — no behavior change.
2310 pub async fn submit_output(
2311 &self,
2312 token: &crate::types::CapToken,
2313 task_id: &StepId,
2314 attempt: u32,
2315 event: crate::worker::output::OutputEvent,
2316 ) -> Result<(), EngineError> {
2317 self.verify_token_for_task(token, crate::types::Verb::EmitOutput, task_id)
2318 .await?;
2319 // GH #51 — completion-time verdict-contract enforcement, embedded
2320 // choke point 2 of 2 (see `Self::verdict_contract_completion_check`'s
2321 // doc). Guarded to `Final` only — the ONLY `OutputEvent` variant a
2322 // verdict contract's completion can meaningfully address; this
2323 // guard is defensive (this function is empirically called with
2324 // `Final` only today, both from `worker.rs`'s `worker_result` and
2325 // from `operator.rs`'s WS fallback) but costs nothing and protects
2326 // against a future non-`Final` caller. Runs BEFORE the
2327 // `output_tail` write immediately below: on `Err`, this returns
2328 // immediately and the write never happens — a rejected value
2329 // never reaches `output_tail` / the flow ctx.
2330 if let crate::worker::output::OutputEvent::Final { content, ok } = &event {
2331 let comparable_value = content_ref_to_comparable_string(content.clone());
2332 self.verdict_contract_completion_check(task_id, attempt, *ok, &comparable_value)
2333 .await?;
2334 }
2335 let task_id_for_apply = task_id.clone();
2336 let event_clone = event.clone();
2337 self.with_state("submit_output", move |s| {
2338 s.output_store
2339 .entry((task_id_for_apply.clone(), attempt))
2340 .or_default()
2341 .push(event_clone.clone());
2342 s.push_event(crate::core::state::Event::WorkerOutput {
2343 task_id: task_id_for_apply,
2344 attempt,
2345 event: event_clone,
2346 });
2347 })
2348 .await?;
2349 match &event {
2350 crate::worker::output::OutputEvent::Final { content, ok } => {
2351 self.materialize_final_submission(task_id, attempt, content, *ok)
2352 .await?;
2353 }
2354 crate::worker::output::OutputEvent::Artifact { name, content } => {
2355 self.materialize_artifact_submission(task_id, attempt, name, content)
2356 .await?;
2357 }
2358 _ => {}
2359 }
2360 Ok(())
2361 }
2362
2363 /// Submit-time projection sink (subtask-4 / ST2 rework) shared by
2364 /// [`Self::submit_output`] and [`Self::submit_worker_result_trusted`].
2365 /// Best-effort / fail-open throughout (see `submit_output`'s doc
2366 /// Invariants): every failure path only `tracing::warn!`s and returns.
2367 ///
2368 /// Reads `(producer_agent, view)` via one read-only [`Self::with_state`]
2369 /// call — `producer_agent` off `TaskState.spec.agent`, `view` (the
2370 /// full [`crate::core::agent_context::AgentContextView`]) off
2371 /// `EngineState.agent_ctx[(task_id, attempt)]`, the same snapshot
2372 /// `crate::middleware::agent_context::AgentContextMiddleware` writes at
2373 /// spawn time — then does its actual (dual-write / file-write) work
2374 /// *outside* that lock, so a slow disk write or Data-plane store call
2375 /// never holds up unrelated `Engine::with_state` callers. `root` itself
2376 /// is resolved from `view` AFTER the lock via
2377 /// [`crate::core::projection_placement::ProjectionPlacement::resolve_root`]
2378 /// (GH #27, follow-up to #23) — the SAME resolver
2379 /// [`Self::step_naming_for`]'s sibling accessor
2380 /// [`Self::projection_placement_for`] snapshotted at dispatch time, so
2381 /// this sink's root-preference / fallback order is identical to the
2382 /// server read-back and the spawn-time pointer.
2383 async fn materialize_final_submission(
2384 &self,
2385 task_id: &StepId,
2386 attempt: u32,
2387 content: &crate::worker::output::ContentRef,
2388 ok: bool,
2389 ) -> Result<(), EngineError> {
2390 let server_policy = self.cfg().check_policy;
2391 let task_id_for_lookup = task_id.clone();
2392 let lookup = self
2393 .with_state("materialize_final_submission.lookup", move |s| {
2394 let entry = s.tasks.get(&task_id_for_lookup);
2395 let producer_agent = entry.map(|t| t.spec.agent.clone());
2396 let task_policy = entry.and_then(|t| t.spec.check_policy);
2397 let view = s
2398 .agent_ctx
2399 .get(&(task_id_for_lookup.clone(), attempt))
2400 .map(|e| e.view.clone());
2401 (producer_agent, task_policy, view)
2402 })
2403 .await;
2404 // Per-task `TaskSpec.check_policy` (ST1c) wins
2405 // over the server-wide `EngineCfg.check_policy` when set — a
2406 // per-run override forwarded from the launch entry point (see
2407 // `TaskLaunchRequest.check_policy` /
2408 // `TaskLaunchInput.check_policy`). `None` leaves the server
2409 // default in effect (backward compat).
2410 let policy = lookup
2411 .as_ref()
2412 .ok()
2413 .and_then(|(_, tp, _)| *tp)
2414 .unwrap_or(server_policy);
2415 let (producer_agent, view) = match lookup.map(|(pa, _, view)| (pa, view)) {
2416 Ok(pair) => pair,
2417 Err(err) => {
2418 if !matches!(policy, crate::core::config::CheckPolicy::Silent) {
2419 tracing::warn!(
2420 %task_id,
2421 error = %err,
2422 "submit-time projection sink: state lookup failed; skipping (fail-open)"
2423 );
2424 }
2425 apply_check_policy(
2426 policy,
2427 "submit-time projection sink: state lookup",
2428 "state lookup failed; skipping (fail-open)",
2429 )?;
2430 return Ok(());
2431 }
2432 };
2433 let Some(producer_agent) = producer_agent else {
2434 // Defensive only: `task_id` is always a just-looked-up task at
2435 // every real call site. No task, no addressable producer name
2436 // — nothing to project. Not gated by `CheckPolicy` — a missing
2437 // task is an intentional early-exit path, not a fail-open
2438 // condition to surface.
2439 return Ok(());
2440 };
2441 let placement = self
2442 .projection_placement_for(task_id)
2443 .await
2444 .unwrap_or_default();
2445 let root = view.and_then(|v| placement.resolve_root(&v));
2446
2447 // GH #23 subtask-2: resolve `producer_agent` to its canonical
2448 // projection name via the Blueprint-wide `StepNaming` table
2449 // snapshotted at dispatch time (`Engine::step_naming_for`). Both
2450 // write paths below ((a) data-plane, (b) file stem) use the
2451 // *canonical* name — `StepNaming::canonical_of_producer` returns
2452 // `producer_agent` unchanged for undeclared steps (byte-identical
2453 // to pre-GH-#23 behavior), and `None` (no table for this
2454 // `task_id`, e.g. a spawn that never went through
2455 // `EngineDispatcher::with_step_naming`) is a defensive fail-open
2456 // to the raw `producer_agent`, same discipline as the rest of this
2457 // sink.
2458 let canonical_agent = self
2459 .step_naming_for(task_id)
2460 .await
2461 .and_then(|naming| {
2462 naming
2463 .canonical_of_producer(&producer_agent)
2464 .map(str::to_string)
2465 })
2466 .unwrap_or_else(|| producer_agent.clone());
2467
2468 // (a) Data-plane dual-write, when an OutputStore backend is wired.
2469 if let Some(store) = self.output_store_backend() {
2470 if let Err(err) = store
2471 .append(
2472 task_id.as_str(),
2473 attempt,
2474 &canonical_agent,
2475 crate::worker::output::OutputEvent::Final {
2476 content: content.clone(),
2477 ok,
2478 },
2479 Vec::new(),
2480 )
2481 .await
2482 {
2483 if !matches!(policy, crate::core::config::CheckPolicy::Silent) {
2484 tracing::warn!(
2485 %task_id,
2486 agent = %producer_agent,
2487 canonical = %canonical_agent,
2488 error = %err,
2489 "submit-time projection sink: OutputStore dual-write failed (fail-open)"
2490 );
2491 }
2492 apply_check_policy(
2493 policy,
2494 "submit-time projection sink: OutputStore dual-write",
2495 "OutputStore dual-write failed (fail-open)",
2496 )?;
2497 }
2498 }
2499
2500 // (b) File materialize, when a root resolved.
2501 let Some(root) = root else {
2502 if !matches!(policy, crate::core::config::CheckPolicy::Silent) {
2503 tracing::warn!(
2504 %task_id,
2505 agent = %producer_agent,
2506 canonical = %canonical_agent,
2507 "submit-time projection sink: no work_dir/project_root resolved; skipping file materialize (fail-open)"
2508 );
2509 }
2510 apply_check_policy(
2511 policy,
2512 "submit-time projection sink: file materialize",
2513 "no work_dir/project_root resolved; skipping file materialize (fail-open)",
2514 )?;
2515 return Ok(());
2516 };
2517 let value = match content {
2518 crate::worker::output::ContentRef::Inline { value } => value.clone(),
2519 crate::worker::output::ContentRef::FileRef {
2520 path,
2521 mime,
2522 size_hint,
2523 } => serde_json::json!({
2524 "file_ref": path.to_string_lossy(),
2525 "mime": mime,
2526 "size_hint": size_hint,
2527 }),
2528 };
2529 let key = crate::core::projection::ProjectionKey {
2530 task_id: task_id.to_string(),
2531 run_id: None,
2532 step: Some(canonical_agent.clone()),
2533 path: None,
2534 };
2535 let adapter = crate::core::projection::FileProjectionAdapter::with_placement(
2536 root,
2537 (*placement).clone(),
2538 );
2539 if let Err(err) = adapter.materialize_submission(&key, &value, attempt, ok) {
2540 if !matches!(policy, crate::core::config::CheckPolicy::Silent) {
2541 tracing::warn!(
2542 %task_id,
2543 agent = %producer_agent,
2544 canonical = %canonical_agent,
2545 error = %err,
2546 "submit-time projection sink: file materialize failed (fail-open)"
2547 );
2548 }
2549 apply_check_policy(
2550 policy,
2551 "submit-time projection sink: file materialize",
2552 "file materialize failed (fail-open)",
2553 )?;
2554 }
2555 Ok(())
2556 }
2557
2558 /// Submit-time projection sink for `OutputEvent::Artifact` (GH #34
2559 /// subtask-3, later extended to drive the file half too). Two halves, the
2560 /// [`Self::materialize_final_submission`] mirror for staged named parts:
2561 ///
2562 /// - **Data-plane dual-write** — when [`Self::set_output_store`] has
2563 /// wired a [`crate::store::output::OutputStore`], the artifact
2564 /// dual-writes there under its own `name`, verbatim (general form:
2565 /// every `Artifact` staged via [`Self::submit_output`] /
2566 /// [`Self::stage_worker_artifact_trusted`] materializes this way, no
2567 /// name-prefix gate).
2568 /// - **File materialize** — when a `root` resolves off the spawn-time
2569 /// [`crate::core::agent_context::AgentContextView`], the part's
2570 /// content is written raw to `<ctx-dir>/<name>` via
2571 /// [`crate::core::projection::FileProjectionAdapter::materialize_part`].
2572 /// That file is the IN file the *next* Agent step reads: materializing
2573 /// a Step's OUTPUT to disk is the
2574 /// [`crate::core::projection::FileProjectionAdapter`]'s
2575 /// responsibility, and a staged named part is as much an OUTPUT the
2576 /// next step consumes as a `Final` is — so the sink materializes it
2577 /// too, rather than leaving parts Data-plane-only.
2578 ///
2579 /// Unlike the Final sink, no `StepNaming` canonicalization is applied:
2580 /// an artifact's `name` already IS the key both halves address (it
2581 /// names the file directly, extension included — `plan.md` — so
2582 /// `materialize_part` writes it verbatim, not through the `<stem>.md`
2583 /// synthesis the Final sink's canonical-agent path uses).
2584 ///
2585 /// Fail-open throughout, the same `check_policy` cascade as
2586 /// [`Self::materialize_final_submission`]: a per-task lookup error falls
2587 /// back to the server default (and a `None` view ⇒ the file half's
2588 /// unresolved-root path), an unconfigured `OutputStore` skips the
2589 /// dual-write, an unresolved root skips the file half, and a
2590 /// dual-write / file-write / name-guard error only `tracing::warn!`s
2591 /// (`Silent` suppresses even that) before applying [`apply_check_policy`]
2592 /// (`Strict` surfaces an [`EngineError`], `Warn` / `Silent` return
2593 /// `Ok(())`) — a staged part never turns a would-have-succeeded submit
2594 /// into a failure under the default policy.
2595 async fn materialize_artifact_submission(
2596 &self,
2597 task_id: &StepId,
2598 attempt: u32,
2599 name: &str,
2600 content: &crate::worker::output::ContentRef,
2601 ) -> Result<(), EngineError> {
2602 // Per-task `TaskSpec.check_policy` override + the `AgentContextView`
2603 // snapshot, resolved in ONE read-only `with_state` (the same lock
2604 // the policy lookup already needed — no extra `with_state` for the
2605 // view). Silent per-task lookup failure (`with_state` error) falls
2606 // back to the server-wide default and a `None` view (⇒ the file
2607 // half's own unresolved-root fail-open path); this sink never
2608 // surfaces the lookup error itself as a step failure.
2609 let server_policy = self.cfg().check_policy;
2610 let task_id_for_lookup = task_id.clone();
2611 let lookup = self
2612 .with_state("materialize_artifact_submission.lookup", move |s| {
2613 let task_policy = s
2614 .tasks
2615 .get(&task_id_for_lookup)
2616 .and_then(|t| t.spec.check_policy);
2617 let view = s
2618 .agent_ctx
2619 .get(&(task_id_for_lookup.clone(), attempt))
2620 .map(|e| e.view.clone());
2621 (task_policy, view)
2622 })
2623 .await
2624 .ok();
2625 let policy = lookup
2626 .as_ref()
2627 .and_then(|(tp, _)| *tp)
2628 .unwrap_or(server_policy);
2629 let view = lookup.and_then(|(_, view)| view);
2630
2631 // (a) Data-plane dual-write, when an OutputStore backend is wired —
2632 // the artifact's own `name` is its Data-plane key (no
2633 // canonicalization, unlike the Final sink's `StepNaming`
2634 // resolution).
2635 if let Some(store) = self.output_store_backend() {
2636 if let Err(err) = store
2637 .append(
2638 task_id.as_str(),
2639 attempt,
2640 name,
2641 crate::worker::output::OutputEvent::Artifact {
2642 name: name.to_string(),
2643 content: content.clone(),
2644 },
2645 Vec::new(),
2646 )
2647 .await
2648 {
2649 if !matches!(policy, crate::core::config::CheckPolicy::Silent) {
2650 tracing::warn!(
2651 %task_id,
2652 artifact = %name,
2653 error = %err,
2654 "submit-time projection sink: OutputStore dual-write failed for Artifact (fail-open)"
2655 );
2656 }
2657 apply_check_policy(
2658 policy,
2659 "submit-time projection sink: Artifact OutputStore dual-write",
2660 "OutputStore dual-write failed for Artifact (fail-open)",
2661 )?;
2662 }
2663 }
2664
2665 // (b) File materialize, when a root resolved — writes the staged
2666 // part raw to `<ctx-dir>/<name>`, the IN file the next Agent step
2667 // reads (see `FileProjectionAdapter::materialize_part`'s doc for
2668 // why raw / why the name is verbatim). A name-guard violation lands
2669 // on the same fail-open path as any other write error below.
2670 let placement = self
2671 .projection_placement_for(task_id)
2672 .await
2673 .unwrap_or_default();
2674 let Some(root) = view.and_then(|v| placement.resolve_root(&v)) else {
2675 if !matches!(policy, crate::core::config::CheckPolicy::Silent) {
2676 tracing::warn!(
2677 %task_id,
2678 artifact = %name,
2679 "submit-time projection sink: no work_dir/project_root resolved; skipping part file materialize (fail-open)"
2680 );
2681 }
2682 apply_check_policy(
2683 policy,
2684 "submit-time projection sink: part file materialize",
2685 "no work_dir/project_root resolved; skipping part file materialize (fail-open)",
2686 )?;
2687 return Ok(());
2688 };
2689 let value = match content {
2690 crate::worker::output::ContentRef::Inline { value } => value.clone(),
2691 crate::worker::output::ContentRef::FileRef {
2692 path,
2693 mime,
2694 size_hint,
2695 } => serde_json::json!({
2696 "file_ref": path.to_string_lossy(),
2697 "mime": mime,
2698 "size_hint": size_hint,
2699 }),
2700 };
2701 let adapter = crate::core::projection::FileProjectionAdapter::with_placement(
2702 root,
2703 (*placement).clone(),
2704 );
2705 if let Err(err) = adapter.materialize_part(task_id.as_str(), name, &value) {
2706 if !matches!(policy, crate::core::config::CheckPolicy::Silent) {
2707 tracing::warn!(
2708 %task_id,
2709 artifact = %name,
2710 error = %err,
2711 "submit-time projection sink: part file materialize failed (fail-open)"
2712 );
2713 }
2714 apply_check_policy(
2715 policy,
2716 "submit-time projection sink: part file materialize",
2717 "part file materialize failed (fail-open)",
2718 )?;
2719 }
2720 Ok(())
2721 }
2722
2723 /// Snapshot the entire output tail for a given `(task_id, attempt)`.
2724 /// Used by the dispatch path when pulling `Final`, and by observers
2725 /// reading the trace.
2726 pub async fn output_tail(
2727 &self,
2728 task_id: &StepId,
2729 attempt: u32,
2730 ) -> Vec<crate::worker::output::OutputEvent> {
2731 let key = (task_id.clone(), attempt);
2732 self.with_state("output_tail", move |s| {
2733 s.output_store.get(&key).cloned().unwrap_or_default()
2734 })
2735 .await
2736 .unwrap_or_default()
2737 }
2738
2739 /// Record an interim `last_result` for `task_id` without changing its
2740 /// `status`. Distinct from the terminal `Final` output event handled
2741 /// through `submit_output` / `dispatch_attempt_with`.
2742 pub async fn post_result(
2743 &self,
2744 token: &CapToken,
2745 task_id: &StepId,
2746 result: Value,
2747 ) -> Result<(), EngineError> {
2748 self.verify_token_for_task(token, Verb::PostResult, task_id)
2749 .await?;
2750 let task_id = task_id.clone();
2751 let result_clone = result.clone();
2752 self.with_state("post_result", move |s| {
2753 let task = s
2754 .tasks
2755 .get_mut(&task_id)
2756 .ok_or_else(|| EngineError::TaskNotFound(task_id.to_string()))?;
2757 task.last_result = Some(result_clone);
2758 task.updated_at = now_unix();
2759 Ok::<(), EngineError>(())
2760 })
2761 .await??;
2762 Ok(())
2763 }
2764
2765 /// Store a named resource value, retrievable later via `fetch_data`.
2766 /// No token is required — this is a server-side/admin-style setter
2767 /// (mirrors `bake_worker_system_prompt`).
2768 pub async fn set_resource(
2769 &self,
2770 key: impl Into<String>,
2771 value: Value,
2772 ) -> Result<(), EngineError> {
2773 let key = key.into();
2774 self.with_state("set_resource", move |s| {
2775 s.resources.insert(key, value);
2776 })
2777 .await?;
2778 Ok(())
2779 }
2780
2781 // ═══════════════════════════════════════════════════════════════════════
2782 // Senior suspend / resume
2783 // ═══════════════════════════════════════════════════════════════════════
2784
2785 /// Ask a question of the Senior, mark the task `Suspended`, and
2786 /// return a `ResumeKey`. The suspended state persists until another
2787 /// task calls `resume(key, answer)`.
2788 ///
2789 /// Resume-side waiting is `Notify`-based, so a caller (typically
2790 /// MainAI) can detach, reattach from a different process, and still
2791 /// pull the answer out via `await_resume(key, timeout)` — the answer
2792 /// is stored inside `EngineState`.
2793 pub async fn query_senior(
2794 &self,
2795 token: &CapToken,
2796 task_id: &StepId,
2797 question: Value,
2798 ) -> Result<ResumeKey, EngineError> {
2799 self.verify_token(token, Verb::QuerySenior).await?;
2800 let task_id = task_id.clone();
2801 let key = ResumeKey::for_senior(&task_id);
2802 let task_notify = self
2803 .with_state("query_senior.notify_ensure", |s| {
2804 s.ensure_task_notify(&task_id)
2805 })
2806 .await?;
2807
2808 let key_clone = key.clone();
2809 let task_id_inner = task_id.clone();
2810 let question_clone = question.clone();
2811 self.with_state("query_senior.suspend", move |s| {
2812 let task = s
2813 .tasks
2814 .get_mut(&task_id_inner)
2815 .ok_or_else(|| EngineError::TaskNotFound(task_id_inner.to_string()))?;
2816 task.status = TaskStatus::Suspended;
2817 task.suspended_on = Some(key_clone.clone());
2818 task.updated_at = now_unix();
2819 s.pending_resumes
2820 .insert(key_clone.clone(), ResumePending::new());
2821 s.push_event(Event::SeniorQueried {
2822 task_id: task_id_inner.clone(),
2823 question: question_clone.clone(),
2824 });
2825 s.push_event(Event::TaskSuspended {
2826 task_id: task_id_inner.clone(),
2827 key: key_clone.clone(),
2828 });
2829 Ok::<(), EngineError>(())
2830 })
2831 .await??;
2832
2833 // Notify callers waiting for a task status change (Running → Suspended).
2834 task_notify.notify_waiters();
2835
2836 let _ = self
2837 .inner
2838 .event_tx
2839 .send(Event::SeniorQueried { task_id, question });
2840 Ok(key)
2841 }
2842
2843 /// Store the answer for a `ResumeKey` in `EngineState` and wake the
2844 /// waiting caller via `Notify`. Also flips the suspended task's
2845 /// status back to `Running` and fires the per-task notifier.
2846 pub async fn resume(&self, key: ResumeKey, answer: Value) -> Result<(), EngineError> {
2847 let answer_for_state = answer.clone();
2848 let answer_for_event = answer.clone();
2849 let key_clone = key.clone();
2850 let (notify, task_notify, task_id_opt) = self
2851 .with_state("resume.set", move |s| {
2852 let pending = s
2853 .pending_resumes
2854 .get_mut(&key_clone)
2855 .ok_or(EngineError::ResumeKeyNotFound)?;
2856 pending.answer = Some(answer_for_state);
2857 let notify = pending.notify.clone();
2858
2859 let task_id = s
2860 .tasks
2861 .iter()
2862 .find(|(_, t)| t.suspended_on.as_ref() == Some(&key_clone))
2863 .map(|(id, _)| id.clone());
2864
2865 let task_notify = task_id.as_ref().map(|tid| s.ensure_task_notify(tid));
2866
2867 if let Some(tid) = &task_id {
2868 if let Some(task) = s.tasks.get_mut(tid) {
2869 task.suspended_on = None;
2870 task.status = TaskStatus::Running;
2871 task.updated_at = now_unix();
2872 }
2873 s.push_event(Event::TaskResumed {
2874 task_id: tid.clone(),
2875 key: key_clone.clone(),
2876 });
2877 s.push_event(Event::SeniorAnswered {
2878 task_id: tid.clone(),
2879 answer: answer_for_event.clone(),
2880 });
2881 }
2882 Ok::<_, EngineError>((notify, task_notify, task_id))
2883 })
2884 .await??;
2885
2886 // Outside the lock: notify_waiters for both the ResumePending and task-status waits.
2887 notify.notify_waiters();
2888 if let Some(n) = task_notify {
2889 n.notify_waiters();
2890 }
2891
2892 if let Some(tid) = task_id_opt {
2893 let _ = self
2894 .inner
2895 .event_tx
2896 .send(Event::TaskResumed { task_id: tid, key });
2897 }
2898 Ok(())
2899 }
2900
2901 /// Wait for the resume answer. Even if the caller (an Operator)
2902 /// detached and reattached, the answer is available immediately here
2903 /// — if it was already stored, this returns without waiting on the
2904 /// notifier.
2905 ///
2906 /// `timeout = Duration::ZERO` performs an instant check without
2907 /// waiting.
2908 pub async fn await_resume(
2909 &self,
2910 key: ResumeKey,
2911 timeout: Duration,
2912 ) -> Result<Value, EngineError> {
2913 // (1) Under the lock: clone the notify handle and check for an existing answer.
2914 let key_clone = key.clone();
2915 let (notify, existing) = self
2916 .with_state("await_resume.snapshot", move |s| {
2917 let pending = s
2918 .pending_resumes
2919 .get(&key_clone)
2920 .ok_or(EngineError::ResumeKeyNotFound)?;
2921 Ok::<_, EngineError>((pending.notify.clone(), pending.answer.clone()))
2922 })
2923 .await??;
2924
2925 // (2) If an answer has already been stored, return immediately (detach / reattach pattern).
2926 if let Some(v) = existing {
2927 return Ok(v);
2928 }
2929
2930 // (3) Outside the lock: wait on the notify with a timeout.
2931 if timeout.is_zero() {
2932 return Err(EngineError::PollTimeout);
2933 }
2934 let waited = tokio::time::timeout(timeout, notify.notified()).await;
2935 if waited.is_err() {
2936 return Err(EngineError::PollTimeout);
2937 }
2938
2939 // (4) Under the lock: re-read the answer (should be present now that we were notified).
2940 let key_clone = key.clone();
2941 self.with_state("await_resume.read", move |s| {
2942 let pending = s
2943 .pending_resumes
2944 .get(&key_clone)
2945 .ok_or(EngineError::ResumeKeyNotFound)?;
2946 pending
2947 .answer
2948 .clone()
2949 .ok_or_else(|| EngineError::Internal("notified but answer missing".into()))
2950 })
2951 .await?
2952 }
2953
2954 // ═══════════════════════════════════════════════════════════════════════
2955 // poll_task — the "wait" path that waits for task-status changes (works for long-poll and regular wait).
2956 // ═══════════════════════════════════════════════════════════════════════
2957
2958 /// Wait until the task's status **transitions to terminal or
2959 /// `Suspended`**, then return the latest `TaskState`. Returns
2960 /// immediately if the task is already in a terminal state.
2961 /// Exceeding the timeout returns `EngineError::PollTimeout`.
2962 ///
2963 /// A `hold` of `Duration::from_secs(0)` returns a snapshot immediately
2964 /// (no wait). Larger holds — tens of minutes up to days — are fine;
2965 /// the wait state is kept in memory inside the engine and does not
2966 /// degrade.
2967 pub async fn poll_task(
2968 &self,
2969 token: &CapToken,
2970 task_id: &StepId,
2971 hold: Duration,
2972 ) -> Result<TaskState, EngineError> {
2973 self.verify_token_for_task(token, Verb::PollTask, task_id)
2974 .await?;
2975 let task_id_inner = task_id.clone();
2976
2977 // (1) Under the lock: take a snapshot and clone task_notify.
2978 let (state, notify) = self
2979 .with_state("poll_task.snapshot", move |s| {
2980 let task = s
2981 .tasks
2982 .get(&task_id_inner)
2983 .cloned()
2984 .ok_or_else(|| EngineError::TaskNotFound(task_id_inner.to_string()))?;
2985 let notify = s.ensure_task_notify(&task_id_inner);
2986 Ok::<_, EngineError>((task, notify))
2987 })
2988 .await??;
2989
2990 // (2) Immediate-return condition: already terminal / Suspended (nothing left to wait on).
2991 if matches!(
2992 state.status,
2993 TaskStatus::Pass | TaskStatus::Blocked | TaskStatus::Cancelled | TaskStatus::Suspended
2994 ) {
2995 return Ok(state);
2996 }
2997 if hold.is_zero() {
2998 return Ok(state);
2999 }
3000
3001 // (3) Outside the lock: wait on Notify with a timeout.
3002 let waited = tokio::time::timeout(hold, notify.notified()).await;
3003 if waited.is_err() {
3004 return Err(EngineError::PollTimeout);
3005 }
3006
3007 // (4) Under the lock: take a fresh snapshot.
3008 let task_id_inner = task_id.clone();
3009 self.with_state("poll_task.reread", move |s| {
3010 s.tasks
3011 .get(&task_id_inner)
3012 .cloned()
3013 .ok_or_else(|| EngineError::TaskNotFound(task_id_inner.to_string()))
3014 })
3015 .await?
3016 }
3017
3018 // ═══════════════════════════════════════════════════════════════════════
3019 // Background: heartbeat miss → detach loop
3020 // ═══════════════════════════════════════════════════════════════════════
3021
3022 /// Background loop that scans sessions every `heartbeat_interval` and
3023 /// flips `attached = false` on any session whose `last_seen` exceeds
3024 /// `heartbeat_miss_threshold * interval`.
3025 ///
3026 /// The tasks themselves are kept (assuming
3027 /// `keepalive_on_idle = true`), so another client can reattach with
3028 /// the same token and resume immediately. Dropping the returned
3029 /// `JoinHandle` does not stop the loop — the handle exists so callers
3030 /// who want to abort can hold onto it.
3031 pub fn start_detach_loop(&self) -> tokio::task::JoinHandle<()> {
3032 let engine = self.clone();
3033 let cfg = self.inner.cfg.long_hold.clone();
3034 let interval = cfg.heartbeat_interval;
3035 let miss_secs = cfg.heartbeat_interval.as_secs() * cfg.heartbeat_miss_threshold as u64;
3036
3037 tokio::spawn(async move {
3038 let mut ticker = tokio::time::interval(interval);
3039 ticker.tick().await; // first tick is immediate
3040 loop {
3041 ticker.tick().await;
3042 let now = now_unix();
3043 let detached = engine
3044 .with_state("detach_loop.scan", |s| {
3045 let mut detached = Vec::new();
3046 for (sid, sess) in s.sessions.iter_mut() {
3047 if !sess.attached {
3048 continue;
3049 }
3050 if now.saturating_sub(sess.last_seen) >= miss_secs {
3051 sess.attached = false;
3052 detached.push(sid.clone());
3053 }
3054 }
3055 for sid in &detached {
3056 s.push_event(Event::SessionDetached {
3057 session_id: sid.clone(),
3058 });
3059 }
3060 detached
3061 })
3062 .await
3063 .unwrap_or_default();
3064 for sid in detached {
3065 let _ = engine
3066 .inner
3067 .event_tx
3068 .send(Event::SessionDetached { session_id: sid });
3069 }
3070 }
3071 })
3072 }
3073
3074 /// Helper: wake a task whose status has changed. Called from the
3075 /// method body outside the lock.
3076 async fn wake_task(&self, task_id: &StepId) -> Result<(), EngineError> {
3077 let task_id = task_id.clone();
3078 let notify_opt = self
3079 .with_state("wake_task.get_notify", move |s| {
3080 s.task_notifies.get(&task_id).cloned()
3081 })
3082 .await?;
3083 if let Some(n) = notify_opt {
3084 n.notify_waiters();
3085 }
3086 Ok(())
3087 }
3088}
3089
3090/// Decide what a submit-time projection sink should do at a fail-open
3091/// branch given the configured [`crate::core::config::CheckPolicy`].
3092///
3093/// Returns `Ok(())` under [`CheckPolicy::Silent`] and
3094/// [`CheckPolicy::Warn`] — the caller continues with fail-open. Returns
3095/// [`EngineError::CheckPolicyStrict`] under [`CheckPolicy::Strict`],
3096/// carrying the caller-supplied `context` (call-site identifier) and
3097/// `message` (the pre-existing warn-log message literal, preserved
3098/// verbatim for log parse compatibility).
3099///
3100/// This helper deliberately does **not** call `tracing::warn!` itself —
3101/// the caller is responsible for firing the existing warn! (with its
3102/// full structured-field payload — `%task_id`, `agent`, `canonical`,
3103/// `error`, etc.) under `Warn` mode, and for skipping the warn! under
3104/// `Silent` mode. Keeping the warn! at the call site preserves the
3105/// exact structured-field shape every existing log-parse consumer sees;
3106/// forwarding it through the helper would either drop those fields or
3107/// require a macro (deferred, see subtask-1b).
3108///
3109/// Design intent: the fail-open discipline of every submit-time
3110/// projection sink is byte-identical to the pre-`CheckPolicy` behaviour
3111/// under the default [`CheckPolicy::Warn`]. `Silent` is a per-run opt-in
3112/// to suppress noise (e.g., a caller that has already verified upstream
3113/// invariants); `Strict` is a per-run opt-in to fail loudly (e.g., a
3114/// caller that requires all parts to materialize). See
3115/// [`crate::core::config::CheckPolicy`] for the "state dirty on fail"
3116/// semantics of `Strict`.
3117pub(crate) fn apply_check_policy(
3118 policy: crate::core::config::CheckPolicy,
3119 context: &str,
3120 message: &str,
3121) -> Result<(), EngineError> {
3122 match policy {
3123 crate::core::config::CheckPolicy::Silent | crate::core::config::CheckPolicy::Warn => Ok(()),
3124 crate::core::config::CheckPolicy::Strict => Err(EngineError::CheckPolicyStrict {
3125 context: context.to_string(),
3126 message: message.to_string(),
3127 }),
3128 }
3129}
3130
3131#[cfg(test)]
3132mod check_policy_helper_tests {
3133 use super::apply_check_policy;
3134 use crate::core::config::CheckPolicy;
3135 use crate::core::errors::EngineError;
3136
3137 /// `Silent` returns `Ok(())` without producing an error. Log
3138 /// suppression (the "no `tracing::warn!`" half of the semantics) is
3139 /// enforced at the call site, not inside the helper — see the
3140 /// helper's doc comment for why.
3141 #[test]
3142 fn silent_returns_ok() {
3143 let result = apply_check_policy(CheckPolicy::Silent, "call/site", "sink message");
3144 assert!(matches!(result, Ok(())));
3145 }
3146
3147 /// `Warn` (the default) returns `Ok(())` — the caller continues
3148 /// with fail-open, having already fired its own `tracing::warn!`
3149 /// with the full structured-field payload.
3150 #[test]
3151 fn warn_returns_ok() {
3152 let result = apply_check_policy(CheckPolicy::Warn, "call/site", "sink message");
3153 assert!(matches!(result, Ok(())));
3154 }
3155
3156 /// `Strict` returns
3157 /// [`EngineError::CheckPolicyStrict`] with `context` and `message`
3158 /// copied verbatim from the caller — the completion route surfaces
3159 /// this as a step / launch error so a caller that has opted in can
3160 /// fail fast instead of proceeding with a partially-realized
3161 /// submission.
3162 #[test]
3163 fn strict_returns_error_with_context_and_message() {
3164 let result = apply_check_policy(
3165 CheckPolicy::Strict,
3166 "submit-time projection sink: file materialize",
3167 "no work_dir/project_root resolved; skipping file materialize (fail-open)",
3168 );
3169 match result {
3170 Err(EngineError::CheckPolicyStrict { context, message }) => {
3171 assert_eq!(context, "submit-time projection sink: file materialize");
3172 assert_eq!(
3173 message,
3174 "no work_dir/project_root resolved; skipping file materialize (fail-open)"
3175 );
3176 }
3177 other => panic!("expected CheckPolicyStrict, got {:?}", other),
3178 }
3179 }
3180}
3181
3182// ─── UT: issue #14 — token store keyed by fingerprint, not nonce ────────────
3183#[cfg(test)]
3184mod token_fingerprint_store_tests {
3185 use super::*;
3186
3187 /// A token that was never attached fails verify with a `TokenNotFound`
3188 /// that carries the fingerprint — never the nonce. The error string can
3189 /// surface in HTTP error bodies, so this is the secret-hygiene contract.
3190 #[tokio::test]
3191 async fn verify_unknown_token_reports_fingerprint_not_nonce() {
3192 let engine = Engine::new(EngineCfg::default());
3193 // Signed by the engine's own signer (sig passes) but never inserted
3194 // into the store — verify must fail at step (4), the store lookup.
3195 let token = engine.signer().session(
3196 "ghost",
3197 Role::Operator,
3198 vec!["*".into()],
3199 Duration::from_secs(60),
3200 );
3201 let err = engine
3202 .verify_token(&token, Verb::ReadTaskState)
3203 .await
3204 .expect_err("token is not in the store");
3205 let msg = err.to_string();
3206 assert!(
3207 msg.contains(&token.fingerprint()),
3208 "error must carry the fingerprint: {msg}"
3209 );
3210 assert!(
3211 !msg.contains(&token.nonce),
3212 "error must not leak the nonce: {msg}"
3213 );
3214 }
3215
3216 /// attach → verify → heartbeat → detach all resolve the session /
3217 /// token record through fingerprint keys (mint/verify lifecycle
3218 /// regression guard for the issue #14 key migration).
3219 #[tokio::test]
3220 async fn attach_verify_heartbeat_detach_cycle_with_fp_keying() {
3221 let engine = Engine::new(EngineCfg::default());
3222 let token = engine
3223 .attach("op-1", Role::Operator, Duration::from_secs(60))
3224 .await
3225 .expect("attach");
3226 engine
3227 .verify_token(&token, Verb::ReadTaskState)
3228 .await
3229 .expect("verify consumes via fp key");
3230 engine
3231 .heartbeat(&token)
3232 .await
3233 .expect("heartbeat finds the session by fp");
3234 engine
3235 .detach(&token)
3236 .await
3237 .expect("detach finds the session by fp");
3238 }
3239}
3240
3241// ─── UT: `OperatorKind` "Runtime Global" tier — `Option` semantics ─────────
3242//
3243// Regression coverage for the "explicit Automate is indistinguishable from
3244// unspecified" defect: `OperatorSession.operator_kind` (and the
3245// `attach_with_ids` `kind` parameter it stores) is `Option<OperatorKind>`,
3246// so `Some(Automate)` is an explicit Runtime Global request that must
3247// outrank `bp_global`, while `None` must let `bp_global` decide. Exercises
3248// the real `resolve_operator_info` cascade path (not just
3249// `collapse_operator_kind` in isolation), attaching via `attach_with_ids`
3250// exactly as `TaskLaunchService::launch` does.
3251#[cfg(test)]
3252mod resolve_operator_info_runtime_global_tests {
3253 use super::*;
3254
3255 async fn attach_and_resolve(
3256 runtime_global: Option<OperatorKind>,
3257 bp_global: Option<OperatorKind>,
3258 ) -> OperatorInfo {
3259 let engine = Engine::new(EngineCfg::default());
3260 let token = engine
3261 .attach_with_ids(
3262 "ut-op",
3263 Role::Operator,
3264 Duration::from_secs(30),
3265 runtime_global,
3266 None,
3267 None,
3268 None,
3269 HashMap::new(),
3270 HashMap::new(),
3271 bp_global,
3272 )
3273 .await
3274 .expect("attach_with_ids ok");
3275 let session = engine
3276 .with_state("test.find_session", |s| {
3277 s.sessions
3278 .values()
3279 .find(|sess| sess.token_fp == token.fingerprint())
3280 .cloned()
3281 })
3282 .await
3283 .expect("with_state ok")
3284 .expect("session present after attach_with_ids");
3285 engine.resolve_operator_info(&session, "agent-x").await
3286 }
3287
3288 #[tokio::test]
3289 async fn explicit_some_automate_outranks_bp_global_main_ai() {
3290 // Runtime Global explicitly requests Automate; bp_global is MainAi.
3291 // The explicit `Some(Automate)` must win — this is exactly the case
3292 // the old `== OperatorKind::default()` convention got wrong (it
3293 // could not tell "explicitly Automate" from "unspecified" and would
3294 // have let `bp_global` (MainAi) take over instead).
3295 let info =
3296 attach_and_resolve(Some(OperatorKind::Automate), Some(OperatorKind::MainAi)).await;
3297 assert_eq!(
3298 info.kind,
3299 OperatorKind::Automate,
3300 "explicit Some(Automate) runtime_global must outrank bp_global MainAi"
3301 );
3302 }
3303
3304 #[tokio::test]
3305 async fn none_lets_bp_global_main_ai_win() {
3306 // Runtime Global left unspecified (`None`); bp_global is MainAi.
3307 // With nothing more specific set, `bp_global` must decide.
3308 let info = attach_and_resolve(None, Some(OperatorKind::MainAi)).await;
3309 assert_eq!(
3310 info.kind,
3311 OperatorKind::MainAi,
3312 "None runtime_global must let bp_global MainAi win"
3313 );
3314 }
3315}
3316
3317/// issue #13 run_id propagation: `dispatch_attempt_with`'s `run_id` param
3318/// must land in `Ctx.meta.runtime["run_id"]` (the same slot pattern as the
3319/// pre-existing `worker_handle`), or be omitted entirely when `None`. Same
3320/// `CtxProbe` shape as `middleware::worker_binding`'s test module — an
3321/// inner `SpawnerAdapter` that snapshots the `Ctx` it was called with and
3322/// fails the spawn (only the ctx snapshot matters here).
3323#[cfg(test)]
3324mod dispatch_attempt_with_run_id_tests {
3325 use super::*;
3326 use crate::worker::adapter::{SpawnError, SpawnerAdapter};
3327 use crate::worker::Worker;
3328 use std::sync::Mutex as StdMutex;
3329
3330 struct CtxProbe {
3331 seen: Arc<StdMutex<Option<Ctx>>>,
3332 }
3333
3334 #[async_trait::async_trait]
3335 impl SpawnerAdapter for CtxProbe {
3336 async fn spawn(
3337 &self,
3338 _engine: &Engine,
3339 ctx: &Ctx,
3340 _task_id: StepId,
3341 _attempt: u32,
3342 _token: CapToken,
3343 ) -> Result<Box<dyn Worker>, SpawnError> {
3344 *self.seen.lock().unwrap() = Some(ctx.clone());
3345 Err(SpawnError::Internal("probe stop".into()))
3346 }
3347 }
3348
3349 async fn dispatch_with_probe(run_id: Option<&RunId>) -> Ctx {
3350 let engine = Engine::new(EngineCfg::default());
3351 let token = engine
3352 .attach("ut-op", Role::Operator, Duration::from_secs(30))
3353 .await
3354 .expect("attach");
3355 let tid = engine
3356 .start_task(
3357 &token,
3358 TaskSpec {
3359 agent: "probe".into(),
3360 initial_directive: "hi".into(),
3361 step_ctx: None,
3362 check_policy: None,
3363 },
3364 )
3365 .await
3366 .expect("start_task");
3367 let seen: Arc<StdMutex<Option<Ctx>>> = Arc::new(StdMutex::new(None));
3368 let spawner: Arc<dyn SpawnerAdapter> = Arc::new(CtxProbe { seen: seen.clone() });
3369 // The probe always errors the spawn (`SpawnError::Internal`); we
3370 // only care about the `Ctx` snapshot it captured, so the dispatch
3371 // outcome itself (`Err`) is discarded.
3372 let _ = engine
3373 .dispatch_attempt_with(&token, &tid, &spawner, run_id)
3374 .await;
3375 let captured = seen.lock().unwrap().clone();
3376 captured.expect("inner ctx captured")
3377 }
3378
3379 #[tokio::test]
3380 async fn run_id_lands_in_ctx_meta_runtime_when_some() {
3381 let run_id = RunId::new();
3382 let observed = dispatch_with_probe(Some(&run_id)).await;
3383 assert_eq!(
3384 observed.meta.runtime.get("run_id").and_then(|v| v.as_str()),
3385 Some(run_id.as_str()),
3386 "ctx.meta.runtime[\"run_id\"] must carry the run_id passed to dispatch_attempt_with"
3387 );
3388 }
3389
3390 #[tokio::test]
3391 async fn run_id_key_absent_when_none() {
3392 let observed = dispatch_with_probe(None).await;
3393 assert!(
3394 !observed.meta.runtime.contains_key("run_id"),
3395 "no run_id key must be injected when dispatch_attempt_with is called with None"
3396 );
3397 }
3398}
3399
3400/// GH #21 Phase 2: `TaskSpec.step_ctx` must land in
3401/// `Ctx.meta.runtime[STEP_CTX_KEY]` — re-read from the spec on EVERY
3402/// attempt (the prep closure re-reads `task.spec.step_ctx` every call, not
3403/// caching it once at `start_task`), so a retry (attempt 2) carries it
3404/// too. Same `CtxProbe` shape as `dispatch_attempt_with_run_id_tests`.
3405#[cfg(test)]
3406mod dispatch_attempt_with_step_ctx_tests {
3407 use super::*;
3408 use crate::worker::adapter::{SpawnError, SpawnerAdapter};
3409 use crate::worker::Worker;
3410 use std::sync::Mutex as StdMutex;
3411
3412 struct CtxProbe {
3413 seen: Arc<StdMutex<Option<Ctx>>>,
3414 }
3415
3416 #[async_trait::async_trait]
3417 impl SpawnerAdapter for CtxProbe {
3418 async fn spawn(
3419 &self,
3420 _engine: &Engine,
3421 ctx: &Ctx,
3422 _task_id: StepId,
3423 _attempt: u32,
3424 _token: CapToken,
3425 ) -> Result<Box<dyn Worker>, SpawnError> {
3426 *self.seen.lock().unwrap() = Some(ctx.clone());
3427 Err(SpawnError::Internal("probe stop".into()))
3428 }
3429 }
3430
3431 #[tokio::test]
3432 async fn step_ctx_lands_in_ctx_meta_runtime_on_attempt_1_and_2() {
3433 let engine = Engine::new(EngineCfg::default());
3434 let token = engine
3435 .attach("ut-op", Role::Operator, Duration::from_secs(30))
3436 .await
3437 .expect("attach");
3438 let tid = engine
3439 .start_task(
3440 &token,
3441 TaskSpec {
3442 agent: "probe".into(),
3443 initial_directive: "hi".into(),
3444 step_ctx: Some(serde_json::json!({ "work_dir": "/step" })),
3445 check_policy: None,
3446 },
3447 )
3448 .await
3449 .expect("start_task");
3450 let seen: Arc<StdMutex<Option<Ctx>>> = Arc::new(StdMutex::new(None));
3451 let spawner: Arc<dyn SpawnerAdapter> = Arc::new(CtxProbe { seen: seen.clone() });
3452
3453 // The probe always errors the spawn; only the ctx snapshot matters.
3454 let _ = engine
3455 .dispatch_attempt_with(&token, &tid, &spawner, None)
3456 .await;
3457 let first = seen
3458 .lock()
3459 .unwrap()
3460 .clone()
3461 .expect("attempt 1 ctx captured");
3462 assert_eq!(
3463 first.meta.runtime.get(STEP_CTX_KEY),
3464 Some(&serde_json::json!({ "work_dir": "/step" })),
3465 "attempt 1 must carry TaskSpec.step_ctx in ctx.meta.runtime[STEP_CTX_KEY]"
3466 );
3467
3468 let _ = engine
3469 .dispatch_attempt_with(&token, &tid, &spawner, None)
3470 .await;
3471 let second = seen
3472 .lock()
3473 .unwrap()
3474 .clone()
3475 .expect("attempt 2 ctx captured");
3476 assert_eq!(
3477 second.meta.runtime.get(STEP_CTX_KEY),
3478 Some(&serde_json::json!({ "work_dir": "/step" })),
3479 "attempt 2 (retry) must ALSO carry TaskSpec.step_ctx — prep re-reads the spec every attempt"
3480 );
3481 }
3482
3483 #[tokio::test]
3484 async fn step_ctx_key_absent_when_none() {
3485 let engine = Engine::new(EngineCfg::default());
3486 let token = engine
3487 .attach("ut-op", Role::Operator, Duration::from_secs(30))
3488 .await
3489 .expect("attach");
3490 let tid = engine
3491 .start_task(
3492 &token,
3493 TaskSpec {
3494 agent: "probe".into(),
3495 initial_directive: "hi".into(),
3496 step_ctx: None,
3497 check_policy: None,
3498 },
3499 )
3500 .await
3501 .expect("start_task");
3502 let seen: Arc<StdMutex<Option<Ctx>>> = Arc::new(StdMutex::new(None));
3503 let spawner: Arc<dyn SpawnerAdapter> = Arc::new(CtxProbe { seen: seen.clone() });
3504 let _ = engine
3505 .dispatch_attempt_with(&token, &tid, &spawner, None)
3506 .await;
3507 let observed = seen.lock().unwrap().clone().expect("ctx captured");
3508 assert!(
3509 !observed.meta.runtime.contains_key(STEP_CTX_KEY),
3510 "no step_ctx key must be injected when TaskSpec.step_ctx is None"
3511 );
3512 }
3513}
3514
3515// ─── issue #18: `TaskSpec.initial_directive` `Value` pass-through ──────────
3516#[cfg(test)]
3517mod initial_directive_value_passthrough_tests {
3518 use super::*;
3519
3520 async fn seeded_engine(initial_directive: Value) -> (Engine, CapToken, StepId) {
3521 let engine = Engine::new(EngineCfg::default());
3522 let op_token = engine
3523 .attach("ut-op", Role::Operator, Duration::from_secs(30))
3524 .await
3525 .expect("attach");
3526 let task_id = engine
3527 .start_task(
3528 &op_token,
3529 TaskSpec {
3530 agent: "planner".to_string(),
3531 initial_directive,
3532 step_ctx: None,
3533 check_policy: None,
3534 },
3535 )
3536 .await
3537 .expect("start_task");
3538 (engine, op_token, task_id)
3539 }
3540
3541 /// Mint + register a `Role::Worker` token the same way
3542 /// `dispatch_attempt_with` does — `fetch_prompt` is worker-verb-gated.
3543 async fn mint_worker_token(engine: &Engine, task_id: &StepId) -> CapToken {
3544 let worker_token = engine.signer().session(
3545 format!("worker-of-{task_id}"),
3546 Role::Worker,
3547 vec!["*".into()],
3548 Duration::from_secs(600),
3549 );
3550 let fp = worker_token.fingerprint();
3551 let record = CapTokenRecord::from_worker_token(worker_token.clone(), task_id.clone());
3552 engine
3553 .with_state("test.mint_worker", move |s| {
3554 s.tokens.insert(fp, record);
3555 })
3556 .await
3557 .expect("mint worker token");
3558 worker_token
3559 }
3560
3561 /// `EngineDispatcher::dispatch` no longer stringifies the evaluated
3562 /// `Step.in` value before seeding `TaskSpec.initial_directive` — an
3563 /// Object seed must round-trip through `start_task` /
3564 /// `read_task_state` byte-for-byte as the same `Value::Object`, not a
3565 /// JSON-stringified `Value::String`.
3566 #[tokio::test]
3567 async fn object_seed_passes_through_task_spec_unchanged() {
3568 let seed = serde_json::json!({"key": "value"});
3569 let (engine, token, task_id) = seeded_engine(seed.clone()).await;
3570 let state = engine
3571 .read_task_state(&token, &task_id)
3572 .await
3573 .expect("read_task_state");
3574 assert_eq!(
3575 state.spec.initial_directive, seed,
3576 "TaskSpec.initial_directive must equal the raw Object seed, not a stringified copy"
3577 );
3578 }
3579
3580 /// `Engine::fetch_prompt` returns the `Value` end-to-end (issue #18):
3581 /// an Object seed stays a `Value::Object` and is not stringified in
3582 /// the engine layer. The Worker HTTP boundary
3583 /// (`fetch_worker_payload*`) is what performs the render down to a
3584 /// JSON literal `String` for `WorkerPayload.prompt`.
3585 #[tokio::test]
3586 async fn object_seed_passes_through_fetch_prompt_as_value() {
3587 let seed = serde_json::json!({"key": "value"});
3588 let (engine, _token, task_id) = seeded_engine(seed.clone()).await;
3589 let worker_token = mint_worker_token(&engine, &task_id).await;
3590 let prompt = engine
3591 .fetch_prompt(&worker_token, &task_id)
3592 .await
3593 .expect("fetch_prompt");
3594 assert_eq!(
3595 prompt, seed,
3596 "fetch_prompt must return the raw Object Value, not a stringified copy"
3597 );
3598 }
3599
3600 /// The Worker HTTP boundary is the render point: `fetch_worker_payload*`
3601 /// coerces the stored `Value` down to `WorkerPayload.prompt: String`
3602 /// (JSON-literal shape for non-strings). Verifies the boundary render
3603 /// stays intact for an Object seed.
3604 #[tokio::test]
3605 async fn object_seed_renders_as_json_literal_at_worker_payload_boundary() {
3606 let seed = serde_json::json!({"key": "value"});
3607 let (engine, _token, task_id) = seeded_engine(seed).await;
3608 let worker_token = mint_worker_token(&engine, &task_id).await;
3609 let payload = engine
3610 .fetch_worker_payload(&worker_token, &task_id)
3611 .await
3612 .expect("fetch_worker_payload");
3613 assert_eq!(
3614 payload.prompt, r#"{"key":"value"}"#,
3615 "WorkerPayload.prompt must be the JSON literal String render of the Value seed"
3616 );
3617 }
3618
3619 /// A `String` seed is unaffected — still passes through verbatim, both
3620 /// as the `TaskSpec.initial_directive` `Value` and as the Worker
3621 /// `fetch_prompt` return (issue #18 Invariant 2).
3622 #[tokio::test]
3623 async fn string_seed_passes_through_unchanged() {
3624 let (engine, token, task_id) = seeded_engine(serde_json::json!("do the thing")).await;
3625 let state = engine
3626 .read_task_state(&token, &task_id)
3627 .await
3628 .expect("read_task_state");
3629 assert_eq!(
3630 state.spec.initial_directive,
3631 serde_json::json!("do the thing")
3632 );
3633 let worker_token = mint_worker_token(&engine, &task_id).await;
3634 let prompt = engine
3635 .fetch_prompt(&worker_token, &task_id)
3636 .await
3637 .expect("fetch_prompt");
3638 assert_eq!(prompt, serde_json::json!("do the thing"));
3639 }
3640}
3641
3642/// GH #31: `fetch_worker_payload{,_trusted}`'s size-threshold branch
3643/// between inline (`WorkerPayload.system`) and by-reference
3644/// (`WorkerPayload.system_ref`) delivery, plus the `bake_worker_system_prompt`
3645/// `agent_render_sizes` bookkeeping that feeds `agent_last_rendered_size`.
3646#[cfg(test)]
3647mod system_ref_threshold_tests {
3648 use super::*;
3649
3650 async fn seeded_engine_with_cfg(cfg: EngineCfg) -> (Engine, CapToken, StepId) {
3651 let engine = Engine::new(cfg);
3652 let op_token = engine
3653 .attach("ut-op", Role::Operator, Duration::from_secs(30))
3654 .await
3655 .expect("attach");
3656 let task_id = engine
3657 .start_task(
3658 &op_token,
3659 TaskSpec {
3660 agent: "planner".to_string(),
3661 initial_directive: serde_json::json!("do the thing"),
3662 step_ctx: None,
3663 check_policy: None,
3664 },
3665 )
3666 .await
3667 .expect("start_task");
3668 (engine, op_token, task_id)
3669 }
3670
3671 /// Same worker-token-minting fixture as
3672 /// `initial_directive_value_passthrough_tests::mint_worker_token`
3673 /// (kept local to this module — the two `mod`s do not share private
3674 /// helpers across `cfg(test)` boundaries).
3675 async fn mint_worker_token(engine: &Engine, task_id: &StepId) -> CapToken {
3676 let worker_token = engine.signer().session(
3677 format!("worker-of-{task_id}"),
3678 Role::Worker,
3679 vec!["*".into()],
3680 Duration::from_secs(600),
3681 );
3682 let fp = worker_token.fingerprint();
3683 let record = CapTokenRecord::from_worker_token(worker_token.clone(), task_id.clone());
3684 engine
3685 .with_state("test.mint_worker", move |s| {
3686 s.tokens.insert(fp, record);
3687 })
3688 .await
3689 .expect("mint worker token");
3690 worker_token
3691 }
3692
3693 /// Under-threshold: `system` stays inline, `system_ref` stays `None`.
3694 #[tokio::test]
3695 async fn under_threshold_stays_inline() {
3696 let (engine, _op_token, task_id) = seeded_engine_with_cfg(EngineCfg::default()).await;
3697 let worker_token = mint_worker_token(&engine, &task_id).await;
3698 let rendered = "a short system prompt".to_string();
3699 engine
3700 .bake_worker_system_prompt(&task_id, 1, Some(rendered.clone()))
3701 .await
3702 .expect("bake");
3703 let payload = engine
3704 .fetch_worker_payload(&worker_token, &task_id)
3705 .await
3706 .expect("fetch_worker_payload");
3707 assert_eq!(payload.system, Some(rendered));
3708 assert!(payload.system_ref.is_none());
3709 }
3710
3711 /// Over-threshold: `system` is cleared and `system_ref` is populated
3712 /// with a `sha256` matching the known input string. Exercises
3713 /// `fetch_worker_payload_trusted` (the `_trusted` sibling must be
3714 /// behaviorally identical to `fetch_worker_payload`).
3715 #[tokio::test]
3716 async fn over_threshold_switches_to_system_ref_with_matching_sha256() {
3717 let mut cfg = EngineCfg::default();
3718 cfg.system_ref.threshold_bytes = 16;
3719 cfg.system_ref.mode = crate::types::SystemRefMode::File;
3720 cfg.system_ref.store_dir =
3721 std::env::temp_dir().join(format!("mse-system-ref-test-{}", crate::types::now_unix()));
3722 let (engine, _op_token, task_id) = seeded_engine_with_cfg(cfg).await;
3723 let rendered =
3724 "this system prompt is deliberately longer than the 16 byte threshold".to_string();
3725 engine
3726 .bake_worker_system_prompt(&task_id, 1, Some(rendered.clone()))
3727 .await
3728 .expect("bake");
3729 let payload = engine
3730 .fetch_worker_payload_trusted(&task_id)
3731 .await
3732 .expect("fetch_worker_payload_trusted");
3733 assert!(
3734 payload.system.is_none(),
3735 "over-threshold response must not also inline `system`"
3736 );
3737 let system_ref = payload
3738 .system_ref
3739 .expect("over-threshold response must populate system_ref");
3740 assert_eq!(system_ref.size_bytes, rendered.len() as u64);
3741 assert_eq!(system_ref.mode, crate::types::SystemRefMode::File);
3742 use sha2::Digest;
3743 let expected_sha256 = hex::encode(sha2::Sha256::digest(rendered.as_bytes()));
3744 assert_eq!(system_ref.sha256, expected_sha256);
3745 assert!(system_ref.uri.starts_with("file://"));
3746 let written = tokio::fs::read_to_string(system_ref.uri.trim_start_matches("file://"))
3747 .await
3748 .expect("File mode must have written the referenced path");
3749 assert_eq!(written, rendered);
3750 }
3751
3752 /// `Http` mode never writes a file — `system_ref.uri` is the bare path
3753 /// the engine can construct on its own, scheme/host-free.
3754 #[tokio::test]
3755 async fn over_threshold_http_mode_constructs_path_only_uri() {
3756 let mut cfg = EngineCfg::default();
3757 cfg.system_ref.threshold_bytes = 16;
3758 cfg.system_ref.mode = crate::types::SystemRefMode::Http;
3759 let (engine, _op_token, task_id) = seeded_engine_with_cfg(cfg).await;
3760 let worker_token = mint_worker_token(&engine, &task_id).await;
3761 let rendered =
3762 "this system prompt is deliberately longer than the 16 byte threshold".to_string();
3763 engine
3764 .bake_worker_system_prompt(&task_id, 1, Some(rendered))
3765 .await
3766 .expect("bake");
3767 let payload = engine
3768 .fetch_worker_payload(&worker_token, &task_id)
3769 .await
3770 .expect("fetch_worker_payload");
3771 let system_ref = payload.system_ref.expect("system_ref must be populated");
3772 assert_eq!(system_ref.mode, crate::types::SystemRefMode::Http);
3773 assert_eq!(
3774 system_ref.uri,
3775 format!("/v1/worker/prompt/system?task_id={task_id}&attempt=1")
3776 );
3777 }
3778
3779 /// `bake_worker_system_prompt` records the render size keyed by agent
3780 /// name (last-write-wins), readable via `agent_last_rendered_size`.
3781 #[tokio::test]
3782 async fn bake_records_agent_render_size_last_write_wins() {
3783 let (engine, _op_token, task_id) = seeded_engine_with_cfg(EngineCfg::default()).await;
3784 assert_eq!(engine.agent_last_rendered_size("planner").await, None);
3785 engine
3786 .bake_worker_system_prompt(&task_id, 1, Some("a".repeat(10)))
3787 .await
3788 .expect("bake 1");
3789 assert_eq!(engine.agent_last_rendered_size("planner").await, Some(10));
3790 engine
3791 .bake_worker_system_prompt(&task_id, 2, Some("b".repeat(20)))
3792 .await
3793 .expect("bake 2");
3794 assert_eq!(
3795 engine.agent_last_rendered_size("planner").await,
3796 Some(20),
3797 "most-recently-observed size wins, not the largest"
3798 );
3799 }
3800}
3801
3802/// subtask-4 / ST2 rework: `submit_output` / `submit_worker_result_trusted`'s
3803/// submit-time projection sink (`Engine::materialize_final_submission`) —
3804/// the Data-plane `OutputStore` dual-write plus the
3805/// `FileProjectionAdapter`-backed file materialize, both fail-open. See
3806/// the subtask-4 Tests this module covers inline on each test.
3807#[cfg(test)]
3808mod submit_time_projection_sink_tests {
3809 use super::*;
3810 use crate::core::agent_context::AgentContextView;
3811 use crate::store::output::{ContentRef, InMemoryOutputStore, OutputEvent};
3812
3813 /// Starts a task under `agent`, returning `(engine, op_token, task_id,
3814 /// worker_token)` — same helper shape as the sibling test modules
3815 /// above (`initial_directive_value_passthrough_tests::seeded_engine` /
3816 /// `mint_worker_token`), duplicated locally per this file's
3817 /// established per-module convention.
3818 async fn seeded_task(agent: &str) -> (Engine, CapToken, StepId, CapToken) {
3819 let engine = Engine::new(EngineCfg::default());
3820 let op_token = engine
3821 .attach("ut-op", Role::Operator, Duration::from_secs(30))
3822 .await
3823 .expect("attach");
3824 let task_id = engine
3825 .start_task(
3826 &op_token,
3827 TaskSpec {
3828 agent: agent.to_string(),
3829 initial_directive: Value::String("go".into()),
3830 step_ctx: None,
3831 check_policy: None,
3832 },
3833 )
3834 .await
3835 .expect("start_task");
3836 let worker_token = engine.signer().session(
3837 format!("worker-of-{task_id}"),
3838 Role::Worker,
3839 vec!["*".into()],
3840 Duration::from_secs(600),
3841 );
3842 let fp = worker_token.fingerprint();
3843 let record = CapTokenRecord::from_worker_token(worker_token.clone(), task_id.clone());
3844 engine
3845 .with_state("test.mint_worker", move |s| {
3846 s.tokens.insert(fp, record);
3847 })
3848 .await
3849 .expect("mint worker token");
3850 (engine, op_token, task_id, worker_token)
3851 }
3852
3853 /// Sibling of [`seeded_task`] that lets a caller pin the engine's
3854 /// `EngineCfg.check_policy` before the engine is constructed — used
3855 /// by the `check_policy_*` regression tests below to exercise the
3856 /// three [`crate::core::config::CheckPolicy`] modes without touching
3857 /// the shared `seeded_task` helper (which every unrelated sink test
3858 /// depends on).
3859 async fn seeded_task_with_policy(
3860 agent: &str,
3861 policy: crate::core::config::CheckPolicy,
3862 ) -> (Engine, CapToken, StepId, CapToken) {
3863 let cfg = EngineCfg {
3864 check_policy: policy,
3865 ..EngineCfg::default()
3866 };
3867 let engine = Engine::new(cfg);
3868 let op_token = engine
3869 .attach("ut-op", Role::Operator, Duration::from_secs(30))
3870 .await
3871 .expect("attach");
3872 let task_id = engine
3873 .start_task(
3874 &op_token,
3875 TaskSpec {
3876 agent: agent.to_string(),
3877 initial_directive: Value::String("go".into()),
3878 step_ctx: None,
3879 check_policy: None,
3880 },
3881 )
3882 .await
3883 .expect("start_task");
3884 let worker_token = engine.signer().session(
3885 format!("worker-of-{task_id}"),
3886 Role::Worker,
3887 vec!["*".into()],
3888 Duration::from_secs(600),
3889 );
3890 let fp = worker_token.fingerprint();
3891 let record = CapTokenRecord::from_worker_token(worker_token.clone(), task_id.clone());
3892 engine
3893 .with_state("test.mint_worker", move |s| {
3894 s.tokens.insert(fp, record);
3895 })
3896 .await
3897 .expect("mint worker token");
3898 (engine, op_token, task_id, worker_token)
3899 }
3900
3901 /// Seeds `EngineState.agent_ctx[(task_id, attempt)].view` directly —
3902 /// the same snapshot `AgentContextMiddleware` writes at spawn time
3903 /// (see its module doc), stood up here without the full spawner
3904 /// stack so these tests can exercise `submit_output` in isolation.
3905 async fn seed_agent_context(engine: &Engine, task_id: &StepId, attempt: u32, work_dir: &str) {
3906 let task_id = task_id.clone();
3907 let work_dir = work_dir.to_string();
3908 engine
3909 .with_state("test.seed_agent_context", move |s| {
3910 s.agent_ctx.insert(
3911 (task_id, attempt),
3912 crate::core::state::AgentCtxEntry {
3913 view: AgentContextView {
3914 work_dir: Some(work_dir),
3915 ..Default::default()
3916 },
3917 policy: Default::default(),
3918 },
3919 );
3920 })
3921 .await
3922 .expect("seed agent_ctx");
3923 }
3924
3925 /// GH #27 (follow-up to #23): seeds `EngineState.agent_ctx` with an
3926 /// arbitrary `work_dir` / `project_root` pair (either may be `None`),
3927 /// unlike [`seed_agent_context`] (which only ever sets `work_dir`) —
3928 /// lets these tests exercise `ProjectionPlacement::resolve_root`'s
3929 /// fallback in both directions.
3930 async fn seed_agent_context_roots(
3931 engine: &Engine,
3932 task_id: &StepId,
3933 attempt: u32,
3934 work_dir: Option<&str>,
3935 project_root: Option<&str>,
3936 ) {
3937 let task_id = task_id.clone();
3938 let work_dir = work_dir.map(str::to_string);
3939 let project_root = project_root.map(str::to_string);
3940 engine
3941 .with_state("test.seed_agent_context_roots", move |s| {
3942 s.agent_ctx.insert(
3943 (task_id, attempt),
3944 crate::core::state::AgentCtxEntry {
3945 view: AgentContextView {
3946 work_dir,
3947 project_root,
3948 ..Default::default()
3949 },
3950 policy: Default::default(),
3951 },
3952 );
3953 })
3954 .await
3955 .expect("seed agent_ctx");
3956 }
3957
3958 /// GH #27 (follow-up to #23): seeds `EngineState.projection_placements`
3959 /// directly — the same snapshot `EngineDispatcher::dispatch` stashes
3960 /// at dispatch time (mirroring [`seed_step_naming`]'s contract) — so
3961 /// these tests can exercise a declared `ProjectionPlacement` without
3962 /// driving a real `Compiler::compile`.
3963 async fn seed_projection_placement(
3964 engine: &Engine,
3965 task_id: &StepId,
3966 placement: crate::core::projection_placement::ProjectionPlacement,
3967 ) {
3968 let task_id = task_id.clone();
3969 let placement = Arc::new(placement);
3970 engine
3971 .with_state("test.seed_projection_placement", move |s| {
3972 s.projection_placements.insert(task_id, placement);
3973 })
3974 .await
3975 .expect("seed projection_placements");
3976 }
3977
3978 /// GH #23 subtask-2: builds a fixture
3979 /// [`crate::core::step_naming::StepNaming`] table declaring `producer`
3980 /// → `canonical` (`AgentMeta.projection_name`), then seeds it into
3981 /// `EngineState.step_namings` for `task_id` — the same snapshot
3982 /// `EngineDispatcher::dispatch` stashes at dispatch time
3983 /// (`blueprint.rs`'s "construct once, read many" contract), stood up
3984 /// here without the full Blueprint-compile path so these tests can
3985 /// exercise the canonical-sink resolution in isolation.
3986 async fn seed_step_naming(engine: &Engine, task_id: &StepId, producer: &str, canonical: &str) {
3987 use crate::blueprint::{
3988 current_schema_version, AgentDef, AgentKind, AgentMeta, Blueprint, BlueprintMetadata,
3989 CompilerHints, CompilerStrategy,
3990 };
3991 use crate::core::step_naming::StepNaming;
3992 use mlua_flow_ir::{Expr, Node};
3993
3994 let flow = Node::Step {
3995 ref_: producer.to_string(),
3996 in_: Expr::Path {
3997 at: "$.in".parse().expect("literal test path: $.in"),
3998 },
3999 out: Expr::Path {
4000 at: format!("$.{producer}_out")
4001 .parse()
4002 .expect("literal test path"),
4003 },
4004 };
4005 let bp = Blueprint {
4006 schema_version: current_schema_version(),
4007 id: "sink-canonical-ut".into(),
4008 flow,
4009 agents: vec![AgentDef {
4010 name: producer.to_string(),
4011 kind: AgentKind::RustFn,
4012 spec: serde_json::json!({ "fn_id": producer }),
4013 profile: None,
4014 meta: Some(AgentMeta {
4015 projection_name: Some(canonical.to_string()),
4016 ..Default::default()
4017 }),
4018 runner: None,
4019 runner_ref: None,
4020 verdict: None,
4021 }],
4022 operators: vec![],
4023 metas: vec![],
4024 hints: CompilerHints::default(),
4025 strategy: CompilerStrategy::default(),
4026 metadata: BlueprintMetadata::default(),
4027 spawner_hints: Default::default(),
4028 default_agent_kind: AgentKind::Operator,
4029 default_operator_kind: None,
4030 default_init_ctx: None,
4031 default_agent_ctx: None,
4032 default_context_policy: None,
4033 projection_placement: None,
4034 audits: vec![],
4035 degradation_policy: None,
4036 runners: vec![],
4037 default_runner: None,
4038 check_policy: None,
4039 blueprint_ref_includes: Vec::new(),
4040 };
4041 let (naming, warnings) = StepNaming::from_blueprint(&bp).expect("no collision");
4042 assert!(warnings.is_empty(), "single-step fixture has no collisions");
4043 let naming = Arc::new(naming);
4044 let task_id = task_id.clone();
4045 engine
4046 .with_state("test.seed_step_naming", move |s| {
4047 s.step_namings.insert(task_id, naming);
4048 })
4049 .await
4050 .expect("seed step_namings");
4051 }
4052
4053 fn final_event(value: Value, ok: bool) -> crate::worker::output::OutputEvent {
4054 crate::worker::output::OutputEvent::Final {
4055 content: crate::worker::output::ContentRef::Inline { value },
4056 ok,
4057 }
4058 }
4059
4060 /// Subtask 4 Test #2: `submit_output`'s `Final` writes
4061 /// `<root>/workspace/tasks/<task_id>/ctx/<agent>.md`, content matching
4062 /// the submitted value.
4063 #[tokio::test]
4064 async fn submit_output_final_materializes_file_when_work_dir_resolved() {
4065 let dir = tempfile::TempDir::new().unwrap();
4066 let (engine, _op, task_id, worker_token) = seeded_task("planner").await;
4067 seed_agent_context(&engine, &task_id, 1, &dir.path().to_string_lossy()).await;
4068
4069 engine
4070 .submit_output(
4071 &worker_token,
4072 &task_id,
4073 1,
4074 final_event(serde_json::json!({"plan": "do it"}), true),
4075 )
4076 .await
4077 .expect("submit_output");
4078
4079 let expected_file = dir
4080 .path()
4081 .join("workspace/tasks")
4082 .join(task_id.as_str())
4083 .join("ctx/planner.md");
4084 assert!(
4085 expected_file.exists(),
4086 "materialized submission file missing at {expected_file:?}"
4087 );
4088 let body = std::fs::read_to_string(expected_file).unwrap();
4089 assert!(body.contains(r#""plan": "do it""#), "body: {body}");
4090 }
4091
4092 /// Subtask 4 Test #3: `work_dir` unresolved (no `agent_ctx`
4093 /// snapshot for this `(task_id, attempt)`) — submit still succeeds,
4094 /// fail-open, no file.
4095 #[tokio::test]
4096 async fn submit_output_final_skips_file_when_root_unresolved() {
4097 let (engine, _op, task_id, worker_token) = seeded_task("planner").await;
4098 // No seed_agent_context call — root is unresolved.
4099
4100 let result = engine
4101 .submit_output(
4102 &worker_token,
4103 &task_id,
4104 1,
4105 final_event(serde_json::json!("hi"), true),
4106 )
4107 .await;
4108 assert!(
4109 result.is_ok(),
4110 "submit must succeed even with no resolvable root (fail-open, Invariant 1)"
4111 );
4112 }
4113
4114 /// Regression for the check_policy cascade: the default
4115 /// [`crate::core::config::CheckPolicy::Warn`] preserves the
4116 /// pre-`CheckPolicy` fail-open semantics — a submit whose root is
4117 /// unresolved still succeeds. Byte-compat with
4118 /// `submit_output_final_skips_file_when_root_unresolved`; this test
4119 /// pins the mode explicitly so a future default change to
4120 /// `Strict` (silent breakage) is caught here.
4121 #[tokio::test]
4122 async fn submit_output_final_check_policy_warn_preserves_fail_open() {
4123 let (engine, _op, task_id, worker_token) =
4124 seeded_task_with_policy("planner", crate::core::config::CheckPolicy::Warn).await;
4125
4126 let result = engine
4127 .submit_output(
4128 &worker_token,
4129 &task_id,
4130 1,
4131 final_event(serde_json::json!("hi"), true),
4132 )
4133 .await;
4134 assert!(
4135 result.is_ok(),
4136 "Warn mode preserves fail-open: submit must succeed when root unresolved"
4137 );
4138 }
4139
4140 /// Regression for the check_policy cascade:
4141 /// [`crate::core::config::CheckPolicy::Strict`] surfaces the "no
4142 /// work_dir/project_root resolved" fail-open condition as an
4143 /// [`EngineError::CheckPolicyStrict`], letting a caller who has
4144 /// opted in fail fast instead of proceeding with a partially-
4145 /// realized submission. The error's `context` identifies the call
4146 /// site (`"file materialize"`), and `message` preserves the
4147 /// pre-`CheckPolicy` warn literal verbatim (log-parse compat).
4148 #[tokio::test]
4149 async fn submit_output_final_check_policy_strict_surfaces_error_when_root_unresolved() {
4150 let (engine, _op, task_id, worker_token) =
4151 seeded_task_with_policy("planner", crate::core::config::CheckPolicy::Strict).await;
4152
4153 let err = engine
4154 .submit_output(
4155 &worker_token,
4156 &task_id,
4157 1,
4158 final_event(serde_json::json!("hi"), true),
4159 )
4160 .await
4161 .expect_err("Strict mode must return an error when root unresolved");
4162 match err {
4163 EngineError::CheckPolicyStrict { context, message } => {
4164 assert!(
4165 context.contains("file materialize"),
4166 "context must identify the call site: {context}"
4167 );
4168 assert!(
4169 message.contains("no work_dir/project_root resolved"),
4170 "message must preserve the warn-log literal for log-parse compat: {message}"
4171 );
4172 }
4173 other => panic!(
4174 "expected EngineError::CheckPolicyStrict, got a different variant: {other:?}"
4175 ),
4176 }
4177 }
4178
4179 /// Regression for the check_policy cascade:
4180 /// [`crate::core::config::CheckPolicy::Silent`] returns `Ok(())` (
4181 /// like `Warn`) without surfacing an error. The log-suppression side
4182 /// of `Silent` (no `tracing::warn!`) is enforced at the call site
4183 /// via the `if !matches!(policy, Silent) { warn!(...) }` guard —
4184 /// verifying tracing output shape here would couple the test to a
4185 /// subscriber setup, so the assertion is limited to the error-
4186 /// return semantics (matches the helper unit tests in
4187 /// `check_policy_helper_tests`).
4188 #[tokio::test]
4189 async fn submit_output_final_check_policy_silent_returns_ok_when_root_unresolved() {
4190 let (engine, _op, task_id, worker_token) =
4191 seeded_task_with_policy("planner", crate::core::config::CheckPolicy::Silent).await;
4192
4193 let result = engine
4194 .submit_output(
4195 &worker_token,
4196 &task_id,
4197 1,
4198 final_event(serde_json::json!("hi"), true),
4199 )
4200 .await;
4201 assert!(
4202 result.is_ok(),
4203 "Silent mode returns Ok(()) at the error surface: submit must succeed"
4204 );
4205 }
4206
4207 /// Subtask 4 Test #4 (file half): re-submitting under the same
4208 /// `(task_id, agent)` overwrites the materialized file with the
4209 /// latest value.
4210 #[tokio::test]
4211 async fn resubmit_overwrites_materialized_file_with_latest() {
4212 let dir = tempfile::TempDir::new().unwrap();
4213 let (engine, _op, task_id, worker_token) = seeded_task("planner").await;
4214 seed_agent_context(&engine, &task_id, 1, &dir.path().to_string_lossy()).await;
4215
4216 engine
4217 .submit_output(
4218 &worker_token,
4219 &task_id,
4220 1,
4221 final_event(serde_json::json!("first"), true),
4222 )
4223 .await
4224 .expect("first submit");
4225 engine
4226 .submit_output(
4227 &worker_token,
4228 &task_id,
4229 1,
4230 final_event(serde_json::json!("second"), true),
4231 )
4232 .await
4233 .expect("second submit");
4234
4235 let expected_file = dir
4236 .path()
4237 .join("workspace/tasks")
4238 .join(task_id.as_str())
4239 .join("ctx/planner.md");
4240 let body = std::fs::read_to_string(expected_file).unwrap();
4241 assert!(body.contains("second"), "body must reflect latest: {body}");
4242 assert!(
4243 !body.contains("first"),
4244 "body must not carry the stale value: {body}"
4245 );
4246 }
4247
4248 /// GH #27 (follow-up to #23): the byte-compat default
4249 /// `ProjectionPlacement` (`root_preference = WorkDir`) falls back to
4250 /// `project_root` when `work_dir` is absent — the same fallback
4251 /// [`crate::core::projection_placement::ProjectionPlacement::resolve_root`]
4252 /// now performs for every one of the "3 path" call sites, this one
4253 /// exercised at the submit-sink layer.
4254 #[tokio::test]
4255 async fn submit_output_final_falls_back_to_project_root_when_work_dir_absent() {
4256 let dir = tempfile::TempDir::new().unwrap();
4257 let (engine, _op, task_id, worker_token) = seeded_task("planner").await;
4258 seed_agent_context_roots(
4259 &engine,
4260 &task_id,
4261 1,
4262 None,
4263 Some(&dir.path().to_string_lossy()),
4264 )
4265 .await;
4266
4267 engine
4268 .submit_output(
4269 &worker_token,
4270 &task_id,
4271 1,
4272 final_event(serde_json::json!({"plan": "via project_root"}), true),
4273 )
4274 .await
4275 .expect("submit_output");
4276
4277 let expected_file = dir
4278 .path()
4279 .join("workspace/tasks")
4280 .join(task_id.as_str())
4281 .join("ctx/planner.md");
4282 assert!(
4283 expected_file.exists(),
4284 "materialized submission file missing at {expected_file:?} \
4285 (work_dir absent must fall back to project_root)"
4286 );
4287 }
4288
4289 /// GH #27 (follow-up to #23): a declared `ProjectionPlacement`
4290 /// (`root_preference = ProjectRoot`, custom `dir_template`) changes
4291 /// BOTH which root is preferred (project_root wins even though
4292 /// work_dir is also present) AND the target directory layout — proof
4293 /// the submit sink consults the snapshotted resolver rather than a
4294 /// hardcoded layout.
4295 #[tokio::test]
4296 async fn submit_output_final_uses_declared_projection_placement() {
4297 let work_dir = tempfile::TempDir::new().unwrap();
4298 let project_root = tempfile::TempDir::new().unwrap();
4299 let (engine, _op, task_id, worker_token) = seeded_task("planner").await;
4300 seed_agent_context_roots(
4301 &engine,
4302 &task_id,
4303 1,
4304 Some(&work_dir.path().to_string_lossy()),
4305 Some(&project_root.path().to_string_lossy()),
4306 )
4307 .await;
4308 seed_projection_placement(
4309 &engine,
4310 &task_id,
4311 crate::core::projection_placement::ProjectionPlacement {
4312 root_preference: crate::core::projection_placement::RootPreference::ProjectRoot,
4313 dir_template: "custom/{task_id}/out".to_string(),
4314 },
4315 )
4316 .await;
4317
4318 engine
4319 .submit_output(
4320 &worker_token,
4321 &task_id,
4322 1,
4323 final_event(serde_json::json!({"plan": "via custom placement"}), true),
4324 )
4325 .await
4326 .expect("submit_output");
4327
4328 let expected_file = project_root
4329 .path()
4330 .join("custom")
4331 .join(task_id.as_str())
4332 .join("out/planner.md");
4333 assert!(
4334 expected_file.exists(),
4335 "materialized submission file missing at custom placement target {expected_file:?}"
4336 );
4337 let unexpected_file = work_dir
4338 .path()
4339 .join("workspace/tasks")
4340 .join(task_id.as_str())
4341 .join("ctx/planner.md");
4342 assert!(
4343 !unexpected_file.exists(),
4344 "declared root_preference=ProjectRoot must not fall back to work_dir: {unexpected_file:?}"
4345 );
4346 }
4347
4348 /// Subtask 4 Invariant 3 / crux requirement #3: when
4349 /// [`Engine::set_output_store`] wires a Data-plane [`crate::store::output::OutputStore`],
4350 /// `submit_output`'s `Final` dual-writes into it under
4351 /// `producer_agent = TaskState.spec.agent` — the store becomes
4352 /// queryable via `get_latest_by_name`, independent of whether a root
4353 /// resolved for the file half.
4354 #[tokio::test]
4355 async fn submit_output_final_dual_writes_into_configured_output_store() {
4356 let (engine, _op, task_id, worker_token) = seeded_task("reviewer").await;
4357 let data_store: Arc<dyn crate::store::output::OutputStore> =
4358 Arc::new(InMemoryOutputStore::new());
4359 engine.set_output_store(data_store.clone());
4360
4361 engine
4362 .submit_output(
4363 &worker_token,
4364 &task_id,
4365 1,
4366 final_event(serde_json::json!({"verdict": "pass"}), true),
4367 )
4368 .await
4369 .expect("submit_output");
4370
4371 let record = data_store
4372 .get_latest_by_name("reviewer")
4373 .await
4374 .expect("dual-written record");
4375 match record.event {
4376 OutputEvent::Final { content, ok } => {
4377 assert!(ok);
4378 match content {
4379 ContentRef::Inline { value } => {
4380 assert_eq!(value, serde_json::json!({"verdict": "pass"}));
4381 }
4382 other => panic!("expected Inline content, got {other:?}"),
4383 }
4384 }
4385 other => panic!("expected Final event, got {other:?}"),
4386 }
4387 }
4388
4389 /// GH #34 subtask-3 gap fix: an `Artifact` event submitted via
4390 /// `submit_output` dual-writes into a wired Data-plane `OutputStore`
4391 /// under its OWN `name`, verbatim — mirrors
4392 /// `submit_output_final_dual_writes_into_configured_output_store`
4393 /// above, but for the `Artifact` variant.
4394 #[tokio::test]
4395 async fn submit_output_artifact_dual_writes_into_configured_output_store() {
4396 let (engine, _op, task_id, worker_token) = seeded_task("echo").await;
4397 let data_store: Arc<dyn crate::store::output::OutputStore> =
4398 Arc::new(InMemoryOutputStore::new());
4399 engine.set_output_store(data_store.clone());
4400
4401 engine
4402 .submit_output(
4403 &worker_token,
4404 &task_id,
4405 1,
4406 OutputEvent::Artifact {
4407 name: "audit:echo".to_string(),
4408 content: ContentRef::Inline {
4409 value: serde_json::json!({"finding": "clean"}),
4410 },
4411 },
4412 )
4413 .await
4414 .expect("submit_output");
4415
4416 let record = data_store
4417 .get_latest_by_name("audit:echo")
4418 .await
4419 .expect("dual-written artifact record");
4420 match record.event {
4421 OutputEvent::Artifact { name, content } => {
4422 assert_eq!(name, "audit:echo");
4423 match content {
4424 ContentRef::Inline { value } => {
4425 assert_eq!(value, serde_json::json!({"finding": "clean"}));
4426 }
4427 other => panic!("expected Inline content, got {other:?}"),
4428 }
4429 }
4430 other => panic!("expected Artifact event, got {other:?}"),
4431 }
4432 // The `Artifact` dual-write must never collide with / overwrite
4433 // the producing step's own `Final` name — `submit_output` never
4434 // materialized a `Final` here, so `"echo"` must stay unresolved.
4435 assert!(
4436 data_store.get_latest_by_name("echo").await.is_err(),
4437 "artifact write must not fabricate a record under the raw producer_agent name"
4438 );
4439 }
4440
4441 /// Invariant 1 (fail-open) for `Artifact`, mirroring
4442 /// `submit_output_final_skips_file_when_root_unresolved`'s Final-side
4443 /// coverage: no `OutputStore` wired at all — submit still succeeds.
4444 #[tokio::test]
4445 async fn submit_output_artifact_is_fail_open_when_no_output_store_configured() {
4446 let (engine, _op, task_id, worker_token) = seeded_task("echo").await;
4447
4448 let result = engine
4449 .submit_output(
4450 &worker_token,
4451 &task_id,
4452 1,
4453 OutputEvent::Artifact {
4454 name: "audit:echo".to_string(),
4455 content: ContentRef::Inline {
4456 value: serde_json::json!("finding"),
4457 },
4458 },
4459 )
4460 .await;
4461 assert!(
4462 result.is_ok(),
4463 "submit must succeed even with no OutputStore wired (fail-open, Invariant 1)"
4464 );
4465 }
4466
4467 /// `submit_worker_result_trusted` (the `/v1/worker/submit` short-handle
4468 /// path) triggers the exact same sink as `submit_output` — parity
4469 /// across both worker-submit entry points.
4470 #[tokio::test]
4471 async fn submit_worker_result_trusted_also_triggers_projection_sink() {
4472 let dir = tempfile::TempDir::new().unwrap();
4473 let (engine, _op, task_id, _worker_token) = seeded_task("planner").await;
4474 seed_agent_context(&engine, &task_id, 1, &dir.path().to_string_lossy()).await;
4475 let data_store: Arc<dyn crate::store::output::OutputStore> =
4476 Arc::new(InMemoryOutputStore::new());
4477 engine.set_output_store(data_store.clone());
4478
4479 engine
4480 .submit_worker_result_trusted(&task_id, 1, serde_json::json!("trusted-value"), true)
4481 .await
4482 .expect("submit_worker_result_trusted");
4483
4484 let expected_file = dir
4485 .path()
4486 .join("workspace/tasks")
4487 .join(task_id.as_str())
4488 .join("ctx/planner.md");
4489 assert!(expected_file.exists());
4490 let record = data_store
4491 .get_latest_by_name("planner")
4492 .await
4493 .expect("dual-written record");
4494 assert!(matches!(record.event, OutputEvent::Final { ok: true, .. }));
4495 }
4496
4497 /// GH #23 subtask-2 (canonical sink): a declared `projection_name`
4498 /// (`AgentMeta.projection_name`, surfaced via `StepNaming`) redirects
4499 /// `submit_output`'s Final canonical sink — both the Data-plane
4500 /// dual-write name and the materialized file stem resolve to the
4501 /// canonical name, not the raw `producer_agent`.
4502 #[tokio::test]
4503 async fn submit_output_final_uses_canonical_name_when_step_naming_declares_one() {
4504 let dir = tempfile::TempDir::new().unwrap();
4505 let (engine, _op, task_id, worker_token) = seeded_task("reviewer").await;
4506 seed_agent_context(&engine, &task_id, 1, &dir.path().to_string_lossy()).await;
4507 seed_step_naming(&engine, &task_id, "reviewer", "verdict-final").await;
4508 let data_store: Arc<dyn crate::store::output::OutputStore> =
4509 Arc::new(InMemoryOutputStore::new());
4510 engine.set_output_store(data_store.clone());
4511
4512 engine
4513 .submit_output(
4514 &worker_token,
4515 &task_id,
4516 1,
4517 final_event(serde_json::json!({"verdict": "pass"}), true),
4518 )
4519 .await
4520 .expect("submit_output");
4521
4522 let record = data_store
4523 .get_latest_by_name("verdict-final")
4524 .await
4525 .expect("dual-written record under canonical name");
4526 assert!(matches!(record.event, OutputEvent::Final { ok: true, .. }));
4527 assert!(
4528 data_store.get_latest_by_name("reviewer").await.is_err(),
4529 "raw producer_agent name must not be written once canonical resolves"
4530 );
4531
4532 let expected_file = dir
4533 .path()
4534 .join("workspace/tasks")
4535 .join(task_id.as_str())
4536 .join("ctx/verdict-final.md");
4537 assert!(
4538 expected_file.exists(),
4539 "materialized file stem must be canonical at {expected_file:?}"
4540 );
4541 }
4542
4543 /// GH #23 subtask-2: no `StepNaming` table snapshotted for this
4544 /// `task_id` (the pre-GH-#23 / no-`with_step_naming` path) is a
4545 /// defensive fail-open — the canonical sink falls back to the raw
4546 /// `producer_agent`, byte-identical to
4547 /// `submit_output_final_dual_writes_into_configured_output_store`
4548 /// above (which never calls `seed_step_naming`).
4549 #[tokio::test]
4550 async fn submit_output_final_falls_back_to_producer_agent_when_no_step_naming_table() {
4551 let (engine, _op, task_id, worker_token) = seeded_task("reviewer").await;
4552 let data_store: Arc<dyn crate::store::output::OutputStore> =
4553 Arc::new(InMemoryOutputStore::new());
4554 engine.set_output_store(data_store.clone());
4555
4556 engine
4557 .submit_output(
4558 &worker_token,
4559 &task_id,
4560 1,
4561 final_event(serde_json::json!({"verdict": "pass"}), true),
4562 )
4563 .await
4564 .expect("submit_output");
4565
4566 let record = data_store
4567 .get_latest_by_name("reviewer")
4568 .await
4569 .expect("fail-open dual-write under raw producer_agent name");
4570 assert!(matches!(record.event, OutputEvent::Final { ok: true, .. }));
4571 }
4572
4573 /// GH #23 subtask-2 (Layer 2): `OutputStore::get_latest_by_name_in_run`
4574 /// resolves the value `submit_output` dual-wrote for this exact
4575 /// `(task_id, attempt)` run, independent of `get_latest_by_name`'s
4576 /// cross-Run race (two Runs sharing a producer name never bleed into
4577 /// each other through the Run-scoped accessor).
4578 #[tokio::test]
4579 async fn submit_output_final_is_resolvable_via_run_scoped_lookup() {
4580 let (engine, _op, task_id, worker_token) = seeded_task("reviewer").await;
4581 let data_store: Arc<dyn crate::store::output::OutputStore> =
4582 Arc::new(InMemoryOutputStore::new());
4583 engine.set_output_store(data_store.clone());
4584
4585 engine
4586 .submit_output(
4587 &worker_token,
4588 &task_id,
4589 1,
4590 final_event(serde_json::json!({"verdict": "pass"}), true),
4591 )
4592 .await
4593 .expect("submit_output");
4594
4595 let record = data_store
4596 .get_latest_by_name_in_run(task_id.as_str(), 1, "reviewer")
4597 .await
4598 .expect("run-scoped lookup resolves the dual-written record");
4599 assert!(matches!(record.event, OutputEvent::Final { ok: true, .. }));
4600
4601 // A different attempt of the same task must not resolve — the
4602 // Run-scoped lookup does not fall back across attempts.
4603 assert!(
4604 data_store
4605 .get_latest_by_name_in_run(task_id.as_str(), 2, "reviewer")
4606 .await
4607 .is_err(),
4608 "a different attempt must not resolve the same-named record"
4609 );
4610 }
4611
4612 // ─── staged part file materialize ───
4613
4614 /// Staging a part with a resolved `work_dir` writes
4615 /// `<work_dir>/workspace/tasks/<task_id>/ctx/<name>` with the part's
4616 /// content RAW (no front matter / fenced wrapper).
4617 #[tokio::test]
4618 async fn stage_artifact_materializes_part_file_when_work_dir_resolved() {
4619 let dir = tempfile::TempDir::new().unwrap();
4620 let (engine, _op, task_id, _worker_token) = seeded_task("planner").await;
4621 seed_agent_context(&engine, &task_id, 1, &dir.path().to_string_lossy()).await;
4622
4623 engine
4624 .stage_worker_artifact_trusted(
4625 &task_id,
4626 1,
4627 "plan.md".to_string(),
4628 serde_json::json!("# Plan\n\nstep one\n"),
4629 )
4630 .await
4631 .expect("stage artifact");
4632
4633 let expected_file = dir
4634 .path()
4635 .join("workspace/tasks")
4636 .join(task_id.as_str())
4637 .join("ctx/plan.md");
4638 assert!(
4639 expected_file.exists(),
4640 "materialized part file missing at {expected_file:?}"
4641 );
4642 let body = std::fs::read_to_string(expected_file).unwrap();
4643 // Raw — no YAML front matter / fenced-json wrapper.
4644 assert_eq!(body, "# Plan\n\nstep one\n");
4645 }
4646
4647 /// No resolvable root + `Warn` — staging still
4648 /// succeeds (fail-open), and no part file is written.
4649 #[tokio::test]
4650 async fn stage_artifact_check_policy_warn_skips_part_file_when_root_unresolved() {
4651 let dir = tempfile::TempDir::new().unwrap();
4652 let (engine, _op, task_id, _worker_token) =
4653 seeded_task_with_policy("planner", crate::core::config::CheckPolicy::Warn).await;
4654 // No seed_agent_context — root unresolved.
4655
4656 let result = engine
4657 .stage_worker_artifact_trusted(
4658 &task_id,
4659 1,
4660 "plan.md".to_string(),
4661 serde_json::json!("x"),
4662 )
4663 .await;
4664 assert!(
4665 result.is_ok(),
4666 "Warn mode preserves fail-open: stage must succeed when root unresolved"
4667 );
4668 assert!(
4669 !dir.path().join("workspace").exists(),
4670 "no part file may be materialized when root is unresolved"
4671 );
4672 }
4673
4674 /// No resolvable root + `Strict` — staging surfaces
4675 /// the fail-open condition as an [`EngineError::CheckPolicyStrict`],
4676 /// its message identifying the "part file materialize" call site.
4677 #[tokio::test]
4678 async fn stage_artifact_check_policy_strict_surfaces_error_when_root_unresolved() {
4679 let (engine, _op, task_id, _worker_token) =
4680 seeded_task_with_policy("planner", crate::core::config::CheckPolicy::Strict).await;
4681
4682 let err = engine
4683 .stage_worker_artifact_trusted(
4684 &task_id,
4685 1,
4686 "plan.md".to_string(),
4687 serde_json::json!("x"),
4688 )
4689 .await
4690 .expect_err("Strict mode must return an error when root unresolved");
4691 match err {
4692 EngineError::CheckPolicyStrict { context, message } => {
4693 assert!(
4694 context.contains("part file materialize"),
4695 "context must identify the call site: {context}"
4696 );
4697 assert!(
4698 message.contains("part file materialize"),
4699 "message must identify the part-file sink: {message}"
4700 );
4701 assert!(
4702 message.contains("no work_dir/project_root resolved"),
4703 "message must preserve the warn-log literal: {message}"
4704 );
4705 }
4706 other => panic!(
4707 "expected EngineError::CheckPolicyStrict, got a different variant: {other:?}"
4708 ),
4709 }
4710 }
4711
4712 /// A path-traversal `name` (`../evil.md`) with a
4713 /// resolved root — the name guard fails the write, but fail-open keeps
4714 /// the stage succeeding, and nothing is written outside the ctx dir.
4715 #[tokio::test]
4716 async fn stage_artifact_traversal_name_is_fail_open_and_writes_nothing() {
4717 let dir = tempfile::TempDir::new().unwrap();
4718 let (engine, _op, task_id, _worker_token) = seeded_task("planner").await;
4719 seed_agent_context(&engine, &task_id, 1, &dir.path().to_string_lossy()).await;
4720
4721 let result = engine
4722 .stage_worker_artifact_trusted(
4723 &task_id,
4724 1,
4725 "../evil.md".to_string(),
4726 serde_json::json!("pwned"),
4727 )
4728 .await;
4729 assert!(
4730 result.is_ok(),
4731 "default (Warn) policy is fail-open even on a rejected part name"
4732 );
4733 // The escaped target (ctx dir's parent) must not have been written.
4734 let escaped = dir
4735 .path()
4736 .join("workspace/tasks")
4737 .join(task_id.as_str())
4738 .join("evil.md");
4739 assert!(
4740 !escaped.exists(),
4741 "a traversal name must never write outside the ctx dir: {escaped:?}"
4742 );
4743 }
4744}
4745
4746/// GH #36 ST1: named multi-part worker output. Covers (a) the pure
4747/// `fold_final_and_parts` assembly `dispatch_attempt_with`'s Final-pull
4748/// delegates to, (b) `stage_worker_artifact_trusted`'s per-attempt
4749/// isolation on `EngineState.output_store` / `.worker_artifact_names` (the
4750/// same `HashMap<(StepId, u32), _>` key shape `submit_worker_result_trusted`
4751/// uses — a fresh attempt is a fresh key, so nothing to explicitly "clean
4752/// up"), and (c) the allowlist behavior that keeps a non-opt-in `Artifact`
4753/// producer (e.g. `AfterRunAuditMiddleware`) from being folded in.
4754#[cfg(test)]
4755mod named_multi_part_worker_output_tests {
4756 use super::*;
4757 use crate::worker::output::{ContentRef, OutputEvent};
4758
4759 fn artifact(name: &str, value: Value) -> OutputEvent {
4760 OutputEvent::Artifact {
4761 name: name.to_string(),
4762 content: ContentRef::Inline { value },
4763 }
4764 }
4765
4766 fn final_ev(value: Value, ok: bool) -> OutputEvent {
4767 OutputEvent::Final {
4768 content: ContentRef::Inline { value },
4769 ok,
4770 }
4771 }
4772
4773 fn names(list: &[&str]) -> Vec<String> {
4774 list.iter().map(|s| s.to_string()).collect()
4775 }
4776
4777 /// Two staged parts (both in `staged_names`) + a `Final` fold into
4778 /// `{"out", "parts"}`, each value carried through verbatim.
4779 #[test]
4780 fn fold_final_and_parts_assembles_out_and_parts_shape() {
4781 let tail = vec![
4782 artifact("summary", serde_json::json!("the summary")),
4783 artifact("diff", serde_json::json!({"lines": 3})),
4784 final_ev(serde_json::json!("final text"), true),
4785 ];
4786 let staged = names(&["summary", "diff"]);
4787 let (value, ok) = fold_final_and_parts(&tail, &staged).expect("Final present");
4788 assert!(ok);
4789 assert_eq!(
4790 value,
4791 serde_json::json!({
4792 "out": "final text",
4793 "parts": {
4794 "summary": "the summary",
4795 "diff": {"lines": 3},
4796 }
4797 })
4798 );
4799 }
4800
4801 /// Zero staged parts: the value is exactly the plain `Final` value — no
4802 /// `{"out", "parts"}` wrapping. This is the back-compat guarantee (GH
4803 /// #36 must not change the shape for a worker that never POSTs to
4804 /// `/v1/worker/artifact`).
4805 #[test]
4806 fn fold_final_and_parts_with_no_parts_returns_plain_final_value() {
4807 let tail = vec![final_ev(serde_json::json!("plain value"), true)];
4808 let (value, ok) = fold_final_and_parts(&tail, &[]).expect("Final present");
4809 assert!(ok);
4810 assert_eq!(value, serde_json::json!("plain value"));
4811 }
4812
4813 /// The same staged part `name` appearing twice in one attempt: the
4814 /// LATER (tail-order) value wins — `parts` is a `Map`, not an
4815 /// accumulating list.
4816 #[test]
4817 fn fold_final_and_parts_same_name_twice_last_write_wins() {
4818 let tail = vec![
4819 artifact("a", serde_json::json!("first")),
4820 artifact("a", serde_json::json!("second")),
4821 final_ev(serde_json::json!("f"), true),
4822 ];
4823 let staged = names(&["a"]);
4824 let (value, _ok) = fold_final_and_parts(&tail, &staged).expect("Final present");
4825 assert_eq!(
4826 value,
4827 serde_json::json!({"out": "f", "parts": {"a": "second"}})
4828 );
4829 }
4830
4831 /// No `Final` anywhere in the tail (only staged parts, e.g. the worker
4832 /// crashed before submitting) — `None`, the caller's pre-existing "no
4833 /// Final in output_tail" error path.
4834 #[test]
4835 fn fold_final_and_parts_returns_none_when_no_final_present() {
4836 let tail = vec![artifact("a", serde_json::json!("v"))];
4837 let staged = names(&["a"]);
4838 assert!(fold_final_and_parts(&tail, &staged).is_none());
4839 }
4840
4841 /// An `Artifact` on the tail whose name is NOT in `staged_names` (e.g.
4842 /// `AfterRunAuditMiddleware`'s `"audit:<step_ref>"` sidecar finding on
4843 /// an audited step's own tail) must NOT be folded into `"parts"` — the
4844 /// value stays the plain `Final` value, exactly the pre-GH-#36
4845 /// behavior for every producer that isn't the worker's own
4846 /// `/v1/worker/artifact` staging. This is the regression this fold was
4847 /// almost shipped without (see `dispatch_attempt_with`'s doc).
4848 #[test]
4849 fn fold_final_and_parts_ignores_artifacts_outside_the_staged_allowlist() {
4850 let tail = vec![
4851 final_ev(serde_json::json!({"echoed": "hi"}), true),
4852 artifact("audit:echo", serde_json::json!({"finding": "clean"})),
4853 ];
4854 // `staged_names` empty: the worker itself never staged anything —
4855 // the audit sidecar Artifact must be ignored.
4856 let (value, ok) = fold_final_and_parts(&tail, &[]).expect("Final present");
4857 assert!(ok);
4858 assert_eq!(value, serde_json::json!({"echoed": "hi"}));
4859 }
4860
4861 /// Mixed tail: one staged (allowlisted) part and one non-staged
4862 /// (audit-style) `Artifact` — only the staged one is folded in.
4863 #[test]
4864 fn fold_final_and_parts_folds_only_the_staged_subset_of_a_mixed_tail() {
4865 let tail = vec![
4866 artifact("summary", serde_json::json!("s")),
4867 artifact("audit:echo", serde_json::json!({"finding": "clean"})),
4868 final_ev(serde_json::json!("f"), true),
4869 ];
4870 let staged = names(&["summary"]);
4871 let (value, _ok) = fold_final_and_parts(&tail, &staged).expect("Final present");
4872 assert_eq!(
4873 value,
4874 serde_json::json!({"out": "f", "parts": {"summary": "s"}})
4875 );
4876 }
4877
4878 /// `stage_worker_artifact_trusted` writes onto the `(task_id, attempt)`
4879 /// key exactly like `submit_worker_result_trusted` does — a part staged
4880 /// under attempt N is invisible to an `output_tail` / allowlist read of
4881 /// attempt N+1 (a fresh attempt starts empty; nothing carries over).
4882 #[tokio::test]
4883 async fn stage_worker_artifact_trusted_is_isolated_per_attempt() {
4884 let engine = Engine::new(EngineCfg::default());
4885 let task_id = StepId::new();
4886
4887 engine
4888 .stage_worker_artifact_trusted(&task_id, 1, "a".to_string(), serde_json::json!("v1"))
4889 .await
4890 .expect("stage attempt 1");
4891
4892 let attempt_1_tail = engine.output_tail(&task_id, 1).await;
4893 assert_eq!(attempt_1_tail.len(), 1);
4894 assert!(matches!(
4895 &attempt_1_tail[0],
4896 OutputEvent::Artifact { name, .. } if name == "a"
4897 ));
4898 assert_eq!(
4899 engine.worker_artifact_names_for(&task_id, 1).await,
4900 vec!["a".to_string()]
4901 );
4902
4903 let attempt_2_tail = engine.output_tail(&task_id, 2).await;
4904 assert!(
4905 attempt_2_tail.is_empty(),
4906 "attempt 2 must not see attempt 1's staged part"
4907 );
4908 assert!(
4909 engine
4910 .worker_artifact_names_for(&task_id, 2)
4911 .await
4912 .is_empty(),
4913 "attempt 2's allowlist must not see attempt 1's staged name"
4914 );
4915 }
4916}
4917
4918// ─── GH #50 (Subtask 2): `Engine::register_verdict_contracts` /
4919// `Engine::verdict_contract_for_task` ────────────────────────────────────
4920#[cfg(test)]
4921mod verdict_contract_registry_tests {
4922 use super::*;
4923
4924 async fn seeded_engine(agent: &str) -> (Engine, StepId) {
4925 let engine = Engine::new(EngineCfg::default());
4926 let op_token = engine
4927 .attach("ut-op", Role::Operator, Duration::from_secs(30))
4928 .await
4929 .expect("attach");
4930 let task_id = engine
4931 .start_task(
4932 &op_token,
4933 TaskSpec {
4934 agent: agent.to_string(),
4935 initial_directive: serde_json::json!("x"),
4936 step_ctx: None,
4937 check_policy: None,
4938 },
4939 )
4940 .await
4941 .expect("start_task");
4942 (engine, task_id)
4943 }
4944
4945 /// An agent with no registered contract at all → `None` (the opt-in
4946 /// default; every pre-GH-#50 `Engine`).
4947 #[tokio::test]
4948 async fn returns_none_when_no_contract_registered_for_the_agent() {
4949 let (engine, task_id) = seeded_engine("gate").await;
4950 assert_eq!(engine.verdict_contract_for_task(&task_id).await, None);
4951 }
4952
4953 /// A registered contract for the running task's agent is returned
4954 /// verbatim.
4955 #[tokio::test]
4956 async fn returns_the_registered_contract_for_the_running_agent() {
4957 let (engine, task_id) = seeded_engine("gate").await;
4958 let contract = mlua_swarm_schema::VerdictContract {
4959 channel: mlua_swarm_schema::VerdictChannel::Body,
4960 values: vec!["PASS".to_string(), "BLOCKED".to_string()],
4961 };
4962 engine.register_verdict_contracts(HashMap::from([("gate".to_string(), contract.clone())]));
4963 assert_eq!(
4964 engine.verdict_contract_for_task(&task_id).await,
4965 Some(contract)
4966 );
4967 }
4968
4969 /// A registered contract for a DIFFERENT agent name never leaks onto
4970 /// an unrelated task.
4971 #[tokio::test]
4972 async fn does_not_leak_a_contract_registered_for_a_different_agent() {
4973 let (engine, task_id) = seeded_engine("gate").await;
4974 engine.register_verdict_contracts(HashMap::from([(
4975 "other-agent".to_string(),
4976 mlua_swarm_schema::VerdictContract {
4977 channel: mlua_swarm_schema::VerdictChannel::Body,
4978 values: vec!["PASS".to_string()],
4979 },
4980 )]));
4981 assert_eq!(engine.verdict_contract_for_task(&task_id).await, None);
4982 }
4983
4984 /// An unknown `task_id` → `None`, not a panic / error.
4985 #[tokio::test]
4986 async fn returns_none_for_an_unknown_task_id() {
4987 let engine = Engine::new(EngineCfg::default());
4988 let unknown = StepId::new();
4989 assert_eq!(engine.verdict_contract_for_task(&unknown).await, None);
4990 }
4991
4992 /// `register_verdict_contracts` is additive (`HashMap::extend`): a
4993 /// second call registering a DIFFERENT agent does not clobber the
4994 /// first call's entry.
4995 #[tokio::test]
4996 async fn register_verdict_contracts_is_additive_across_calls() {
4997 let (engine, task_id) = seeded_engine("gate").await;
4998 let contract = mlua_swarm_schema::VerdictContract {
4999 channel: mlua_swarm_schema::VerdictChannel::Part,
5000 values: vec!["ALLOW".to_string()],
5001 };
5002 engine.register_verdict_contracts(HashMap::from([("gate".to_string(), contract.clone())]));
5003 engine.register_verdict_contracts(HashMap::from([(
5004 "unrelated-agent".to_string(),
5005 mlua_swarm_schema::VerdictContract {
5006 channel: mlua_swarm_schema::VerdictChannel::Body,
5007 values: vec!["X".to_string()],
5008 },
5009 )]));
5010 assert_eq!(
5011 engine.verdict_contract_for_task(&task_id).await,
5012 Some(contract)
5013 );
5014 }
5015}
5016
5017// ─── GH #51: completion-time verdict-contract enforcement — the shared
5018// `Engine::verdict_contract_completion_check` choke point embedded inside
5019// `submit_worker_result_trusted` / `submit_output`, exercised here at the
5020// `submit_output` level (the WS Operator fallback route's own unit-test
5021// coverage — see `crates/mlua-swarm-server/tests/verdict_contract.rs` for
5022// the HTTP-round-trip coverage of the other 2 routes) ───────────────────
5023#[cfg(test)]
5024mod verdict_contract_completion_tests {
5025 use super::*;
5026
5027 /// Seeds a `Pending` task bound to `agent` and mints a bound
5028 /// `Role::Worker` token for it — the same mint-and-register pattern
5029 /// `initial_directive_value_passthrough_tests::mint_worker_token`
5030 /// uses (duplicated here: that helper is private to its own sibling
5031 /// `#[cfg(test)]` module, not reachable via `super::*` from this one).
5032 async fn seeded_task_with_worker_token(agent: &str) -> (Engine, CapToken, StepId) {
5033 let engine = Engine::new(EngineCfg::default());
5034 let op_token = engine
5035 .attach("ut-op", Role::Operator, Duration::from_secs(30))
5036 .await
5037 .expect("attach");
5038 let task_id = engine
5039 .start_task(
5040 &op_token,
5041 TaskSpec {
5042 agent: agent.to_string(),
5043 initial_directive: serde_json::json!("x"),
5044 step_ctx: None,
5045 check_policy: None,
5046 },
5047 )
5048 .await
5049 .expect("start_task");
5050 let worker_token = engine.signer().session(
5051 format!("worker-of-{task_id}"),
5052 Role::Worker,
5053 vec!["*".into()],
5054 Duration::from_secs(600),
5055 );
5056 let fp = worker_token.fingerprint();
5057 let record = CapTokenRecord::from_worker_token(worker_token.clone(), task_id.clone());
5058 engine
5059 .with_state("test.mint_worker", move |s| {
5060 s.tokens.insert(fp, record);
5061 })
5062 .await
5063 .expect("mint worker token");
5064 (engine, worker_token, task_id)
5065 }
5066
5067 fn body_contract(values: &[&str]) -> mlua_swarm_schema::VerdictContract {
5068 mlua_swarm_schema::VerdictContract {
5069 channel: mlua_swarm_schema::VerdictChannel::Body,
5070 values: values.iter().map(|v| v.to_string()).collect(),
5071 }
5072 }
5073
5074 fn part_contract(values: &[&str]) -> mlua_swarm_schema::VerdictContract {
5075 mlua_swarm_schema::VerdictContract {
5076 channel: mlua_swarm_schema::VerdictChannel::Part,
5077 values: values.iter().map(|v| v.to_string()).collect(),
5078 }
5079 }
5080
5081 fn final_event(value: Value, ok: bool) -> crate::worker::output::OutputEvent {
5082 crate::worker::output::OutputEvent::Final {
5083 content: crate::worker::output::ContentRef::Inline { value },
5084 ok,
5085 }
5086 }
5087
5088 /// Route 3 (WS Operator fallback, `submit_output` level) — a
5089 /// `channel: "part"` contract's attempt completes via a plain
5090 /// `Final` without ever staging a `"verdict"` artifact: rejected
5091 /// with `EngineError::VerdictPartMissing`, and nothing lands on
5092 /// `output_tail` — the rejected value never reaches the flow ctx.
5093 #[tokio::test]
5094 async fn submit_output_rejects_missing_verdict_part() {
5095 let (engine, token, task_id) = seeded_task_with_worker_token("gate").await;
5096 engine.register_verdict_contracts(HashMap::from([(
5097 "gate".to_string(),
5098 part_contract(&["PASS", "BLOCKED"]),
5099 )]));
5100
5101 let err = engine
5102 .submit_output(
5103 &token,
5104 &task_id,
5105 1,
5106 final_event(serde_json::json!("anything"), true),
5107 )
5108 .await
5109 .expect_err("missing staged verdict part must be rejected");
5110 assert!(
5111 matches!(err, EngineError::VerdictPartMissing { .. }),
5112 "unexpected error variant: {err:?}"
5113 );
5114
5115 let tail = engine.output_tail(&task_id, 1).await;
5116 assert!(
5117 !tail
5118 .iter()
5119 .any(|ev| matches!(ev, crate::worker::output::OutputEvent::Final { .. })),
5120 "a rejected completion must not write a Final onto output_tail"
5121 );
5122 }
5123
5124 /// Route 3 — a `channel: "part"` contract completes normally when the
5125 /// worker DID stage a matching `"verdict"` artifact first (defense in
5126 /// depth: presence AND membership both hold).
5127 #[tokio::test]
5128 async fn submit_output_accepts_when_verdict_part_is_staged_and_a_member() {
5129 let (engine, token, task_id) = seeded_task_with_worker_token("gate").await;
5130 engine.register_verdict_contracts(HashMap::from([(
5131 "gate".to_string(),
5132 part_contract(&["PASS", "BLOCKED"]),
5133 )]));
5134 engine
5135 .stage_worker_artifact_trusted(
5136 &task_id,
5137 1,
5138 "verdict".to_string(),
5139 serde_json::json!("PASS"),
5140 )
5141 .await
5142 .expect("stage verdict part");
5143
5144 engine
5145 .submit_output(
5146 &token,
5147 &task_id,
5148 1,
5149 final_event(serde_json::json!("full report"), true),
5150 )
5151 .await
5152 .expect("staged + member verdict part must be accepted");
5153
5154 let tail = engine.output_tail(&task_id, 1).await;
5155 assert!(
5156 tail.iter()
5157 .any(|ev| matches!(ev, crate::worker::output::OutputEvent::Final { .. })),
5158 "an accepted completion must write its Final onto output_tail"
5159 );
5160 }
5161
5162 /// Route 3 — a `channel: "body"` contract's completing value is NOT a
5163 /// member of `values`: rejected with
5164 /// `EngineError::VerdictValueRejected`, no `Final` written.
5165 #[tokio::test]
5166 async fn submit_output_rejects_body_value_outside_contract() {
5167 let (engine, token, task_id) = seeded_task_with_worker_token("gate").await;
5168 engine.register_verdict_contracts(HashMap::from([(
5169 "gate".to_string(),
5170 body_contract(&["PASS", "BLOCKED"]),
5171 )]));
5172
5173 let err = engine
5174 .submit_output(
5175 &token,
5176 &task_id,
5177 1,
5178 final_event(serde_json::json!("UNKNOWN"), true),
5179 )
5180 .await
5181 .expect_err("out-of-contract body value must be rejected");
5182 match err {
5183 EngineError::VerdictValueRejected { value, allowed } => {
5184 assert_eq!(value, "UNKNOWN");
5185 assert_eq!(allowed, vec!["PASS".to_string(), "BLOCKED".to_string()]);
5186 }
5187 other => panic!("unexpected error variant: {other:?}"),
5188 }
5189
5190 let tail = engine.output_tail(&task_id, 1).await;
5191 assert!(
5192 !tail
5193 .iter()
5194 .any(|ev| matches!(ev, crate::worker::output::OutputEvent::Final { .. })),
5195 "a rejected completion must not write a Final onto output_tail"
5196 );
5197 }
5198
5199 /// `ok=false` bypasses the completion-time check entirely, regardless
5200 /// of channel or membership — the exemption acceptance criterion,
5201 /// exercised at the `submit_output` choke point.
5202 #[tokio::test]
5203 async fn submit_output_ok_false_bypasses_the_check() {
5204 let (engine, token, task_id) = seeded_task_with_worker_token("gate").await;
5205 engine.register_verdict_contracts(HashMap::from([(
5206 "gate".to_string(),
5207 body_contract(&["PASS", "BLOCKED"]),
5208 )]));
5209
5210 engine
5211 .submit_output(
5212 &token,
5213 &task_id,
5214 1,
5215 final_event(serde_json::json!("UNKNOWN"), false),
5216 )
5217 .await
5218 .expect("ok=false must bypass the verdict contract check entirely");
5219
5220 let tail = engine.output_tail(&task_id, 1).await;
5221 assert!(
5222 tail.iter()
5223 .any(|ev| matches!(ev, crate::worker::output::OutputEvent::Final { .. })),
5224 "an ok=false completion is exempt, not rejected — its Final must still land"
5225 );
5226 }
5227
5228 /// `staged_verdict_value_for` mirrors `fold_final_and_parts`'s
5229 /// last-write-wins semantics: staging `"verdict"` twice within the
5230 /// same attempt returns the LAST value, not the first.
5231 #[tokio::test]
5232 async fn staged_verdict_value_for_is_last_write_wins() {
5233 let (engine, _token, task_id) = seeded_task_with_worker_token("gate").await;
5234 engine
5235 .stage_worker_artifact_trusted(
5236 &task_id,
5237 1,
5238 "verdict".to_string(),
5239 serde_json::json!("PASS"),
5240 )
5241 .await
5242 .expect("stage first verdict part");
5243 engine
5244 .stage_worker_artifact_trusted(
5245 &task_id,
5246 1,
5247 "verdict".to_string(),
5248 serde_json::json!("BLOCKED"),
5249 )
5250 .await
5251 .expect("stage second verdict part");
5252
5253 assert_eq!(
5254 engine.staged_verdict_value_for(&task_id, 1).await,
5255 Some("BLOCKED".to_string())
5256 );
5257 }
5258
5259 /// `staged_verdict_value_for` ignores artifacts staged under any name
5260 /// OTHER than the literal `"verdict"` — mirrors `channel: "part"`
5261 /// contracts only ever addressing that one part.
5262 #[tokio::test]
5263 async fn staged_verdict_value_for_ignores_other_artifact_names() {
5264 let (engine, _token, task_id) = seeded_task_with_worker_token("gate").await;
5265 engine
5266 .stage_worker_artifact_trusted(
5267 &task_id,
5268 1,
5269 "notes".to_string(),
5270 serde_json::json!("irrelevant"),
5271 )
5272 .await
5273 .expect("stage unrelated part");
5274
5275 assert_eq!(engine.staged_verdict_value_for(&task_id, 1).await, None);
5276 }
5277
5278 /// `staged_verdict_value_for` → `None` when nothing was ever staged —
5279 /// the normal case the completion check turns into
5280 /// `EngineError::VerdictPartMissing`.
5281 #[tokio::test]
5282 async fn staged_verdict_value_for_returns_none_when_nothing_staged() {
5283 let (engine, _token, task_id) = seeded_task_with_worker_token("gate").await;
5284 assert_eq!(engine.staged_verdict_value_for(&task_id, 1).await, None);
5285 }
5286}