Skip to main content

aion_server/api/handlers/
workflows.rs

1//! Start/signal/query/cancel workflow operation handlers.
2
3use aion_proto::{
4    ProtoCancelRequest, ProtoCancelResponse, ProtoPauseRequest, ProtoPauseResponse,
5    ProtoQueryRequest, ProtoQueryResponse, ProtoReopenRequest, ProtoReopenResponse,
6    ProtoResumeRequest, ProtoResumeResponse, ProtoSignalRequest, ProtoSignalResponse,
7    ProtoStartWorkflowRequest, ProtoStartWorkflowResponse, WireError, WireErrorCode,
8    proto_query_response,
9};
10use tracing::{Instrument, info_span};
11
12use super::error::{
13    cancel_terminal_error, log_server_error, map_start_error, map_workflow_operation_error,
14    signal_terminal_error,
15};
16use super::payload::{optional_payload, required_payload, required_workflow_id};
17use super::runs::{resolve_run_id, terminal_status};
18use crate::{
19    CallerIdentity, NamespaceGuard, NamespaceMinter, NamespaceOperation, ServerError, ServerState,
20    WorkflowTarget,
21};
22
23/// Handles a decoded start-workflow request.
24///
25/// The authorized namespace is recorded durably as the `aion.namespace` search
26/// attribute in the same atomic append as the workflow's start event, so
27/// ownership survives server restarts and is never tracked only in memory.
28///
29/// # Errors
30///
31/// Returns a stable [`WireError`] (`invalid_input`) when the payload is absent or malformed, namespace scoping fails,
32/// or the engine start call fails.
33pub async fn start(
34    guard: &NamespaceGuard,
35    caller: &CallerIdentity,
36    request: ProtoStartWorkflowRequest,
37) -> Result<ProtoStartWorkflowResponse, WireError> {
38    start_with_placement(guard, caller, request, None, None).await
39}
40
41/// Start a workflow, optionally with a `placement` id chosen by the routing edge
42/// so the new execution lands on a locally-owned shard (R-1 unsteered-start
43/// remint). `placement = None` is the default path: the engine mints the id, so
44/// the single-node / non-clustered behaviour is unchanged.
45///
46/// `minter` is the minted-on-use safety net (Control-Plane Phase 1, S6): when
47/// `Some`, the resolved-and-authorized namespace is durably minted (open) or
48/// gated (closed) BEFORE the engine start, so a client that starts a workflow
49/// before any worker registers still gets a durable namespace record. It is the
50/// SAME [`NamespaceMinter`] policy the worker-registration seam (S5) applies, so
51/// the two transports and the two mint choke-points can never diverge. `None`
52/// disables the mint entirely (every unit test of the bare handler), leaving the
53/// start path byte-identical.
54///
55/// The mint runs AFTER namespace authorization (`guard.scope`), so it is
56/// auth-scoped by construction — it can only record a namespace the caller is
57/// already permitted to start in. It does NOT change the immutable NSTQ
58/// `aion.namespace` binding ([`start_search_attributes`]) or the start response
59/// shape; the mint is purely additive.
60///
61/// # Errors
62///
63/// Identical to [`start`], plus a durable-store failure (a retryable `NotOwner`
64/// fence surfaces as such) or a `closed`-policy namespace-denied error from the
65/// minter, all mapped to a stable [`WireError`].
66pub async fn start_with_placement(
67    guard: &NamespaceGuard,
68    caller: &CallerIdentity,
69    request: ProtoStartWorkflowRequest,
70    placement: Option<aion_core::WorkflowId>,
71    minter: Option<&NamespaceMinter>,
72) -> Result<ProtoStartWorkflowResponse, WireError> {
73    let scoped = guard
74        .scope(caller, &NamespaceOperation::start(&request))
75        .await
76        .map_err(|error| error.to_wire_error())?;
77    let namespace = scoped.namespace().to_owned();
78    // MINT-ON-START safety net (Phase 1 S6). Runs strictly AFTER the namespace
79    // authorization above, so it can only ever mint a namespace the caller is
80    // already authorized to start in — auth-scoped by construction. A `closed`
81    // policy rejects an unknown namespace with the same namespace-denied error;
82    // a quorum `NotOwner` fence propagates as the retryable wire code, never a
83    // silent success. Shares the EXACT S5 policy via `NamespaceMinter`.
84    if let Some(minter) = minter {
85        minter
86            .mint_or_gate(
87                std::slice::from_ref(&namespace),
88                aion_store::NamespaceOrigin::StartMint,
89            )
90            .await
91            .map_err(|error| error.to_wire_error())?;
92    }
93    let input = required_payload(request.input.clone())?;
94    // An empty task_queue means "not selected": fall back to the namespace's
95    // default queue rather than recording an empty selection.
96    let task_queue = request
97        .task_queue
98        .as_deref()
99        .map(str::trim)
100        .filter(|queue| !queue.is_empty());
101    // #211: a PRESENT but blank display_name is refused, not reinterpreted.
102    //
103    // The server is the trust boundary — the SDKs are not the only callers, and
104    // raw gRPC, the HTTP body, and the MCP `start_run` tool all reach here.
105    // Trimming a blank one to "unnamed" would answer 200 to an operator who
106    // believed they had named the run and say nothing about the name being
107    // dropped. `None`/absent is how a caller says "unnamed"; a blank string is
108    // a mistake, and the rename endpoint already refuses the same input, so
109    // accepting it here would make the two surfaces disagree about what a blank
110    // name means.
111    let display_name = match request.display_name.as_deref().map(str::trim) {
112        Some("") => {
113            return Err(WireError::invalid_input(
114                "display_name must not be blank; omit it to start the run unnamed",
115            ));
116        }
117        other => other,
118    };
119    let span = info_span!(
120        "engine_operation",
121        operation = "start",
122        namespace = %namespace,
123        workflow_id = tracing::field::Empty,
124        workflow_type = %request.workflow_type,
125    );
126    let search_attributes = start_search_attributes(&namespace, task_queue, display_name);
127    let handle = async {
128        let engine = scoped
129            .engine()
130            .map_err(|error| log_server_error("start", Some(&namespace), None, &error))?;
131        // 🔴 A WORKLOOP IS STARTED AS A WORKLOOP, AND ONLY THE PACKAGE KNOWS.
132        //
133        // Nothing in a start request says "this is a loop": the caller names a
134        // workflow type and hands over a payload, exactly as for any workflow.
135        // The DEPLOYED PACKAGE says so, in the workloop declaration the
136        // compiler bound into its identity, and `declared_workloop_spec` is
137        // where that is read.
138        //
139        // Taking the ordinary path for a workloop is not a degraded start, it
140        // is a broken one: the loop would run its first iteration and be
141        // REFUSED at `close_iteration` for not being a registered workloop,
142        // having recorded nothing, with the run left live. And the two starts
143        // are genuinely different operations — `start_workloop` writes the
144        // registration BEFORE the workflow exists (so generation 1 cannot lose
145        // a race with its own registration), seeds the declared carry defaults
146        // into the generation-1 payload, and folds the `aion.kind` stamp into
147        // the start's own append so boot recovery can tell a parked loop from
148        // a crashed workflow.
149        let declared = engine
150            .declared_workloop_spec(&request.workflow_type)
151            .map_err(|error| map_start_error(error, &request.workflow_type))?;
152        match declared {
153            Some(spec) => engine
154                .start_workloop(
155                    &request.workflow_type,
156                    input,
157                    search_attributes,
158                    namespace.clone(),
159                    spec,
160                )
161                .await
162                .map_err(|error| map_start_error(error, &request.workflow_type)),
163            None => engine
164                .start_workflow_with_id(
165                    &request.workflow_type,
166                    input,
167                    search_attributes,
168                    namespace.clone(),
169                    placement,
170                    // Steered-start shard derivation already happened at the
171                    // edge (which holds the concrete cluster store); the engine
172                    // receives the derived placement id, so no routing key is
173                    // threaded here.
174                    None,
175                )
176                .await
177                .map_err(|error| map_start_error(error, &request.workflow_type)),
178        }
179    }
180    .instrument(span.clone())
181    .await?;
182    span.record("workflow_id", tracing::field::display(handle.workflow_id()));
183
184    Ok(ProtoStartWorkflowResponse {
185        workflow_id: Some(handle.workflow_id().clone().into()),
186        run_id: Some(handle.run_id().clone().into()),
187    })
188}
189
190/// Search attribute map stamping the authorized namespace — and, when the start
191/// selected them, the default task queue and the operator-facing display name
192/// (#211) — onto an execution.
193///
194/// All are recorded in the same atomic append as `WorkflowStarted`, so the
195/// `(namespace, task_queue)` targeting selection and the label survive
196/// restarts/failover and are never tracked only in memory. `task_queue` is
197/// omitted when the start did not select one (the workflow falls back to the
198/// namespace's default queue); `display_name` is omitted when the start did not
199/// name it (it renders as its bare UUID until a rename records one).
200///
201/// The recorded `aion.display_name` carries no run id, so readers fold it over
202/// the whole workflow history: the name stamped here is a per-WORKFLOW label a
203/// continue-as-new successor inherits, not a per-run one.
204pub(crate) fn start_search_attributes(
205    namespace: &str,
206    task_queue: Option<&str>,
207    display_name: Option<&str>,
208) -> std::collections::HashMap<String, aion_core::SearchAttributeValue> {
209    let mut attributes = std::collections::HashMap::from([(
210        crate::namespace::NAMESPACE_ATTRIBUTE.to_owned(),
211        aion_core::SearchAttributeValue::String(namespace.to_owned()),
212    )]);
213    if let Some(task_queue) = task_queue {
214        attributes.insert(
215            crate::namespace::TASK_QUEUE_ATTRIBUTE.to_owned(),
216            aion_core::SearchAttributeValue::String(task_queue.to_owned()),
217        );
218    }
219    if let Some(display_name) = display_name {
220        attributes.insert(
221            crate::namespace::DISPLAY_NAME_ATTRIBUTE.to_owned(),
222            aion_core::SearchAttributeValue::String(display_name.to_owned()),
223        );
224    }
225    attributes
226}
227
228/// Handles a decoded signal request.
229///
230/// # Errors
231///
232/// Returns a stable [`WireError`] when IDs or payloads are missing or malformed, namespace scoping
233/// fails, or the engine signal call fails.
234pub async fn signal(
235    guard: &NamespaceGuard,
236    caller: &CallerIdentity,
237    request: ProtoSignalRequest,
238) -> Result<ProtoSignalResponse, WireError> {
239    let workflow_id = required_workflow_id(request.workflow_id.clone())?;
240    let target = WorkflowTarget::workflow(&workflow_id);
241    let scoped = guard
242        .scope(caller, &NamespaceOperation::signal(&request, target))
243        .await
244        .map_err(|error| error.to_wire_error())?;
245    let namespace = scoped.namespace().to_owned();
246    let engine = scoped.engine().map_err(|error| error.to_wire_error())?;
247    let run_id = resolve_run_id(engine.as_ref(), &workflow_id, request.run_id.clone()).await?;
248    let payload = required_payload(request.payload.clone())?;
249    if let Some(status) = terminal_status(engine.as_ref(), &workflow_id).await? {
250        return Err(signal_terminal_error(&workflow_id, status));
251    }
252
253    let signal_name = request.signal_name.clone();
254    let span = info_span!(
255        "engine_operation",
256        operation = "signal",
257        namespace = %namespace,
258        workflow_id = %workflow_id,
259        signal_name = %signal_name,
260    );
261
262    async {
263        engine
264            .signal(&workflow_id, &run_id, signal_name, payload)
265            .await
266            .map_err(|error| map_workflow_operation_error(error, &workflow_id))
267    }
268    .instrument(span)
269    .await?;
270
271    Ok(ProtoSignalResponse {})
272}
273
274/// Handles a decoded query request.
275///
276/// # Errors
277///
278/// Returns a stable [`WireError`] when IDs are missing or malformed, namespace scoping fails, or the
279/// engine query call fails.
280pub async fn query(
281    guard: &NamespaceGuard,
282    caller: &CallerIdentity,
283    request: ProtoQueryRequest,
284) -> Result<ProtoQueryResponse, WireError> {
285    let workflow_id = required_workflow_id(request.workflow_id.clone())?;
286    let target = WorkflowTarget::workflow(&workflow_id);
287    let scoped = guard
288        .scope(caller, &NamespaceOperation::query(&request, target))
289        .await
290        .map_err(|error| error.to_wire_error())?;
291    let namespace = scoped.namespace().to_owned();
292    let engine = scoped.engine().map_err(|error| error.to_wire_error())?;
293    let run_id = resolve_run_id(engine.as_ref(), &workflow_id, request.run_id.clone()).await?;
294    let query_name = request.query_name.clone();
295    // An absent arguments field means the caller supplied none; the handler
296    // still receives one well-formed document, the canonical JSON `null`.
297    let arguments = optional_payload(request.arguments.clone())?;
298    let span = info_span!(
299        "engine_operation",
300        operation = "query",
301        namespace = %namespace,
302        workflow_id = %workflow_id,
303        query_name = %query_name,
304    );
305
306    let outcome = async {
307        engine
308            .query(&workflow_id, &run_id, query_name, arguments)
309            .await
310    }
311    .instrument(span)
312    .await;
313
314    match outcome {
315        Ok(result) => Ok(ProtoQueryResponse {
316            outcome: Some(proto_query_response::Outcome::Result(result.into())),
317        }),
318        // Query-semantic failures (unknown query, timeout, not running,
319        // handler failure, reply dropped) are the operation's documented
320        // outcome and ride the QueryResponse.error oneof, which every SDK
321        // query op parses. Namespace, not-found, and backend failures stay
322        // transport-level errors, exactly as for every other operation.
323        Err(error @ aion::EngineError::Query(_)) => Ok(ProtoQueryResponse {
324            outcome: Some(proto_query_response::Outcome::Error(
325                ServerError::from(error).to_wire_error().into(),
326            )),
327        }),
328        Err(error) => Err(map_workflow_operation_error(error, &workflow_id)),
329    }
330}
331
332/// Handles a decoded cancel request.
333///
334/// After the cancellation is durably recorded, stops every activity of this run
335/// that is still executing (#233): asks the worker holding each remote one, and
336/// signals each declared body THIS SERVER is running itself. Cancelling records
337/// the fact, kills the workflow's own VM process, and settles its outbox rows so
338/// nothing more is dispatched — none of which reaches an activity that is
339/// ALREADY executing, wherever it is executing. Without that second step a
340/// cancelled run keeps a machine busy while the console truthfully reports
341/// `Cancelled`, so the operator stops watching.
342///
343/// The ask comes AFTER the engine call returns, never before: a cancel pushed
344/// ahead of the record could stop work for a cancellation that then fails to
345/// persist. And it is only an ask — a failure to reach a worker is logged with
346/// the worker and activity named, and does not fail the cancellation, because
347/// the cancellation itself genuinely happened.
348///
349/// # Errors
350///
351/// Returns a stable [`WireError`] when IDs are missing or malformed, namespace scoping fails, or the
352/// engine cancel call fails.
353pub async fn cancel(
354    state: &ServerState,
355    guard: &NamespaceGuard,
356    caller: &CallerIdentity,
357    request: ProtoCancelRequest,
358) -> Result<ProtoCancelResponse, WireError> {
359    let workflow_id = required_workflow_id(request.workflow_id.clone())?;
360    let target = WorkflowTarget::workflow(&workflow_id);
361    let scoped = guard
362        .scope(caller, &NamespaceOperation::cancel(&request, target))
363        .await
364        .map_err(|error| error.to_wire_error())?;
365    let namespace = scoped.namespace().to_owned();
366    let engine = scoped.engine().map_err(|error| error.to_wire_error())?;
367    let run_id = resolve_run_id(engine.as_ref(), &workflow_id, request.run_id.clone()).await?;
368    if let Some(status) = terminal_status(engine.as_ref(), &workflow_id).await? {
369        return Err(cancel_terminal_error(&workflow_id, status));
370    }
371
372    let span = info_span!(
373        "engine_operation",
374        operation = "cancel",
375        namespace = %namespace,
376        workflow_id = %workflow_id,
377    );
378
379    async {
380        engine
381            .cancel(&workflow_id, &run_id, request.reason)
382            .await
383            .map_err(|error| map_workflow_operation_error(error, &workflow_id))
384    }
385    .instrument(span)
386    .await?;
387
388    // #233. The cancellation is durably recorded; now stop whatever is executing
389    // this run's activities — a worker holding one, or this server running a
390    // declared body itself. A failure to reach a worker is logged with
391    // the worker and activity named (inside `cancel_in_flight_activities`) and
392    // does NOT fail the response — the cancellation genuinely happened, and
393    // reporting it as failed would be its own lie. A poisoned lock is the one
394    // exception: it means the routing state could not be read at all, and
395    // answering "cancelled" while silently asking nobody is exactly the defect
396    // this closes.
397    state
398        .cancel_in_flight_activities(&workflow_id)
399        .map_err(|error| WireError::new(WireErrorCode::Backend, error.to_string()))?;
400
401    Ok(ProtoCancelResponse {})
402}
403
404/// Retires a WORKLOOP: runs its declared `retire` body (when its deployed
405/// document declares one), records `LoopRetired` + its `WorkflowCompleted`
406/// terminal in one atomic batch, and removes the loop from the sweep set.
407///
408/// # 🔴 THE DECLARED WAY TO STOP A LOOP THAT IS NOT FAILURE
409///
410/// A workloop's completion is an INCIDENT, not its purpose, so `cancel` is the
411/// wrong verb: it records a cancellation and never runs the wind-down the
412/// document declares. Retirement is the declared stop, and it is the only one
413/// that runs the cleanup.
414///
415/// # 🔴 AUTHORIZED AS A CANCEL, DELIBERATELY
416///
417/// Retirement records a terminal on a workflow, which is exactly the authority
418/// class `cancel` guards, so it is scoped through the SAME namespace operation
419/// rather than a new one with its own grant semantics to get wrong. Anyone who
420/// may cancel this run may retire it; nobody else may.
421///
422/// # Errors
423///
424/// Returns a stable [`WireError`] when the id is missing or malformed,
425/// namespace scoping fails, the workflow is not a registered workloop, its run
426/// already recorded a terminal, or the declared retire body failed — in which
427/// case NO terminal is recorded and the loop stays registered.
428pub async fn retire_workloop(
429    guard: &NamespaceGuard,
430    caller: &CallerIdentity,
431    namespace: String,
432    workflow_id: String,
433    reason: String,
434) -> Result<(String, String), WireError> {
435    let parsed = aion_core::WorkflowId::new(
436        uuid::Uuid::parse_str(&workflow_id)
437            .map_err(|error| WireError::invalid_input(format!("workflow id: {error}")))?,
438    );
439    let cancel_request = ProtoCancelRequest {
440        namespace: namespace.clone(),
441        workflow_id: Some(parsed.clone().into()),
442        run_id: None,
443        reason: reason.clone(),
444    };
445    let target = WorkflowTarget::workflow(&parsed);
446    let scoped = guard
447        .scope(caller, &NamespaceOperation::cancel(&cancel_request, target))
448        .await
449        .map_err(|error| error.to_wire_error())?;
450    let namespace = scoped.namespace().to_owned();
451    let engine = scoped.engine().map_err(|error| error.to_wire_error())?;
452    let span = info_span!(
453        "engine_operation",
454        operation = "retire",
455        namespace = %namespace,
456        workflow_id = %parsed,
457    );
458    let recorded_reason = reason.clone();
459    async {
460        engine
461            .retire_declared_workloop(
462                &parsed,
463                reason,
464                // The retirement RESULT is the operator's, not the retire
465                // body's return value. Nothing about the loop's own state
466                // belongs here: the body exists for its effects.
467                aion_core::Payload::new(aion_core::ContentType::Json, b"null".to_vec()),
468            )
469            .await
470            .map_err(|error| map_workflow_operation_error(error, &parsed))
471    }
472    .instrument(span)
473    .await?;
474    Ok((parsed.to_string(), recorded_reason))
475}
476
477/// Handles a decoded reopen request.
478///
479/// Resolves the run (latest when omitted) and calls
480/// [`aion::Engine::reopen_workflow`], returning the reopened run id and its
481/// projected Running status. UNLIKE [`cancel`] this does NOT pre-check terminal
482/// status: the terminal-reopenable precondition is the engine's (AD-012) and the
483/// handler only surfaces its typed [`aion::EngineError::InvalidState`] error.
484///
485/// # Errors
486///
487/// Returns a stable [`WireError`] when IDs are missing or malformed, namespace
488/// scoping fails, or the engine reopen call fails — `invalid_state` for a
489/// non-reopenable-terminal run, `not_found` for an absent workflow.
490pub async fn reopen(
491    guard: &NamespaceGuard,
492    caller: &CallerIdentity,
493    request: ProtoReopenRequest,
494) -> Result<ProtoReopenResponse, WireError> {
495    let workflow_id = required_workflow_id(request.workflow_id.clone())?;
496    let target = WorkflowTarget::workflow(&workflow_id);
497    let scoped = guard
498        .scope(caller, &NamespaceOperation::reopen(&request, target))
499        .await
500        .map_err(|error| error.to_wire_error())?;
501    let namespace = scoped.namespace().to_owned();
502    let engine = scoped.engine().map_err(|error| error.to_wire_error())?;
503    let run_id = resolve_run_id(engine.as_ref(), &workflow_id, request.run_id.clone()).await?;
504
505    let span = info_span!(
506        "engine_operation",
507        operation = "reopen",
508        namespace = %namespace,
509        workflow_id = %workflow_id,
510    );
511
512    let handle = async {
513        engine
514            .reopen_workflow(&workflow_id, &run_id)
515            .await
516            .map_err(|error| map_workflow_operation_error(error, &workflow_id))
517    }
518    .instrument(span)
519    .await?;
520
521    Ok(ProtoReopenResponse {
522        run_id: Some(handle.run_id().clone().into()),
523        status: aion_proto::ProtoWorkflowStatus::from(handle.cached_status()) as i32,
524    })
525}
526
527/// Handles a decoded pause request (#204).
528///
529/// Resolves the run (latest when omitted) and calls
530/// [`aion::Engine::pause_workflow`], returning the run id and its projected
531/// `Paused` status. The Running precondition is the engine's; the handler surfaces
532/// its typed [`aion::EngineError::InvalidState`] error verbatim.
533///
534/// # Errors
535///
536/// Returns a stable [`WireError`] when IDs are missing/malformed, namespace
537/// scoping fails, or the engine pause call fails — `invalid_state` when the run is
538/// not Running, `not_found` for an absent workflow.
539pub async fn pause(
540    guard: &NamespaceGuard,
541    caller: &CallerIdentity,
542    request: ProtoPauseRequest,
543) -> Result<ProtoPauseResponse, WireError> {
544    let workflow_id = required_workflow_id(request.workflow_id.clone())?;
545    let target = WorkflowTarget::workflow(&workflow_id);
546    let scoped = guard
547        .scope(
548            caller,
549            &NamespaceOperation::pause_workflow(&request, target),
550        )
551        .await
552        .map_err(|error| error.to_wire_error())?;
553    let namespace = scoped.namespace().to_owned();
554    let engine = scoped.engine().map_err(|error| error.to_wire_error())?;
555    let run_id = resolve_run_id(engine.as_ref(), &workflow_id, request.run_id.clone()).await?;
556    let reason = if request.reason.is_empty() {
557        None
558    } else {
559        Some(request.reason.clone())
560    };
561
562    let span = info_span!(
563        "engine_operation",
564        operation = "pause",
565        namespace = %namespace,
566        workflow_id = %workflow_id,
567    );
568
569    let handle = async {
570        engine
571            .pause_workflow(&workflow_id, &run_id, reason, None)
572            .await
573            .map_err(|error| map_workflow_operation_error(error, &workflow_id))
574    }
575    .instrument(span)
576    .await?;
577
578    Ok(ProtoPauseResponse {
579        run_id: Some(handle.run_id().clone().into()),
580        // Pause projects Paused regardless of the resident handle's cached status
581        // (which stays Running under the dispatch-hold model).
582        status: aion_proto::ProtoWorkflowStatus::Paused as i32,
583    })
584}
585
586/// Handles a decoded rename request (#211).
587///
588/// Resolves the run (latest when omitted) and calls
589/// [`aion::Engine::rename_workflow`], which records the new display name as a
590/// durable `SearchAttributesUpdated` event — history keeps every name the run
591/// has worn. The name is a LABEL over the UUID identity, never an address:
592/// this request sets a name on an id-addressed run; nothing resolves a
593/// workflow by name. Returns the run id and the name exactly as recorded
594/// (trimmed).
595///
596/// # Errors
597///
598/// Returns a stable [`WireError`] when IDs are missing/malformed, the trimmed
599/// name is empty (`invalid_input`), namespace scoping fails, or the engine
600/// rename call fails — `not_found` for an absent workflow, and `invalid_state`
601/// for a run that is neither terminal nor paused and is not resident on this
602/// node (renaming it then would append behind its live writer, so the engine
603/// refuses for the caller to retry rather than risk a second writer).
604pub async fn rename(
605    guard: &NamespaceGuard,
606    caller: &CallerIdentity,
607    request: aion_proto::ProtoRenameRequest,
608) -> Result<aion_proto::ProtoRenameResponse, WireError> {
609    let workflow_id = required_workflow_id(request.workflow_id.clone())?;
610    // Validate the name BEFORE any engine work: an empty label is a caller
611    // mistake, refused with nothing appended. Trim + empty-filter only (the
612    // task_queue precedent) — no invented caps.
613    let display_name = request.display_name.trim();
614    if display_name.is_empty() {
615        return Err(WireError::invalid_input(
616            "display_name must not be empty; a rename records a non-empty label",
617        ));
618    }
619    let target = WorkflowTarget::workflow(&workflow_id);
620    let scoped = guard
621        .scope(
622            caller,
623            &NamespaceOperation::rename_workflow(&request, target),
624        )
625        .await
626        .map_err(|error| error.to_wire_error())?;
627    let namespace = scoped.namespace().to_owned();
628    let engine = scoped.engine().map_err(|error| error.to_wire_error())?;
629    let run_id = resolve_run_id(engine.as_ref(), &workflow_id, request.run_id.clone()).await?;
630
631    let span = info_span!(
632        "engine_operation",
633        operation = "rename",
634        namespace = %namespace,
635        workflow_id = %workflow_id,
636    );
637
638    let recorded = async {
639        engine
640            .rename_workflow(&workflow_id, &run_id, display_name)
641            .await
642            .map_err(|error| map_workflow_operation_error(error, &workflow_id))
643    }
644    .instrument(span)
645    .await?;
646
647    Ok(aion_proto::ProtoRenameResponse {
648        run_id: Some(run_id.into()),
649        display_name: recorded,
650    })
651}
652
653/// Handles a decoded resume request (#204).
654///
655/// Resolves the run (latest when omitted) and calls
656/// [`aion::Engine::resume_paused_workflow`], returning the run id and its
657/// projected `Running` status.
658///
659/// # Errors
660///
661/// Returns a stable [`WireError`] when IDs are missing/malformed, namespace
662/// scoping fails, or the engine resume call fails — `invalid_state` when the run
663/// is not Paused, `not_found` for an absent workflow.
664pub async fn resume(
665    guard: &NamespaceGuard,
666    caller: &CallerIdentity,
667    request: ProtoResumeRequest,
668) -> Result<ProtoResumeResponse, WireError> {
669    let workflow_id = required_workflow_id(request.workflow_id.clone())?;
670    let target = WorkflowTarget::workflow(&workflow_id);
671    let scoped = guard
672        .scope(
673            caller,
674            &NamespaceOperation::resume_workflow(&request, target),
675        )
676        .await
677        .map_err(|error| error.to_wire_error())?;
678    let namespace = scoped.namespace().to_owned();
679    let engine = scoped.engine().map_err(|error| error.to_wire_error())?;
680    let run_id = resolve_run_id(engine.as_ref(), &workflow_id, request.run_id.clone()).await?;
681
682    let span = info_span!(
683        "engine_operation",
684        operation = "resume",
685        namespace = %namespace,
686        workflow_id = %workflow_id,
687    );
688
689    let handle = async {
690        engine
691            .resume_paused_workflow(&workflow_id, &run_id, None)
692            .await
693            .map_err(|error| map_workflow_operation_error(error, &workflow_id))
694    }
695    .instrument(span)
696    .await?;
697
698    Ok(ProtoResumeResponse {
699        run_id: Some(handle.run_id().clone().into()),
700        status: aion_proto::ProtoWorkflowStatus::Running as i32,
701    })
702}
703
704#[cfg(test)]
705mod tests {
706    use aion_proto::{WireError, WireErrorCode};
707
708    use super::super::test_support::{
709        NAMESPACE, append_completed, append_failed, append_started, append_timed_out,
710        assert_workflow_not_found, cancel_request, context, denied_guard, proto_payload,
711        query_request, reopen_request, run_id, signal_request, workflow_id,
712    };
713    use super::*;
714
715    #[tokio::test]
716    async fn start_handler_scopes_then_invokes_engine_start()
717    -> Result<(), Box<dyn std::error::Error>> {
718        let context = context().await?;
719        let request = ProtoStartWorkflowRequest {
720            namespace: NAMESPACE.to_owned(),
721            workflow_type: "missing-workflow".to_owned(),
722            input: Some(proto_payload()?),
723            routing_key: None,
724            task_queue: None,
725            display_name: None,
726        };
727
728        let error = start(&context.guard, &context.caller, request).await;
729
730        let error = error
731            .err()
732            .ok_or_else(|| WireError::backend("expected error"))?;
733        assert_eq!(error.code, WireErrorCode::NotFound);
734        assert_eq!(error.error_type.as_deref(), Some("WorkflowTypeNotFound"));
735        assert_eq!(
736            error.message,
737            "workflow type missing-workflow is not registered"
738        );
739        Ok(())
740    }
741
742    #[test]
743    fn start_records_namespace_only_when_no_task_queue_selected() {
744        use crate::namespace::{NAMESPACE_ATTRIBUTE, TASK_QUEUE_ATTRIBUTE};
745
746        let attributes = start_search_attributes("tenant-a", None, None);
747        assert_eq!(
748            attributes.get(NAMESPACE_ATTRIBUTE),
749            Some(&aion_core::SearchAttributeValue::String(
750                "tenant-a".to_owned()
751            ))
752        );
753        // No selection => no task_queue attribute is recorded, so the workflow
754        // falls back to the namespace's default queue.
755        assert!(!attributes.contains_key(TASK_QUEUE_ATTRIBUTE));
756    }
757
758    #[test]
759    fn start_records_selected_task_queue_durably_like_namespace() {
760        use crate::namespace::{NAMESPACE_ATTRIBUTE, TASK_QUEUE_ATTRIBUTE};
761
762        let attributes = start_search_attributes("tenant-a", Some("gpu"), None);
763        assert_eq!(
764            attributes.get(NAMESPACE_ATTRIBUTE),
765            Some(&aion_core::SearchAttributeValue::String(
766                "tenant-a".to_owned()
767            ))
768        );
769        // The selected task_queue rides the SAME search-attribute map as the
770        // namespace, so it lands in the same atomic WorkflowStarted append and
771        // survives replay/failover exactly as the namespace does.
772        assert_eq!(
773            attributes.get(TASK_QUEUE_ATTRIBUTE),
774            Some(&aion_core::SearchAttributeValue::String("gpu".to_owned()))
775        );
776    }
777
778    /// #211: an unnamed start records no display-name attribute, so the run
779    /// renders as its bare UUID.
780    #[test]
781    fn start_records_no_display_name_when_unnamed() {
782        use crate::namespace::{DISPLAY_NAME_ATTRIBUTE, NAMESPACE_ATTRIBUTE};
783
784        let attributes = start_search_attributes("tenant-a", None, None);
785        assert_eq!(
786            attributes.get(NAMESPACE_ATTRIBUTE),
787            Some(&aion_core::SearchAttributeValue::String(
788                "tenant-a".to_owned()
789            ))
790        );
791        // No name => no display_name attribute is recorded; the unnamed run
792        // renders as its bare UUID.
793        assert!(!attributes.contains_key(DISPLAY_NAME_ATTRIBUTE));
794    }
795
796    /// #211: a named start records `aion.display_name` in the SAME
797    /// search-attribute map as the namespace, so the label lands in the same
798    /// atomic `WorkflowStarted` append and survives replay/failover.
799    #[test]
800    fn start_records_display_name_durably_like_namespace() {
801        use crate::namespace::{DISPLAY_NAME_ATTRIBUTE, NAMESPACE_ATTRIBUTE};
802
803        let attributes = start_search_attributes("tenant-a", None, Some("Nightly settlement"));
804        assert_eq!(
805            attributes.get(NAMESPACE_ATTRIBUTE),
806            Some(&aion_core::SearchAttributeValue::String(
807                "tenant-a".to_owned()
808            ))
809        );
810        assert_eq!(
811            attributes.get(DISPLAY_NAME_ATTRIBUTE),
812            Some(&aion_core::SearchAttributeValue::String(
813                "Nightly settlement".to_owned()
814            ))
815        );
816    }
817
818    /// #211: a PRESENT but blank `display_name` on START is refused rather than
819    /// reinterpreted as "unnamed" — the server is the trust boundary, and a
820    /// caller who believed they had named the run must not get a 200 and a bare
821    /// UUID. Absent stays the way to say "unnamed".
822    #[tokio::test]
823    async fn start_refuses_a_blank_display_name_rather_than_starting_unnamed()
824    -> Result<(), Box<dyn std::error::Error>> {
825        let context = context().await?;
826        for blank in ["", "   ", "\t\n "] {
827            let request = aion_proto::ProtoStartWorkflowRequest {
828                namespace: NAMESPACE.to_owned(),
829                workflow_type: "checkout".to_owned(),
830                input: Some(proto_payload()?),
831                routing_key: None,
832                task_queue: None,
833                display_name: Some(blank.to_owned()),
834            };
835
836            let error = start(&context.guard, &context.caller, request)
837                .await
838                .err()
839                .ok_or_else(|| WireError::backend("expected a blank-name refusal"))?;
840            assert_eq!(error.code, WireErrorCode::InvalidInput, "blank {blank:?}");
841        }
842
843        // ABSENT is still how a caller says "unnamed", and it is NOT refused —
844        // without this arm the assertion above would also pass if the handler
845        // had started refusing every start.
846        let request = aion_proto::ProtoStartWorkflowRequest {
847            namespace: NAMESPACE.to_owned(),
848            workflow_type: "checkout".to_owned(),
849            input: Some(proto_payload()?),
850            routing_key: None,
851            task_queue: None,
852            display_name: None,
853        };
854        let error = start(&context.guard, &context.caller, request)
855            .await
856            .err()
857            .ok_or_else(|| WireError::backend("expected an error"))?;
858        assert_ne!(
859            error.code,
860            WireErrorCode::InvalidInput,
861            "an absent name must not be refused as invalid input"
862        );
863        Ok(())
864    }
865
866    /// #211: an empty rename is refused as `invalid_input` BEFORE any scoping
867    /// or engine work — a rename records a non-empty label or nothing.
868    #[tokio::test]
869    async fn rename_handler_rejects_blank_display_name() -> Result<(), Box<dyn std::error::Error>> {
870        let context = context().await?;
871        let request = aion_proto::ProtoRenameRequest {
872            namespace: NAMESPACE.to_owned(),
873            workflow_id: Some(workflow_id().into()),
874            run_id: Some(run_id().into()),
875            display_name: "   ".to_owned(),
876        };
877
878        let error = rename(&context.guard, &context.caller, request).await;
879
880        let error = error
881            .err()
882            .ok_or_else(|| WireError::backend("expected error"))?;
883        assert_eq!(error.code, WireErrorCode::InvalidInput);
884        Ok(())
885    }
886
887    /// #211: rename scopes the namespace then resolves the run like
888    /// signal/cancel — an absent workflow is `not_found`, and nothing is
889    /// appended.
890    #[tokio::test]
891    async fn rename_handler_scopes_then_reports_absent_workflow()
892    -> Result<(), Box<dyn std::error::Error>> {
893        let context = context().await?;
894        context.ownership.record(workflow_id(), NAMESPACE)?;
895        let request = aion_proto::ProtoRenameRequest {
896            namespace: NAMESPACE.to_owned(),
897            workflow_id: Some(workflow_id().into()),
898            run_id: Some(run_id().into()),
899            display_name: "Nightly settlement".to_owned(),
900        };
901
902        let error = rename(&context.guard, &context.caller, request).await;
903
904        let error = error
905            .err()
906            .ok_or_else(|| WireError::backend("expected error"))?;
907        assert_eq!(error.code, WireErrorCode::NotFound);
908        assert_eq!(error.error_type.as_deref(), Some("WorkflowNotFound"));
909        Ok(())
910    }
911
912    #[tokio::test]
913    async fn signal_handler_scopes_then_invokes_engine_signal()
914    -> Result<(), Box<dyn std::error::Error>> {
915        let context = context().await?;
916        context.ownership.record(workflow_id(), NAMESPACE)?;
917
918        let error = signal(&context.guard, &context.caller, signal_request()?).await;
919
920        let error = error
921            .err()
922            .ok_or_else(|| WireError::backend("expected error"))?;
923        assert_eq!(error.code, WireErrorCode::NotFound);
924        assert_eq!(error.error_type.as_deref(), Some("WorkflowNotFound"));
925        assert_eq!(
926            error.message,
927            format!("workflow {} not found", workflow_id())
928        );
929        Ok(())
930    }
931
932    #[tokio::test]
933    async fn query_handler_scopes_then_invokes_engine_query()
934    -> Result<(), Box<dyn std::error::Error>> {
935        let context = context().await?;
936        context.ownership.record(workflow_id(), NAMESPACE)?;
937
938        let error = query(&context.guard, &context.caller, query_request()).await;
939
940        let error = error
941            .err()
942            .ok_or_else(|| WireError::backend("expected error"))?;
943        assert_eq!(error.code, WireErrorCode::NotFound);
944        assert_eq!(error.error_type.as_deref(), Some("WorkflowNotFound"));
945        assert_eq!(
946            error.message,
947            format!("workflow {} not found", workflow_id())
948        );
949        Ok(())
950    }
951
952    #[tokio::test]
953    async fn query_handler_returns_not_running_outcome_for_terminal_workflow()
954    -> Result<(), Box<dyn std::error::Error>> {
955        let context = context().await?;
956        context.ownership.record(workflow_id(), NAMESPACE)?;
957        append_completed(context.store.as_ref()).await?;
958        // Resolve the latest run from the chain: the completed history was
959        // recorded for the started run, not the fixed test run id.
960        let mut request = query_request();
961        request.run_id = None;
962
963        let response = query(&context.guard, &context.caller, request).await?;
964
965        // A terminal workflow is a query-semantic outcome: the transport call
966        // succeeds and the typed error rides the QueryResponse.error oneof.
967        let Some(proto_query_response::Outcome::Error(error)) = response.outcome else {
968            return Err("expected a QueryResponse.error outcome".into());
969        };
970        let error = WireError::try_from(error)?;
971        assert_eq!(error.code, WireErrorCode::NotRunning);
972        assert_eq!(error.error_type.as_deref(), Some("QueryNotRunning"));
973        Ok(())
974    }
975
976    #[tokio::test]
977    async fn query_handler_answers_not_running_outcome_for_non_resident_non_terminal_workflow()
978    -> Result<(), Box<dyn std::error::Error>> {
979        // A recorded but non-resident, non-terminal workflow misses the live
980        // registry, yet its run demonstrably exists in the store — so the
981        // engine answers the query-semantic NotRunning, not WorkflowNotFound
982        // (#214: not-found is reserved for a run with no recorded events).
983        // Like the terminal case above, the transport call succeeds and the
984        // typed error rides the QueryResponse.error oneof.
985        let context = context().await?;
986        context.ownership.record(workflow_id(), NAMESPACE)?;
987        append_started(context.store.as_ref()).await?;
988        let mut request = query_request();
989        request.run_id = None;
990
991        let response = query(&context.guard, &context.caller, request).await?;
992
993        let Some(proto_query_response::Outcome::Error(error)) = response.outcome else {
994            return Err("expected a QueryResponse.error outcome".into());
995        };
996        let error = WireError::try_from(error)?;
997        assert_eq!(error.code, WireErrorCode::NotRunning);
998        assert_eq!(error.error_type.as_deref(), Some("QueryNotRunning"));
999        Ok(())
1000    }
1001
1002    #[tokio::test]
1003    async fn cancel_handler_scopes_then_invokes_engine_cancel()
1004    -> Result<(), Box<dyn std::error::Error>> {
1005        let context = context().await?;
1006        context.ownership.record(workflow_id(), NAMESPACE)?;
1007
1008        let error = cancel(
1009            &context.state,
1010            &context.guard,
1011            &context.caller,
1012            cancel_request(),
1013        )
1014        .await;
1015
1016        let error = error
1017            .err()
1018            .ok_or_else(|| WireError::backend("expected error"))?;
1019        assert_eq!(error.code, WireErrorCode::NotFound);
1020        assert_eq!(error.error_type.as_deref(), Some("WorkflowNotFound"));
1021        assert_eq!(
1022            error.message,
1023            format!("workflow {} not found", workflow_id())
1024        );
1025        Ok(())
1026    }
1027
1028    #[tokio::test]
1029    async fn reopen_handler_maps_missing_workflow_to_not_found()
1030    -> Result<(), Box<dyn std::error::Error>> {
1031        let context = context().await?;
1032        context.ownership.record(workflow_id(), NAMESPACE)?;
1033
1034        let error = reopen(&context.guard, &context.caller, reopen_request()).await;
1035
1036        let error = error
1037            .err()
1038            .ok_or_else(|| WireError::backend("expected error"))?;
1039        assert_eq!(error.code, WireErrorCode::NotFound);
1040        assert_eq!(error.error_type.as_deref(), Some("WorkflowNotFound"));
1041        Ok(())
1042    }
1043
1044    #[tokio::test]
1045    async fn reopen_handler_rejects_completed_workflow_as_invalid_state()
1046    -> Result<(), Box<dyn std::error::Error>> {
1047        let context = context().await?;
1048        context.ownership.record(workflow_id(), NAMESPACE)?;
1049        append_completed(context.store.as_ref()).await?;
1050        let mut request = reopen_request();
1051        request.run_id = None;
1052
1053        let error = reopen(&context.guard, &context.caller, request).await;
1054
1055        let error = error
1056            .err()
1057            .ok_or_else(|| WireError::backend("expected error"))?;
1058        assert_eq!(error.code, WireErrorCode::InvalidState);
1059        assert_eq!(error.error_type.as_deref(), Some("InvalidState"));
1060        Ok(())
1061    }
1062
1063    #[tokio::test]
1064    async fn reopen_handler_rejects_timed_out_workflow_as_invalid_state()
1065    -> Result<(), Box<dyn std::error::Error>> {
1066        let context = context().await?;
1067        context.ownership.record(workflow_id(), NAMESPACE)?;
1068        append_timed_out(context.store.as_ref()).await?;
1069        let mut request = reopen_request();
1070        request.run_id = None;
1071
1072        let error = reopen(&context.guard, &context.caller, request).await;
1073
1074        let error = error
1075            .err()
1076            .ok_or_else(|| WireError::backend("expected error"))?;
1077        // TimedOut is a non-reopenable terminal (only Failed and Cancelled reopen).
1078        assert_eq!(error.code, WireErrorCode::InvalidState);
1079        assert_eq!(error.error_type.as_deref(), Some("InvalidState"));
1080        Ok(())
1081    }
1082
1083    #[tokio::test]
1084    async fn reopen_handler_maps_omitted_run_missing_workflow_to_not_found()
1085    -> Result<(), Box<dyn std::error::Error>> {
1086        let context = context().await?;
1087        context.ownership.record(workflow_id(), NAMESPACE)?;
1088        let mut request = reopen_request();
1089        request.run_id = None;
1090
1091        let error = reopen(&context.guard, &context.caller, request).await;
1092
1093        assert_workflow_not_found(error)?;
1094        Ok(())
1095    }
1096
1097    /// A caller WITHOUT a grant for the target namespace is denied reopen with
1098    /// the namespace-denied wire code — mirroring the signal denial test.
1099    #[tokio::test]
1100    async fn denied_reopen_is_namespace_denied_before_engine_check()
1101    -> Result<(), Box<dyn std::error::Error>> {
1102        let (guard, caller) = denied_guard();
1103        let request = ProtoReopenRequest {
1104            namespace: NAMESPACE.to_owned(),
1105            workflow_id: Some(workflow_id().into()),
1106            run_id: Some(run_id().into()),
1107        };
1108
1109        let error = reopen(&guard, &caller, request).await;
1110
1111        assert_eq!(
1112            error.err().map(|error| error.code),
1113            Some(WireErrorCode::NamespaceDenied)
1114        );
1115        Ok(())
1116    }
1117
1118    /// An absent payload is a CLIENT-shaped omission: `invalid_input` (400 /
1119    /// `INVALID_ARGUMENT`) naming the field — never `backend`, which read as
1120    /// "the server broke" (r2-m5). Same for a start with no input.
1121    #[tokio::test]
1122    async fn absent_payloads_are_refused_as_invalid_input_not_backend()
1123    -> Result<(), Box<dyn std::error::Error>> {
1124        let context = context().await?;
1125        context.ownership.record(workflow_id(), NAMESPACE)?;
1126        append_started(context.store.as_ref()).await?;
1127
1128        let mut request = signal_request()?;
1129        request.payload = None;
1130        let error = signal(&context.guard, &context.caller, request)
1131            .await
1132            .err()
1133            .ok_or_else(|| WireError::backend("a signal without a payload must be refused"))?;
1134        assert_eq!(error.code, WireErrorCode::InvalidInput);
1135        assert!(
1136            error.message.contains("payload is required"),
1137            "the refusal names the field: {}",
1138            error.message
1139        );
1140
1141        let request = ProtoStartWorkflowRequest {
1142            namespace: NAMESPACE.to_owned(),
1143            workflow_type: "checkout".to_owned(),
1144            input: None,
1145            routing_key: None,
1146            task_queue: None,
1147            display_name: None,
1148        };
1149        let error = start(&context.guard, &context.caller, request)
1150            .await
1151            .err()
1152            .ok_or_else(|| WireError::backend("a start without input must be refused"))?;
1153        assert_eq!(error.code, WireErrorCode::InvalidInput);
1154        assert!(
1155            error.message.contains("payload is required"),
1156            "{}",
1157            error.message
1158        );
1159        Ok(())
1160    }
1161
1162    #[tokio::test]
1163    async fn signal_handler_rejects_completed_workflow() -> Result<(), Box<dyn std::error::Error>> {
1164        let context = context().await?;
1165        context.ownership.record(workflow_id(), NAMESPACE)?;
1166        append_completed(context.store.as_ref()).await?;
1167
1168        let error = signal(&context.guard, &context.caller, signal_request()?).await;
1169
1170        let error = error
1171            .err()
1172            .ok_or_else(|| WireError::backend("expected error"))?;
1173        assert_eq!(error.code, WireErrorCode::NotRunning);
1174        assert_eq!(error.error_type.as_deref(), Some("WorkflowTerminal"));
1175        assert_eq!(
1176            error.message,
1177            format!(
1178                "workflow {} has already reached terminal state Completed",
1179                workflow_id()
1180            )
1181        );
1182        Ok(())
1183    }
1184
1185    #[tokio::test]
1186    async fn signal_handler_rejects_failed_workflow() -> Result<(), Box<dyn std::error::Error>> {
1187        let context = context().await?;
1188        context.ownership.record(workflow_id(), NAMESPACE)?;
1189        append_failed(context.store.as_ref()).await?;
1190
1191        let error = signal(&context.guard, &context.caller, signal_request()?).await;
1192
1193        let error = error
1194            .err()
1195            .ok_or_else(|| WireError::backend("expected error"))?;
1196        assert_eq!(error.code, WireErrorCode::NotRunning);
1197        assert_eq!(error.error_type.as_deref(), Some("WorkflowTerminal"));
1198        assert_eq!(
1199            error.message,
1200            format!(
1201                "workflow {} has already reached terminal state Failed",
1202                workflow_id()
1203            )
1204        );
1205        Ok(())
1206    }
1207
1208    #[tokio::test]
1209    async fn cancel_handler_rejects_completed_workflow() -> Result<(), Box<dyn std::error::Error>> {
1210        let context = context().await?;
1211        context.ownership.record(workflow_id(), NAMESPACE)?;
1212        append_completed(context.store.as_ref()).await?;
1213
1214        let error = cancel(
1215            &context.state,
1216            &context.guard,
1217            &context.caller,
1218            cancel_request(),
1219        )
1220        .await;
1221
1222        let error = error
1223            .err()
1224            .ok_or_else(|| WireError::backend("expected error"))?;
1225        assert_eq!(error.code, WireErrorCode::NotRunning);
1226        assert_eq!(error.error_type.as_deref(), Some("WorkflowTerminal"));
1227        assert_eq!(
1228            error.message,
1229            format!(
1230                "workflow {} has already completed with status Completed",
1231                workflow_id()
1232            )
1233        );
1234        assert!(!error.message.contains("process 0 is not live"));
1235        Ok(())
1236    }
1237
1238    #[tokio::test]
1239    async fn cancel_handler_rejects_failed_workflow() -> Result<(), Box<dyn std::error::Error>> {
1240        let context = context().await?;
1241        context.ownership.record(workflow_id(), NAMESPACE)?;
1242        append_failed(context.store.as_ref()).await?;
1243
1244        let error = cancel(
1245            &context.state,
1246            &context.guard,
1247            &context.caller,
1248            cancel_request(),
1249        )
1250        .await;
1251
1252        let error = error
1253            .err()
1254            .ok_or_else(|| WireError::backend("expected error"))?;
1255        assert_eq!(error.code, WireErrorCode::NotRunning);
1256        assert_eq!(error.error_type.as_deref(), Some("WorkflowTerminal"));
1257        assert_eq!(
1258            error.message,
1259            format!(
1260                "workflow {} has already completed with status Failed",
1261                workflow_id()
1262            )
1263        );
1264        assert!(!error.message.contains("process 0 is not live"));
1265        Ok(())
1266    }
1267
1268    #[tokio::test]
1269    async fn signal_handler_maps_omitted_run_missing_workflow_to_not_found()
1270    -> Result<(), Box<dyn std::error::Error>> {
1271        let context = context().await?;
1272        context.ownership.record(workflow_id(), NAMESPACE)?;
1273        let mut request = signal_request()?;
1274        request.run_id = None;
1275
1276        let error = signal(&context.guard, &context.caller, request).await;
1277
1278        assert_workflow_not_found(error)?;
1279        Ok(())
1280    }
1281
1282    #[tokio::test]
1283    async fn query_handler_maps_omitted_run_missing_workflow_to_not_found()
1284    -> Result<(), Box<dyn std::error::Error>> {
1285        let context = context().await?;
1286        context.ownership.record(workflow_id(), NAMESPACE)?;
1287        let mut request = query_request();
1288        request.run_id = None;
1289
1290        let error = query(&context.guard, &context.caller, request).await;
1291
1292        assert_workflow_not_found(error)?;
1293        Ok(())
1294    }
1295
1296    #[tokio::test]
1297    async fn cancel_handler_maps_omitted_run_missing_workflow_to_not_found()
1298    -> Result<(), Box<dyn std::error::Error>> {
1299        let context = context().await?;
1300        context.ownership.record(workflow_id(), NAMESPACE)?;
1301        let mut request = cancel_request();
1302        request.run_id = None;
1303
1304        let error = cancel(&context.state, &context.guard, &context.caller, request).await;
1305
1306        assert_workflow_not_found(error)?;
1307        Ok(())
1308    }
1309
1310    #[tokio::test]
1311    async fn denied_start_does_not_decode_missing_payload_before_namespace_check()
1312    -> Result<(), Box<dyn std::error::Error>> {
1313        let (guard, caller) = denied_guard();
1314        let request = ProtoStartWorkflowRequest {
1315            namespace: NAMESPACE.to_owned(),
1316            workflow_type: "fixture".to_owned(),
1317            input: None,
1318            routing_key: None,
1319            task_queue: None,
1320            display_name: None,
1321        };
1322
1323        let error = start(&guard, &caller, request).await;
1324
1325        assert_eq!(
1326            error.err().map(|error| error.code),
1327            Some(WireErrorCode::NamespaceDenied)
1328        );
1329        Ok(())
1330    }
1331
1332    #[tokio::test]
1333    async fn denied_signal_does_not_decode_missing_payload_before_namespace_check()
1334    -> Result<(), Box<dyn std::error::Error>> {
1335        let (guard, caller) = denied_guard();
1336        let request = ProtoSignalRequest {
1337            namespace: NAMESPACE.to_owned(),
1338            workflow_id: Some(workflow_id().into()),
1339            run_id: Some(run_id().into()),
1340            signal_name: "poke".to_owned(),
1341            payload: None,
1342        };
1343
1344        let error = signal(&guard, &caller, request).await;
1345
1346        assert_eq!(
1347            error.err().map(|error| error.code),
1348            Some(WireErrorCode::NamespaceDenied)
1349        );
1350        Ok(())
1351    }
1352
1353    // ---- Minted-on-use START safety net (Control-Plane Phase 1, S6) --------
1354
1355    use std::sync::Arc;
1356
1357    use aion_store::{NamespaceOrigin, NamespaceStore};
1358
1359    use crate::config::AutoCreate;
1360
1361    fn namespace_store() -> Arc<dyn NamespaceStore> {
1362        Arc::new(aion_store::InMemoryStore::default())
1363    }
1364
1365    fn minter(store: &Arc<dyn NamespaceStore>, policy: AutoCreate) -> NamespaceMinter {
1366        NamespaceMinter::new(Arc::clone(store), policy)
1367    }
1368
1369    fn fresh_start_request() -> Result<ProtoStartWorkflowRequest, aion_core::PayloadError> {
1370        Ok(ProtoStartWorkflowRequest {
1371            namespace: NAMESPACE.to_owned(),
1372            workflow_type: "missing-workflow".to_owned(),
1373            input: Some(proto_payload()?),
1374            routing_key: None,
1375            task_queue: None,
1376            display_name: None,
1377        })
1378    }
1379
1380    /// A start into a never-before-seen namespace (no worker registered) mints a
1381    /// durable record under the open policy, even though the start itself fails
1382    /// at the engine (no such workflow type) — the mint runs strictly after
1383    /// authorization and before the engine call. A second start is idempotent:
1384    /// no duplicate row.
1385    #[tokio::test]
1386    async fn open_start_mints_durable_record_and_is_idempotent()
1387    -> Result<(), Box<dyn std::error::Error>> {
1388        let context = context().await?;
1389        let store = namespace_store();
1390        let minter = minter(&store, AutoCreate::Open);
1391
1392        // No worker ever registered, so the namespace has no row yet.
1393        assert!(store.get_namespace(NAMESPACE).await?.is_none());
1394
1395        // The start fails at the engine (unknown workflow type) but the mint
1396        // already ran: a durable record exists afterwards.
1397        let first = start_with_placement(
1398            &context.guard,
1399            &context.caller,
1400            fresh_start_request()?,
1401            None,
1402            Some(&minter),
1403        )
1404        .await;
1405        assert!(
1406            first.is_err(),
1407            "the fixture start has no registered workflow type"
1408        );
1409        let record = store
1410            .get_namespace(NAMESPACE)
1411            .await?
1412            .ok_or("expected a durable record minted by the start")?;
1413        assert_eq!(record.name, NAMESPACE);
1414        assert_eq!(record.origin, NamespaceOrigin::StartMint);
1415
1416        // A second start is idempotent: still exactly one row, no duplicate.
1417        let _second = start_with_placement(
1418            &context.guard,
1419            &context.caller,
1420            fresh_start_request()?,
1421            None,
1422            Some(&minter),
1423        )
1424        .await;
1425        let all = store.list_namespaces().await?;
1426        assert_eq!(
1427            all.iter().filter(|r| r.name == NAMESPACE).count(),
1428            1,
1429            "a second start must not create a duplicate namespace row"
1430        );
1431        Ok(())
1432    }
1433
1434    /// Under the closed policy a start into an unknown namespace is rejected with
1435    /// the same namespace-denied error the worker-registration seam uses, and the
1436    /// namespace is not created.
1437    #[tokio::test]
1438    async fn closed_start_rejects_unknown_namespace_and_does_not_create_it()
1439    -> Result<(), Box<dyn std::error::Error>> {
1440        let context = context().await?;
1441        let store = namespace_store();
1442        let minter = minter(&store, AutoCreate::Closed);
1443
1444        let denied = start_with_placement(
1445            &context.guard,
1446            &context.caller,
1447            fresh_start_request()?,
1448            None,
1449            Some(&minter),
1450        )
1451        .await;
1452
1453        let error = denied
1454            .err()
1455            .ok_or_else(|| WireError::backend("expected a namespace-denied error"))?;
1456        assert_eq!(error.code, WireErrorCode::NamespaceDenied);
1457        assert!(
1458            store.get_namespace(NAMESPACE).await?.is_none(),
1459            "closed policy must NOT create the namespace it rejected"
1460        );
1461        Ok(())
1462    }
1463
1464    /// Under the closed policy a start into a namespace that already has a
1465    /// durable record (the `POST /namespaces` escape hatch's effect) is admitted
1466    /// — it proceeds to the engine exactly as the open path does.
1467    #[tokio::test]
1468    async fn closed_start_admits_a_known_namespace() -> Result<(), Box<dyn std::error::Error>> {
1469        let context = context().await?;
1470        let store = namespace_store();
1471        store
1472            .register_namespace(NAMESPACE, NamespaceOrigin::Explicit)
1473            .await?;
1474        let minter = minter(&store, AutoCreate::Closed);
1475
1476        // The known namespace passes the gate, so the start reaches the engine
1477        // and fails only on the unknown workflow type — never on the namespace.
1478        let error = start_with_placement(
1479            &context.guard,
1480            &context.caller,
1481            fresh_start_request()?,
1482            None,
1483            Some(&minter),
1484        )
1485        .await
1486        .err()
1487        .ok_or_else(|| WireError::backend("expected the fixture workflow-type miss"))?;
1488        assert_eq!(
1489            error.code,
1490            WireErrorCode::NotFound,
1491            "a known namespace must pass the gate and fail only at the engine"
1492        );
1493        assert_eq!(error.error_type.as_deref(), Some("WorkflowTypeNotFound"));
1494        Ok(())
1495    }
1496
1497    /// With no minter installed the start path is byte-identical to before S6:
1498    /// the namespace is never touched and the start reaches the engine as usual.
1499    #[tokio::test]
1500    async fn no_minter_leaves_start_untouched() -> Result<(), Box<dyn std::error::Error>> {
1501        let context = context().await?;
1502        let error = start_with_placement(
1503            &context.guard,
1504            &context.caller,
1505            fresh_start_request()?,
1506            None,
1507            None,
1508        )
1509        .await
1510        .err()
1511        .ok_or_else(|| WireError::backend("expected the fixture workflow-type miss"))?;
1512        assert_eq!(error.code, WireErrorCode::NotFound);
1513        assert_eq!(error.error_type.as_deref(), Some("WorkflowTypeNotFound"));
1514        Ok(())
1515    }
1516}