Skip to main content

aion_client/transport/
embedded.rs

1//! Transport backed by an in-process [`aion::Engine`].
2//!
3//! Event subscriptions honour the same resume/replay-splice contract as the
4//! server's `/events/stream` endpoint, built directly on [`aion::Engine`]
5//! seams (`Engine::subscribe` for the live broadcast, `engine.store()` for
6//! the history snapshot — never a client-held stream over engine internals):
7//!
8//! 1. attach the live broadcast subscription FIRST (time T0);
9//! 2. snapshot recorded history via `engine.store().read_history` (T1 > T0);
10//! 3. validate the cursor against the snapshot head;
11//! 4. splice: replay `[resume_from_seq ..= head]` from the snapshot, then the
12//!    live tail filtered to `seq > head`.
13//!
14//! Gap-free: publish strictly follows durable commit, so every event with
15//! `seq > head` was committed — and therefore broadcast — after T0.
16//! Duplicate-free: the live filter drops every `seq <= head`, so an event
17//! present in both the snapshot and the broadcast is emitted exactly once,
18//! from the snapshot. Engine-side lag is never silent: each
19//! `Err(EventStreamLagged)` item surfaces as `Err(ClientError::Unavailable)`
20//! so the resume loop reconnects with its cursor.
21
22use std::sync::Arc;
23
24use aion_core::Event;
25use async_trait::async_trait;
26use futures::stream::BoxStream;
27use futures::{StreamExt, stream};
28
29use crate::error::ClientError;
30use crate::transport::contract::{SubscriptionAttempt, WorkflowTransport};
31
32/// Transport backed by an in-process [`aion::Engine`].
33pub struct EmbeddedWorkflowTransport {
34    engine: Arc<aion::Engine>,
35}
36
37impl EmbeddedWorkflowTransport {
38    /// Creates an embedded transport for `engine`.
39    #[must_use]
40    pub fn new(engine: Arc<aion::Engine>) -> Self {
41        Self { engine }
42    }
43
44    /// Resolve the target run id: the supplied one, or the latest run from the
45    /// workflow's run chain when omitted (mirrors the server's `resolve_run_id`).
46    async fn resolve_run_id(
47        &self,
48        workflow_id: &aion_core::WorkflowId,
49        run_id: Option<aion_proto::ProtoRunId>,
50    ) -> Result<aion_core::RunId, ClientError> {
51        if let Some(run_id) = run_id {
52            return run_id.try_into().map_err(ClientError::from_wire_error);
53        }
54        let chain = self
55            .engine
56            .store()
57            .read_run_chain(workflow_id)
58            .await
59            .map_err(|error| store_error_class(&error, error.to_string()))?;
60        chain
61            .last()
62            .map(|summary| summary.run_id.clone())
63            .ok_or_else(|| ClientError::not_found(format!("workflow {workflow_id} not found")))
64    }
65}
66
67#[async_trait]
68impl WorkflowTransport for EmbeddedWorkflowTransport {
69    async fn start_workflow(
70        &self,
71        request: aion_proto::ProtoStartWorkflowRequest,
72    ) -> Result<aion_proto::ProtoStartWorkflowResponse, ClientError> {
73        let input = request
74            .input
75            .ok_or_else(|| ClientError::invalid_argument("start request input payload is missing"))
76            .and_then(|payload| {
77                aion_core::Payload::try_from(payload).map_err(ClientError::from_wire_error)
78            })?;
79        // The embedded engine is single-tenant and in-process: there is no
80        // namespace authority stamping visibility attributes, so the start
81        // carries no search attributes.
82        let handle = self
83            .engine
84            .start_workflow(
85                &request.workflow_type,
86                input,
87                std::collections::HashMap::new(),
88                String::from("default"),
89            )
90            .await
91            .map_err(|error| map_engine_error(&error))?;
92        Ok(aion_proto::ProtoStartWorkflowResponse {
93            workflow_id: Some(aion_proto::ProtoWorkflowId::from(
94                handle.workflow_id().clone(),
95            )),
96            run_id: Some(aion_proto::ProtoRunId::from(handle.run_id().clone())),
97        })
98    }
99
100    async fn signal(
101        &self,
102        request: aion_proto::ProtoSignalRequest,
103    ) -> Result<aion_proto::ProtoSignalResponse, ClientError> {
104        let workflow_id = decode_required_workflow_id(request.workflow_id)?;
105        let run_id = decode_required_run_id(request.run_id)?;
106        let payload = request
107            .payload
108            .ok_or_else(|| ClientError::invalid_argument("signal request payload is missing"))
109            .and_then(|payload| {
110                aion_core::Payload::try_from(payload).map_err(ClientError::from_wire_error)
111            })?;
112        self.engine
113            .signal(&workflow_id, &run_id, request.signal_name, payload)
114            .await
115            .map_err(|error| map_engine_error(&error))?;
116        Ok(aion_proto::ProtoSignalResponse {})
117    }
118
119    async fn query(
120        &self,
121        request: aion_proto::ProtoQueryRequest,
122    ) -> Result<aion_proto::ProtoQueryResponse, ClientError> {
123        let workflow_id = decode_required_workflow_id(request.workflow_id)?;
124        let run_id = decode_required_run_id(request.run_id)?;
125        // Absent arguments mean the caller supplied none; the handler still
126        // receives one well-formed document.
127        let arguments = request.arguments.map_or_else(
128            || Ok(aion_core::Payload::json_null()),
129            |arguments| {
130                aion_core::Payload::try_from(arguments).map_err(ClientError::from_wire_error)
131            },
132        )?;
133        let payload = self
134            .engine
135            .query(&workflow_id, &run_id, request.query_name, arguments)
136            .await
137            .map_err(|error| map_engine_error(&error))?;
138        Ok(aion_proto::ProtoQueryResponse {
139            outcome: Some(aion_proto::proto_query_response::Outcome::Result(
140                aion_proto::ProtoPayload::from(payload),
141            )),
142        })
143    }
144
145    async fn cancel(
146        &self,
147        request: aion_proto::ProtoCancelRequest,
148    ) -> Result<aion_proto::ProtoCancelResponse, ClientError> {
149        let workflow_id = decode_required_workflow_id(request.workflow_id)?;
150        let run_id = decode_required_run_id(request.run_id)?;
151        self.engine
152            .cancel(&workflow_id, &run_id, request.reason)
153            .await
154            .map_err(|error| map_engine_error(&error))?;
155        Ok(aion_proto::ProtoCancelResponse {})
156    }
157
158    async fn reopen(
159        &self,
160        request: aion_proto::ProtoReopenRequest,
161    ) -> Result<aion_proto::ProtoReopenResponse, ClientError> {
162        let workflow_id = decode_required_workflow_id(request.workflow_id)?;
163        let run_id = self.resolve_run_id(&workflow_id, request.run_id).await?;
164        let handle = self
165            .engine
166            .reopen_workflow(&workflow_id, &run_id)
167            .await
168            .map_err(|error| map_engine_error(&error))?;
169        Ok(aion_proto::ProtoReopenResponse {
170            run_id: Some(handle.run_id().clone().into()),
171            status: aion_proto::ProtoWorkflowStatus::from(handle.cached_status()) as i32,
172        })
173    }
174
175    async fn pause(
176        &self,
177        request: aion_proto::ProtoPauseRequest,
178    ) -> Result<aion_proto::ProtoPauseResponse, ClientError> {
179        let workflow_id = decode_required_workflow_id(request.workflow_id)?;
180        let run_id = self.resolve_run_id(&workflow_id, request.run_id).await?;
181        let reason = if request.reason.is_empty() {
182            None
183        } else {
184            Some(request.reason)
185        };
186        let handle = self
187            .engine
188            .pause_workflow(&workflow_id, &run_id, reason, None)
189            .await
190            .map_err(|error| map_engine_error(&error))?;
191        Ok(aion_proto::ProtoPauseResponse {
192            run_id: Some(handle.run_id().clone().into()),
193            status: aion_proto::ProtoWorkflowStatus::Paused as i32,
194        })
195    }
196
197    async fn resume(
198        &self,
199        request: aion_proto::ProtoResumeRequest,
200    ) -> Result<aion_proto::ProtoResumeResponse, ClientError> {
201        let workflow_id = decode_required_workflow_id(request.workflow_id)?;
202        let run_id = self.resolve_run_id(&workflow_id, request.run_id).await?;
203        let handle = self
204            .engine
205            .resume_paused_workflow(&workflow_id, &run_id, None)
206            .await
207            .map_err(|error| map_engine_error(&error))?;
208        Ok(aion_proto::ProtoResumeResponse {
209            run_id: Some(handle.run_id().clone().into()),
210            status: aion_proto::ProtoWorkflowStatus::Running as i32,
211        })
212    }
213
214    async fn list_workflows(
215        &self,
216        request: aion_proto::ProtoListWorkflowsRequest,
217    ) -> Result<aion_proto::ProtoListWorkflowsResponse, ClientError> {
218        let filter = match request.filter.as_ref() {
219            Some(filter) => {
220                aion_proto::decode_workflow_filter(filter).map_err(ClientError::from_wire_error)?
221            }
222            None => aion_core::WorkflowFilter::default(),
223        };
224        let summaries = self
225            .engine
226            .list_workflows(filter)
227            .await
228            .map_err(|error| map_engine_error(&error))?
229            .iter()
230            .map(|summary| {
231                aion_proto::encode_workflow_summary(request.namespace.clone(), None, summary)
232            })
233            .map(|result| result.map_err(ClientError::from_wire_error))
234            .collect::<Result<Vec<_>, _>>()?;
235        Ok(aion_proto::ProtoListWorkflowsResponse { summaries })
236    }
237
238    async fn describe_workflow(
239        &self,
240        request: aion_proto::ProtoDescribeWorkflowRequest,
241    ) -> Result<aion_proto::ProtoDescribeWorkflowResponse, ClientError> {
242        let workflow_id = decode_required_workflow_id(request.workflow_id)?;
243        let run_id = self.resolve_run_id(&workflow_id, request.run_id).await?;
244        let history = self
245            .engine
246            .store()
247            .read_history(&workflow_id)
248            .await
249            // This read goes straight to the store, so the refusal arrives with
250            // no `EngineError` around it — but it is the same refusal, and a
251            // `NotOwner` here means exactly what it means everywhere else: ask a
252            // different owner. Collapsing it into `server` told the caller its
253            // request was unanswerable when a re-route would have served it.
254            .map_err(|error| store_error_class(&error, error.to_string()))?;
255        let Some(summary) = aion_core::WorkflowSummary::from_history(&history) else {
256            return Err(ClientError::not_found(format!(
257                "workflow {workflow_id} has no recorded history"
258            )));
259        };
260        let summary = Some(
261            aion_proto::encode_workflow_summary(request.namespace.clone(), None, &summary)
262                .map_err(ClientError::from_wire_error)?,
263        );
264        let history_head_seq = history.last().map_or(0, aion_core::Event::seq);
265        let terminal_event = aion_core::current_lease_terminal(&history)
266            .map(|event| aion_proto::encode_event(request.namespace.clone(), None, event))
267            .transpose()
268            .map_err(ClientError::from_wire_error)?;
269        let history = if request.include_history {
270            history
271                .iter()
272                .map(|event| aion_proto::encode_event(request.namespace.clone(), None, event))
273                .map(|result| result.map_err(ClientError::from_wire_error))
274                .collect::<Result<Vec<_>, _>>()?
275        } else {
276            Vec::new()
277        };
278        Ok(aion_proto::ProtoDescribeWorkflowResponse {
279            summary,
280            history,
281            run_id: Some(run_id.into()),
282            history_head_seq,
283            terminal_event,
284        })
285    }
286
287    async fn read_history(
288        &self,
289        request: aion_proto::ProtoReadHistoryRequest,
290    ) -> Result<aion_proto::ProtoReadHistoryResponse, ClientError> {
291        let workflow_id = decode_required_workflow_id(request.workflow_id)?;
292        let from_seq = request.from_seq.unwrap_or(0);
293        let mut events = self
294            .engine
295            .store()
296            .read_history_from(&workflow_id, from_seq)
297            .await
298            .map_err(|error| store_error_class(&error, error.to_string()))?;
299        let head_seq = events.last().map_or(0, aion_core::Event::seq);
300        let next_from_seq = request
301            .limit
302            .and_then(|limit| events.get(limit as usize).map(aion_core::Event::seq));
303        if let Some(limit) = request.limit {
304            events.truncate(limit as usize);
305        }
306        let events = events
307            .iter()
308            .map(|event| aion_proto::encode_event(request.namespace.clone(), None, event))
309            .map(|result| result.map_err(ClientError::from_wire_error))
310            .collect::<Result<Vec<_>, _>>()?;
311        Ok(aion_proto::ProtoReadHistoryResponse {
312            events,
313            next_from_seq,
314            head_seq,
315        })
316    }
317
318    async fn subscribe(
319        &self,
320        request: aion_proto::SubscriptionRequest,
321        resume_from_sequence: Option<u64>,
322    ) -> Result<SubscriptionAttempt, ClientError> {
323        let (workflow_target, filter) = embedded_subscription_target(request)?;
324        // T0: attach to the live broadcast BEFORE any history snapshot — one
325        // half of the gap-free splice proof (mirrors the server's
326        // subscribe-then-snapshot ordering).
327        let live = self.engine.subscribe(filter);
328        let events = match (&workflow_target, resume_from_sequence) {
329            (Some(workflow_id), Some(resume_from_seq)) => {
330                // T1 (> T0): snapshot recorded history, then validate the
331                // cursor against its head and build the dedupe splice.
332                let history = self
333                    .engine
334                    .store()
335                    .read_history(workflow_id)
336                    .await
337                    .map_err(|error| store_error_class(&error, error.to_string()))?;
338                splice_resume(live, history, resume_from_seq)?
339            }
340            (None, Some(_)) => {
341                return Err(ClientError::invalid_argument(
342                    "filtered and firehose event streams are live-only by design; resume \
343                     cursors are valid for per-workflow subscriptions only",
344                ));
345            }
346            (_, None) => map_lag(live),
347        };
348        // Per-workflow streams end at the run's terminal event, exactly like
349        // the server socket; callers walk continue-as-new chains by
350        // resubscribing with their cursor.
351        Ok(SubscriptionAttempt::new(match workflow_target {
352            Some(_) => close_after_terminal(events),
353            None => events,
354        }))
355    }
356}
357
358/// Validates a resume cursor against a history snapshot and builds the
359/// replay/live splice (see the module docs for the gap/duplicate proof).
360fn splice_resume(
361    live: BoxStream<'static, Result<Event, aion::EventStreamLagged>>,
362    history: Vec<Event>,
363    resume_from_seq: u64,
364) -> Result<BoxStream<'static, Result<Event, ClientError>>, ClientError> {
365    if resume_from_seq == 0 {
366        return Err(ClientError::invalid_argument(
367            "resume_from_seq must be >= 1 (the first sequence number wanted)",
368        ));
369    }
370    let head = history.last().map_or(0, Event::seq);
371    if resume_from_seq > head.saturating_add(1) {
372        return Err(ClientError::invalid_argument(format!(
373            "resume_from_seq {resume_from_seq} is ahead of recorded history (head seq {head}); \
374             the largest valid cursor is {}",
375            head.saturating_add(1)
376        )));
377    }
378
379    let mut history = history;
380    let replay_start = history.partition_point(|event| event.seq() < resume_from_seq);
381    let replay = history.split_off(replay_start);
382    let tail = live.filter(move |item| {
383        let keep = match item {
384            Ok(event) => event.seq() > head,
385            // Lag is information, never filtered away.
386            Err(aion::EventStreamLagged { .. }) => true,
387        };
388        futures::future::ready(keep)
389    });
390
391    Ok(stream::iter(replay.into_iter().map(Ok))
392        .chain(map_lag(tail.boxed()))
393        .boxed())
394}
395
396/// Maps engine-side lag items to retryable [`ClientError::Unavailable`] so
397/// the resume loop reconnects with its cursor instead of silently gapping.
398fn map_lag(
399    live: BoxStream<'static, Result<Event, aion::EventStreamLagged>>,
400) -> BoxStream<'static, Result<Event, ClientError>> {
401    live.map(|item| {
402        item.map_err(|lagged| {
403            ClientError::from_wire_error(aion_proto::WireError::lagged(lagged.to_string()))
404        })
405    })
406    .boxed()
407}
408
409/// Ends the stream after the first terminal workflow event, mirroring the
410/// server socket's per-workflow run-boundary close.
411fn close_after_terminal(
412    events: BoxStream<'static, Result<Event, ClientError>>,
413) -> BoxStream<'static, Result<Event, ClientError>> {
414    stream::unfold(Some(events), |state| async move {
415        let mut events = state?;
416        let item = events.next().await?;
417        // The terminal event is delivered and the inner stream is dropped
418        // immediately afterwards (releasing the broadcast receiver), so the
419        // close is eager — it never waits for a further event to be polled.
420        let closed = matches!(&item, Ok(event) if is_terminal_workflow_event(event));
421        Some((item, if closed { None } else { Some(events) }))
422    })
423    .boxed()
424}
425
426fn is_terminal_workflow_event(event: &Event) -> bool {
427    matches!(
428        event,
429        Event::WorkflowCompleted { .. }
430            | Event::WorkflowFailed { .. }
431            | Event::WorkflowCancelled { .. }
432            | Event::WorkflowTimedOut { .. }
433            | Event::WorkflowContinuedAsNew { .. }
434    )
435}
436
437fn decode_required_workflow_id(
438    value: Option<aion_proto::ProtoWorkflowId>,
439) -> Result<aion_core::WorkflowId, ClientError> {
440    value
441        .ok_or_else(|| ClientError::invalid_argument("request workflow id is missing"))?
442        .try_into()
443        .map_err(ClientError::from_wire_error)
444}
445
446fn decode_required_run_id(
447    value: Option<aion_proto::ProtoRunId>,
448) -> Result<aion_core::RunId, ClientError> {
449    value
450        .ok_or_else(|| ClientError::invalid_argument("request run id is missing"))?
451        .try_into()
452        .map_err(ClientError::from_wire_error)
453}
454
455/// Maps a wire subscription request onto the engine filter surface plus the
456/// per-workflow target the splice and run-boundary close key on.
457fn embedded_subscription_target(
458    request: aion_proto::SubscriptionRequest,
459) -> Result<(Option<aion_core::WorkflowId>, aion::EventFilter), ClientError> {
460    match request.subscription {
461        Some(aion_proto::subscription_request::Subscription::PerWorkflow(subscription)) => {
462            let workflow_id = subscription
463                .workflow_id
464                .ok_or_else(|| {
465                    ClientError::invalid_argument(
466                        "per-workflow subscription requires a workflow id",
467                    )
468                })?
469                .try_into()
470                .map_err(ClientError::from_wire_error)?;
471            Ok((
472                Some(aion_core::WorkflowId::clone(&workflow_id)),
473                aion::EventFilter {
474                    workflow_id: Some(workflow_id),
475                    run: None,
476                    family: None,
477                },
478            ))
479        }
480        Some(
481            aion_proto::subscription_request::Subscription::Filtered(_)
482            | aion_proto::subscription_request::Subscription::Firehose(_),
483        ) => Ok((None, aion::EventFilter::default())),
484        Some(aion_proto::subscription_request::Subscription::Cluster(_)) => {
485            // The WS3 cluster topology/ownership channel is a server-side
486            // projection of distributed cluster state. The embedded in-process
487            // transport drives a single local engine with no cluster topology to
488            // project, so a cluster subscription is not serviceable here; reject it
489            // cleanly rather than silently degrading to a workflow event stream.
490            Err(ClientError::invalid_argument(
491                "cluster topology subscriptions are not supported by the embedded in-process \
492                 transport; connect to an aion-server over gRPC/WebSocket to subscribe to the \
493                 cluster channel",
494            ))
495        }
496        Some(aion_proto::subscription_request::Subscription::Transcript(_)) => {
497            // The NOI-5b agent-observability transcript channel is a server-side
498            // projection over the durable `O` keyspace + the server's transcript
499            // sequencer. The embedded in-process transport has no such server
500            // bridge, so a transcript subscription is not serviceable here; reject
501            // it cleanly rather than degrading to a workflow event stream.
502            Err(ClientError::invalid_argument(
503                "agent-observability transcript subscriptions are not supported by the embedded \
504                 in-process transport; connect to an aion-server over gRPC/WebSocket to subscribe \
505                 to the transcript channel",
506            ))
507        }
508        None => Err(ClientError::invalid_argument(
509            "subscription request is missing its subscription variant",
510        )),
511    }
512}
513
514/// Translate an engine failure into the client-facing error class.
515///
516/// # The `_` arm is a family default, and it is not free
517///
518/// `EngineError` has 51 variants and is not `#[non_exhaustive]`, so an
519/// exhaustive match here *would* compile-fail on every new variant. That is
520/// deliberately not what this does: a 51-arm list in a transport adapter is a
521/// list nothing reads, and it would go stale as silently as the wildcard does.
522/// The wildcard is chosen, with its cost stated — **a new variant lands in the
523/// generic server bucket and the compiler will not say so.**
524///
525/// What makes that acceptable is the direction it fails in. `server` is the
526/// "something went wrong, this is not your request's fault and not a wait"
527/// class; landing there is uninformative, never *wrong* in a way that makes a
528/// caller retry something unretryable or give up on something transient. Every
529/// arm above it exists because its variant would be actively mis-served by that
530/// default — a not-found retried forever, a shutdown reported as an engine bug.
531///
532/// So the rule for adding an arm is not "is this variant new" but "would the
533/// generic bucket mislead a caller about what to DO". Only those get named.
534fn map_engine_error(error: &aion::EngineError) -> ClientError {
535    match error {
536        aion::EngineError::WorkflowNotFound { .. } => ClientError::not_found(error.to_string()),
537        // Reopen precondition failure (AD-012): distinct typed variant, never
538        // conflated with not-found or the generic server bucket.
539        aion::EngineError::InvalidState { .. } => ClientError::invalid_state(error.to_string()),
540        // The caller's own payload did not satisfy the contract the target
541        // package declares. Both are raised BEFORE anything is recorded, so
542        // there is nothing to clean up and nothing to retry — the remedy is to
543        // send different bytes. `server` would say the opposite: "not your
544        // fault, try again", which is how a caller ends up retrying a payload
545        // that can never be accepted. `aion-server` classifies these two
546        // through `declared_contract_wire` for the same reason; this is the
547        // embedded surface reaching the same verdict, not a second opinion.
548        aion::EngineError::StartInputRefused { .. } | aion::EngineError::SignalRefused { .. } => {
549            ClientError::invalid_argument(error.to_string())
550        }
551        // Six refusals about the state of a run, or of the package a run would
552        // start from, rather than the health of the engine — all reachable
553        // through the embedded surface, all classified `invalid_state` by
554        // `aion-server`, and all erased into "engine bug" by the generic
555        // bucket, which is the wrong thing to say about any of them.
556        //
557        // Two are transient by construction (`TerminalWriterUnavailable`,
558        // `TerminalWriterHeld`): a writer reservation lives across one terminal
559        // transition. Two name a precondition the caller can act on: redeploy
560        // and cancel through the ordinary path (`RunIsRecoverable`), or address
561        // this engine's missing startup verdict (`NoResidencyVerdict`). The
562        // last two are admission refusals raised before the run starts — the
563        // package's declared contract does not identify what the request names
564        // (`ContractIdentity`), or it declares no task queue to serve from
565        // (`NoQueueDeclaration`) — and the remedy for both is a redeploy.
566        aion::EngineError::TerminalWriterUnavailable { .. }
567        | aion::EngineError::TerminalWriterHeld { .. }
568        | aion::EngineError::RunIsRecoverable { .. }
569        | aion::EngineError::NoResidencyVerdict { .. }
570        | aion::EngineError::ContractIdentity { .. }
571        | aion::EngineError::NoQueueDeclaration { .. } => {
572            ClientError::invalid_state(error.to_string())
573        }
574        // 🔴 BOTH SHAPES A STORE REFUSAL ARRIVES IN, CLASSIFIED IN ONE PLACE.
575        //
576        // An earlier revision matched only the bare `Store(..)` shape and its
577        // test hand-built that shape, so the arm looked covered and fired on
578        // nothing a caller could provoke: every durable write these operations
579        // trigger goes through the `Recorder`, which returns `DurabilityError`,
580        // and the engine wraps that as `EngineError::Durability` — so the
581        // production shape of a lost-ownership refusal is
582        // `Durability(Store(NotOwner))`. Both shapes now route through
583        // `store_error_class`, which is the only place the rule is written.
584        aion::EngineError::Store(store)
585        | aion::EngineError::Durability(aion::durability::DurabilityError::Store(store)) => {
586            store_error_class(store, error.to_string())
587        }
588        // Live-query dispatch. `aion-server` gives this family five distinct
589        // wire codes and the caller taxonomy has a class for each; collapsing
590        // them into `server` left `unknown_query`, `query_timeout`,
591        // `query_failed` and the query half of `not_running` unreachable
592        // in-process — four classes that exist solely for this operation. The
593        // same caller code branches correctly over the wire and could not
594        // branch at all here.
595        aion::EngineError::Query(query) => query_error_class(query, error),
596        // This engine has stopped serving. `aion-server`'s `wire_from_engine`
597        // (`error.rs:645`) answers `not_running`, and an over-the-wire caller of
598        // these operations therefore sees `ClientError::NotRunning`; an earlier
599        // revision of this arm said `unavailable`, which is the transport-level
600        // class the SDK also raises for a dial failure. That told an in-process
601        // caller its ENDPOINT was unreachable when what had happened was that
602        // the engine it holds is shutting down.
603        //
604        // ⚠️ "Mirror the server" is not one answer — the server gives two, and
605        // saying otherwise would hide a real divergence. `wire_from_engine` is
606        // the mapping that governs the eight engine operations THIS transport
607        // exposes, and it is the one mirrored here. Two control-plane handlers
608        // answer `Unavailable` for the same variant —
609        // `aion-server/src/authoring/handlers.rs:404-406` and
610        // `api/handlers/deploy.rs:415` — and both serve deploy/authoring
611        // operations that this transport does not expose, so neither is
612        // reachable through a caller holding an `EmbeddedWorkflowTransport`.
613        // Recorded rather than reconciled: a shared classifier is what would
614        // actually force the two into agreement, and that needs a crate both
615        // sides can depend on.
616        //
617        // `EngineTaskEpochClosed` deliberately has no arm. It is constructed at
618        // exactly one site — the terminal-append boundary in the engine's
619        // process-exit path — which is reached by the completion monitor, never
620        // by any of the operations this transport exposes. An arm for it would
621        // be a classification nobody can observe, and a wrong claim about what
622        // this surface can return.
623        // 🔴 CARRY THE DISCRIMINATOR, because the class alone is a false hint.
624        // `not_running` is the right CLASS and mirrors `wire_from_engine` — but
625        // the operator-facing hint for a bare `not_running` says the run is no
626        // longer running and points at `aion list --status running`, which is
627        // false twice here: the run is fine, it is the ENGINE that stopped
628        // serving, and `aion list` would fail identically. That is the same
629        // defect this lane's F4 fix cited when it moved `EngineTaskEpochClosed`
630        // OFF `not_running`; leaving the discriminator behind would have this
631        // lane arguing both sides of one rule.
632        //
633        // The wire surface already carries it — `wire_from_engine` builds
634        // `not_running_with_type("ShuttingDown", …)` — so attaching it here
635        // makes the embedded and wire surfaces agree rather than diverge, and
636        // gives `render.rs` the one fact it needs to say something true.
637        aion::EngineError::ShuttingDown => ClientError::NotRunning {
638            detail: crate::ErrorDetail::with_type(error.to_string(), "ShuttingDown"),
639        },
640        _ => ClientError::server(error.to_string()),
641    }
642}
643
644/// The class a store-layer refusal carries, whichever shape it reached
645/// [`map_engine_error`] in.
646///
647/// Mirrors `aion-server`'s `wire_from_store` composed with
648/// [`ClientError::from_wire_error`], so the same underlying refusal names the
649/// same class whether the caller is in-process or across the wire.
650///
651/// `not_owner` is the one class that names a ROUTING failure — the endpoint
652/// answered, it simply does not own this target's shard, so the remedy is a
653/// DIFFERENT owner rather than a retry against this one. Collapsing it into
654/// `server` costs the caller that distinction entirely: `server` says the
655/// request is unanswerable, and a caller that believes it abandons work a
656/// re-route would have served.
657///
658/// ⚠️ Nothing in this crate retries it AUTOMATICALLY, and an earlier revision of
659/// this comment claimed otherwise. `aion-client`'s only retry classifier is
660/// `stream.rs`'s `is_retryable`, which is `matches!(error, ClientError::
661/// Unavailable { .. })` — `NotOwner` is not in it, so `ResumingEventStream`
662/// terminates the stream on one. The distinction this arm preserves is
663/// therefore one the CALLER acts on, and the surface that already does is the
664/// CLI's operator hint (`aion-cli/src/render.rs:175-178`), which tells the
665/// operator to retry or point `--endpoint` at another node instead of sending
666/// them hunting a network fault that does not exist.
667///
668/// Takes the caller-facing `message` separately rather than deriving it, because
669/// a store refusal reaches this transport in three shapes and only two of them
670/// have an `EngineError` to render: wrapped as `EngineError::Store`, wrapped as
671/// `EngineError::Durability(Store(..))`, and — at the three call sites that read
672/// the store directly rather than through an engine API — as a bare
673/// [`aion_store::StoreError`] with no wrapper at all. The rule is written once
674/// and all three shapes are held to it.
675fn store_error_class(error: &aion_store::StoreError, message: String) -> ClientError {
676    match error {
677        aion_store::StoreError::NotOwner { .. } => ClientError::not_owner(message),
678        aion_store::StoreError::NotFound { .. } => ClientError::not_found(message),
679        // A `SequenceConflict` is this codebase's single-writer invariant
680        // violation — a double-writer bug, not a caller mistake and not an
681        // idempotency conflict — and `Backend`/`Serialization` are
682        // infrastructure faults. All three are the engine's to answer for, and
683        // `from_wire_error` puts all three in the generic bucket for a caller
684        // arriving over the wire.
685        aion_store::StoreError::SequenceConflict { .. }
686        | aion_store::StoreError::Backend(_)
687        | aion_store::StoreError::Serialization(_) => ClientError::server(message),
688    }
689}
690
691/// The class a live-query dispatch failure carries.
692///
693/// Mirrors `aion-server`'s `query_wire` composed with
694/// [`ClientError::from_wire_error`]. Every arm is a distinct remedy: name a
695/// query the workflow declares, wait or retry, address a run that is not
696/// answering, look up a workflow that does not exist, or fix the handler.
697fn query_error_class(error: &aion::QueryError, source: &aion::EngineError) -> ClientError {
698    match error {
699        aion::QueryError::UnknownQuery(_) => ClientError::unknown_query(source.to_string()),
700        aion::QueryError::Timeout => ClientError::query_timeout(source.to_string()),
701        // A run that cannot answer and a reply channel that closed are the same
702        // fact to a caller: the workflow is not there to answer right now.
703        aion::QueryError::NotRunning(_) | aion::QueryError::ReplyDropped => {
704            ClientError::not_running(source.to_string())
705        }
706        aion::QueryError::Unknown(_) => ClientError::not_found(source.to_string()),
707        // The handler ran and reported an application-level failure — the
708        // workflow author's to fix, and distinct from the engine failing to
709        // deliver the query at all.
710        aion::QueryError::HandlerFailed { .. } => ClientError::query_failed(source.to_string()),
711        // The caller sent arguments the engine cannot carry to a handler: the
712        // caller's request to fix, never the workflow's or the server's.
713        aion::QueryError::InvalidArguments { .. } => {
714            ClientError::invalid_argument(source.to_string())
715        }
716        aion::QueryError::Engine(_) => ClientError::server(source.to_string()),
717    }
718}
719
720#[cfg(test)]
721mod tests {
722    use std::num::NonZeroUsize;
723    use std::time::Duration;
724
725    use aion::EventStreamLagged;
726    use aion_core::{Event, EventEnvelope, Payload, RunId, WorkflowId};
727    use chrono::Utc;
728    use futures::{StreamExt, stream};
729
730    use super::{close_after_terminal, map_lag, splice_resume};
731    use crate::error::ClientError;
732
733    fn workflow_id() -> WorkflowId {
734        WorkflowId::new(uuid::Uuid::from_u128(1))
735    }
736
737    fn envelope(seq: u64) -> EventEnvelope {
738        EventEnvelope {
739            seq,
740            recorded_at: Utc::now(),
741            workflow_id: workflow_id(),
742        }
743    }
744
745    fn signal(seq: u64) -> Result<Event, aion_core::PayloadError> {
746        Ok(Event::SignalReceived {
747            envelope: envelope(seq),
748            name: format!("signal-{seq}"),
749            payload: Payload::from_json(&serde_json::json!({ "seq": seq }))?,
750        })
751    }
752
753    fn completed(seq: u64) -> Result<Event, aion_core::PayloadError> {
754        Ok(Event::WorkflowCompleted {
755            envelope: envelope(seq),
756            result: Payload::from_json(&serde_json::json!({ "seq": seq }))?,
757        })
758    }
759
760    fn history(seqs: std::ops::RangeInclusive<u64>) -> Result<Vec<Event>, aion_core::PayloadError> {
761        seqs.map(signal).collect()
762    }
763
764    fn live(
765        items: Vec<Result<Event, EventStreamLagged>>,
766    ) -> futures::stream::BoxStream<'static, Result<Event, EventStreamLagged>> {
767        stream::iter(items).boxed()
768    }
769
770    async fn delivered_seqs(
771        events: futures::stream::BoxStream<'static, Result<Event, ClientError>>,
772    ) -> Result<Vec<u64>, ClientError> {
773        events
774            .map(|item| item.map(|event| event.seq()))
775            .collect::<Vec<_>>()
776            .await
777            .into_iter()
778            .collect()
779    }
780
781    #[tokio::test]
782    async fn cursor_zero_is_invalid_argument() -> Result<(), Box<dyn std::error::Error>> {
783        let error = splice_resume(live(Vec::new()), history(1..=3)?, 0).err();
784
785        let Some(ClientError::InvalidArgument { detail }) = error else {
786            return Err(format!("cursor 0 must be InvalidArgument, got {error:?}").into());
787        };
788        assert!(detail.message.contains(">= 1"), "detail: {detail}");
789        Ok(())
790    }
791
792    #[tokio::test]
793    async fn cursor_ahead_of_history_is_invalid_argument() -> Result<(), Box<dyn std::error::Error>>
794    {
795        let error = splice_resume(live(Vec::new()), history(1..=5)?, 7).err();
796
797        let Some(ClientError::InvalidArgument { detail }) = error else {
798            return Err(format!("cursor head+2 must be InvalidArgument, got {error:?}").into());
799        };
800        assert!(
801            detail.message.contains("ahead of recorded history"),
802            "{detail}"
803        );
804
805        let empty = splice_resume(live(Vec::new()), Vec::new(), 2).err();
806        assert!(
807            matches!(empty, Some(ClientError::InvalidArgument { .. })),
808            "cursor 2 over empty history must be rejected, got {empty:?}"
809        );
810        Ok(())
811    }
812
813    #[tokio::test]
814    async fn overlap_between_snapshot_and_live_is_deduplicated_contiguous_unique()
815    -> Result<(), Box<dyn std::error::Error>> {
816        // Snapshot holds 1..=5; the live broadcast re-emits 4 and 5 (arrived
817        // between attach and snapshot) before the genuinely new 6.
818        let events = splice_resume(
819            live(vec![Ok(signal(4)?), Ok(signal(5)?), Ok(signal(6)?)]),
820            history(1..=5)?,
821            1,
822        )?;
823
824        assert_eq!(delivered_seqs(events).await?, vec![1, 2, 3, 4, 5, 6]);
825        Ok(())
826    }
827
828    #[tokio::test]
829    async fn mid_history_cursor_replays_suffix_only() -> Result<(), Box<dyn std::error::Error>> {
830        let events = splice_resume(live(vec![Ok(signal(6)?)]), history(1..=5)?, 3)?;
831
832        assert_eq!(delivered_seqs(events).await?, vec![3, 4, 5, 6]);
833        Ok(())
834    }
835
836    #[tokio::test]
837    async fn cursor_at_head_plus_one_yields_empty_replay_and_live_tail_only()
838    -> Result<(), Box<dyn std::error::Error>> {
839        let events = splice_resume(
840            live(vec![Ok(signal(6)?), Ok(signal(7)?)]),
841            history(1..=5)?,
842            6,
843        )?;
844
845        assert_eq!(delivered_seqs(events).await?, vec![6, 7]);
846        Ok(())
847    }
848
849    #[tokio::test]
850    async fn lag_mid_splice_surfaces_unavailable_after_the_replay()
851    -> Result<(), Box<dyn std::error::Error>> {
852        let events = splice_resume(
853            live(vec![Err(EventStreamLagged { skipped: 3 })]),
854            history(1..=2)?,
855            1,
856        )?;
857        let collected: Vec<_> = events.collect().await;
858
859        assert_eq!(collected.len(), 3, "two replay events then the lag item");
860        assert!(collected[0].is_ok() && collected[1].is_ok());
861        assert!(
862            matches!(
863                collected[2].as_ref().err(),
864                Some(ClientError::Unavailable { .. })
865            ),
866            "lag must surface as retryable Unavailable, never a silent gap, got {:?}",
867            collected[2]
868        );
869        Ok(())
870    }
871
872    #[tokio::test]
873    async fn per_workflow_stream_closes_after_terminal_event()
874    -> Result<(), Box<dyn std::error::Error>> {
875        // Terminal at seq 3 mid-replay: deliver 1..=3 and close without
876        // draining the live tail (continue-as-new/terminal run boundary).
877        let mut history = history(1..=2)?;
878        history.push(completed(3)?);
879        history.push(signal(4)?);
880        let events = splice_resume(live(vec![Ok(signal(5)?)]), history, 1)?;
881
882        assert_eq!(
883            delivered_seqs(close_after_terminal(events)).await?,
884            vec![1, 2, 3],
885            "the stream must close after the terminal event"
886        );
887        Ok(())
888    }
889
890    #[tokio::test]
891    async fn live_lag_maps_to_unavailable() -> Result<(), Box<dyn std::error::Error>> {
892        let events = map_lag(live(vec![
893            Ok(signal(1)?),
894            Err(EventStreamLagged { skipped: 9 }),
895        ]));
896        let collected: Vec<_> = events.collect().await;
897
898        assert_eq!(collected.len(), 2);
899        assert!(
900            matches!(
901                collected[1].as_ref().err(),
902                Some(ClientError::Unavailable { .. })
903            ),
904            "got {:?}",
905            collected[1]
906        );
907        Ok(())
908    }
909
910    /// End-to-end through a real engine: the embedded resume splice delivers
911    /// recorded history and live appends gap-free and duplicate-free, built
912    /// on `Engine::subscribe` + `engine.store()` (the pin-note seams).
913    #[tokio::test]
914    async fn embedded_resume_splices_recorded_history_with_live_appends()
915    -> Result<(), Box<dyn std::error::Error>> {
916        use crate::stream::SubscribeTarget;
917        use crate::transport::{EmbeddedWorkflowTransport, WorkflowTransport};
918
919        let capacity = NonZeroUsize::new(16).ok_or("capacity must be non-zero")?;
920        let engine = std::sync::Arc::new(
921            aion::EngineBuilder::new()
922                .store(aion_store::InMemoryStore::default())
923                .in_memory_visibility()
924                .event_streaming(capacity)
925                .build()
926                .await?,
927        );
928        let workflow_id = WorkflowId::new_v4();
929        let mut recorder = aion::durability::Recorder::new(workflow_id.clone(), engine.store());
930        recorder
931            .record_workflow_started(
932                Utc::now(),
933                aion::durability::WorkflowStartRecord {
934                    workflow_type: String::from("checkout"),
935                    input: Payload::from_json(&serde_json::json!({ "cart": [] }))?,
936                    run_id: RunId::new(uuid::Uuid::from_u128(7)),
937                    parent_run_id: None,
938                    parent_workflow_id: None,
939                    package_version: aion_core::PackageVersion::new("a".repeat(64)),
940                },
941            )
942            .await?;
943        for seq in 2..=3 {
944            recorder
945                .record_signal_received(
946                    Utc::now(),
947                    format!("signal-{seq}"),
948                    Payload::from_json(&serde_json::json!({ "seq": seq }))?,
949                )
950                .await?;
951        }
952
953        // Resume from seq 2: replay [2, 3] from the snapshot, then splice the
954        // live append (4) with no gaps and no duplicates.
955        let transport = EmbeddedWorkflowTransport::new(std::sync::Arc::clone(&engine));
956        let request = SubscribeTarget::Workflow {
957            workflow_id: workflow_id.clone(),
958        }
959        .request("default");
960        let attempt = transport.subscribe(request, Some(2)).await?;
961        let mut events = attempt.events;
962
963        let mut delivered = Vec::new();
964        for _ in 0..2 {
965            let item = tokio::time::timeout(Duration::from_secs(2), events.next())
966                .await
967                .map_err(|_| "timed out waiting for a replay event")?
968                .ok_or("stream ended before the replay completed")?;
969            delivered.push(item?.seq());
970        }
971        recorder
972            .record_workflow_completed(
973                Utc::now(),
974                Payload::from_json(&serde_json::json!({ "done": true }))?,
975            )
976            .await?;
977        let item = tokio::time::timeout(Duration::from_secs(2), events.next())
978            .await
979            .map_err(|_| "timed out waiting for the live spliced event")?
980            .ok_or("stream ended before the live event arrived")?;
981        delivered.push(item?.seq());
982        assert_eq!(delivered, vec![2, 3, 4]);
983
984        // Seq 4 is terminal: the per-workflow stream must now close.
985        let end = tokio::time::timeout(Duration::from_secs(2), events.next())
986            .await
987            .map_err(|_| "timed out waiting for the post-terminal close")?;
988        assert!(
989            end.is_none(),
990            "per-workflow stream must close after the terminal event, got {end:?}"
991        );
992
993        // A cursor beyond head + 1 is rejected against the same engine.
994        let ahead = transport
995            .subscribe(
996                SubscribeTarget::Workflow { workflow_id }.request("default"),
997                Some(9),
998            )
999            .await
1000            .err();
1001        assert!(
1002            matches!(ahead, Some(ClientError::InvalidArgument { .. })),
1003            "cursor ahead of history must be InvalidArgument, got {ahead:?}"
1004        );
1005
1006        engine.shutdown()?;
1007        Ok(())
1008    }
1009
1010    /// 🔴 THE THREE CALL SITES THAT READ THE STORE DIRECTLY CARRY A ROUTABLE
1011    /// REFUSAL THROUGH — THEY DO NOT FLATTEN IT INTO "ENGINE BUG".
1012    ///
1013    /// [`map_engine_error`] is only reached by the eight operations that go
1014    /// through an engine API. Three do not: `resolve_run_id` reads the run
1015    /// chain, and `describe_workflow` and the resuming half of `subscribe` read
1016    /// history, all straight off `engine.store()`. Each of those refusals
1017    /// arrives as a bare [`aion_store::StoreError`] with no `EngineError` around
1018    /// it, and each was mapped by hand to `ClientError::server`.
1019    ///
1020    /// `not_owner` is the one store class that names a ROUTING failure — try a
1021    /// different owner, never this one. It is a distinction the CALLER acts on,
1022    /// not one this crate retries for it: `stream.rs`'s `is_retryable` matches
1023    /// only `ClientError::Unavailable`. Flattened to `server` the caller is told
1024    /// its request is unanswerable and gives up on work a re-route would have
1025    /// served, and no test could see the difference:
1026    /// [`aion_store::InMemoryStore`] owns every shard and so can never produce
1027    /// `NotOwner` at all. [`FencedHistoryStore`] is the instrument that can.
1028    ///
1029    /// The unarmed pass is the control. Without it an armed `not_owner` would be
1030    /// consistent with a fixture that never worked — the wrong workflow id, an
1031    /// engine that never built — and the test would be measuring its own setup.
1032    #[tokio::test]
1033    async fn a_directly_read_store_refusal_keeps_its_routing_class()
1034    -> Result<(), Box<dyn std::error::Error>> {
1035        use aion_store::testing::FencedHistoryStore;
1036
1037        use crate::stream::SubscribeTarget;
1038        use crate::transport::{EmbeddedWorkflowTransport, WorkflowTransport};
1039
1040        let capacity = NonZeroUsize::new(16).ok_or("capacity must be non-zero")?;
1041        let store = std::sync::Arc::new(FencedHistoryStore::new());
1042        let engine = std::sync::Arc::new(
1043            aion::EngineBuilder::new()
1044                .store_arc(
1045                    std::sync::Arc::clone(&store) as std::sync::Arc<dyn aion_store::EventStore>
1046                )
1047                .in_memory_visibility()
1048                .event_streaming(capacity)
1049                .build()
1050                .await?,
1051        );
1052        let workflow_id = WorkflowId::new_v4();
1053        let mut recorder = aion::durability::Recorder::new(workflow_id.clone(), engine.store());
1054        recorder
1055            .record_workflow_started(
1056                Utc::now(),
1057                aion::durability::WorkflowStartRecord {
1058                    workflow_type: String::from("checkout"),
1059                    input: Payload::from_json(&serde_json::json!({ "cart": [] }))?,
1060                    run_id: RunId::new(uuid::Uuid::from_u128(11)),
1061                    parent_run_id: None,
1062                    parent_workflow_id: None,
1063                    package_version: aion_core::PackageVersion::new("b".repeat(64)),
1064                },
1065            )
1066            .await?;
1067
1068        let transport = EmbeddedWorkflowTransport::new(std::sync::Arc::clone(&engine));
1069        let describe = |include_history: bool| aion_proto::ProtoDescribeWorkflowRequest {
1070            namespace: String::from("default"),
1071            workflow_id: Some(aion_proto::ProtoWorkflowId::from(workflow_id.clone())),
1072            run_id: None,
1073            include_history,
1074        };
1075
1076        // CONTROL, one per operation, so an armed `not_owner` below is
1077        // attributable to the fence rather than to a fixture that never reached
1078        // the read at all.
1079        //
1080        // 🔴 An earlier revision of this comment claimed "with the fence
1081        // disarmed every one of these succeeds" and ran a control for
1082        // `describe_workflow` only. That claim was not merely unproven, it was
1083        // impossible: this fixture records `WorkflowStarted` and nothing else,
1084        // so the run is NON-TERMINAL and `reopen_workflow` answers the AD-012
1085        // `invalid_state` this file names at `:486-487`. `reopen`'s control is
1086        // therefore the sharper one available — disarmed it must fail with some
1087        // class OTHER than `not_owner`, which still separates "the fence did
1088        // it" from "this call always fails".
1089        transport.describe_workflow(describe(true)).await?;
1090        transport
1091            .subscribe(
1092                SubscribeTarget::Workflow {
1093                    workflow_id: workflow_id.clone(),
1094                }
1095                .request("default"),
1096                Some(1),
1097            )
1098            .await?;
1099        let reopen_request = || aion_proto::ProtoReopenRequest {
1100            namespace: String::from("default"),
1101            workflow_id: Some(aion_proto::ProtoWorkflowId::from(workflow_id.clone())),
1102            run_id: None,
1103        };
1104        let reopen_control = transport.reopen(reopen_request()).await;
1105        assert_ne!(
1106            reopen_control.as_ref().err().map(ClientError::class),
1107            Some("not_owner"),
1108            "the reopen control answered `not_owner` with the fence DISARMED, so the armed \
1109             assertion below would prove nothing: {reopen_control:?}"
1110        );
1111
1112        store.arm_fence();
1113
1114        let described = transport.describe_workflow(describe(false)).await;
1115        assert_eq!(
1116            described.as_ref().err().map(ClientError::class),
1117            Some("not_owner"),
1118            "describe_workflow flattened a shard-ownership refusal into an unroutable class: \
1119             {described:?}"
1120        );
1121
1122        let subscribed = transport
1123            .subscribe(
1124                SubscribeTarget::Workflow {
1125                    workflow_id: workflow_id.clone(),
1126                }
1127                .request("default"),
1128                Some(1),
1129            )
1130            .await;
1131        assert_eq!(
1132            subscribed.as_ref().err().map(ClientError::class),
1133            Some("not_owner"),
1134            "the resuming half of subscribe flattened a shard-ownership refusal into an \
1135             unroutable class"
1136        );
1137
1138        // `resolve_run_id` reads the RUN CHAIN rather than history, and it is
1139        // reached only when the caller omits the run id — which is why
1140        // `reopen_request` carries `run_id: None`.
1141        let reopened = transport.reopen(reopen_request()).await;
1142        assert_eq!(
1143            reopened.as_ref().err().map(ClientError::class),
1144            Some("not_owner"),
1145            "resolve_run_id flattened a shard-ownership refusal into an unroutable class: \
1146             {reopened:?}"
1147        );
1148
1149        // Teardown, and the fence comes down FIRST. Every other engine-backed
1150        // test in this file shuts its engine down; this one left the engine to
1151        // `Drop`, which gates the epoch but cannot await, so teardown work that
1152        // must finish was left racing the test's exit. Disarming before the
1153        // shutdown means the shutdown is measured against an honest store — a
1154        // shutdown error arriving through `?` here would then be a real fault
1155        // and not the fixture fencing its own teardown's history access.
1156        store.disarm_fence();
1157        engine.shutdown()?;
1158        Ok(())
1159    }
1160
1161    /// Every arm this map NAMES must reach its own class, and the wildcard must
1162    /// still catch what it is for.
1163    ///
1164    /// 🔴 Structured as a table with a NEGATIVE control (`Runtime`), because the
1165    /// assertion that matters is not "these map somewhere" but "these map
1166    /// somewhere OTHER than the generic bucket". Without the control, deleting
1167    /// every named arm and returning `ClientError::server` for everything would
1168    /// still fail — but a test that only listed the named variants could not
1169    /// tell a correct map from one that had accidentally started naming
1170    /// everything.
1171    ///
1172    /// 🔴 EACH ERROR IS BUILT IN THE SHAPE PRODUCTION PRODUCES IT, NOT THE
1173    /// SHAPE THAT MAKES THE ARM LOOK COVERED.
1174    ///
1175    /// An earlier revision of this table listed a bare
1176    /// `Store(StoreError::NotOwner)` and passed — and the arm it exercised
1177    /// never fired on the shape a caller actually provokes. A fenced APPEND
1178    /// goes through the `Recorder`, which returns `DurabilityError`, so that
1179    /// path's caller-reachable shape is `Durability(Store(NotOwner))` — and
1180    /// against THAT shape the same table failed with `left: "backend"`.
1181    ///
1182    /// The bare shape is reachable too, and an earlier revision of THIS comment
1183    /// wrongly said it was not. `EngineError` carries `Store(#[from]
1184    /// StoreError)`, so every `?` on a store call inside an engine API that
1185    /// returns `EngineError` produces it: `list_workflows`
1186    /// (`api_workflow_ops.rs`, both `store.query` and `store.read_history`),
1187    /// `cancel` (`lifecycle/terminate.rs`) and `signal`
1188    /// (`engine/delegated.rs`) are three of the eight engine operations this
1189    /// transport calls, and all three are store READS rather than appends —
1190    /// which is precisely why no `Recorder` is involved and no `Durability`
1191    /// wrapper appears. Both shapes are listed below because both happen.
1192    ///
1193    /// ⚠️ WHAT THIS TEST DOES NOT DO. The classes are the same ones
1194    /// `aion-server`'s `error.rs` assigns, but this test cannot check that: the
1195    /// dependency runs server → client, so the server's table is not visible
1196    /// from here and these expectations are a TRANSCRIPTION of it. A rule
1197    /// written in two places with nothing forcing agreement has already
1198    /// drifted or will. What holds them together today is that a change to
1199    /// either side must be made deliberately on both; the durable fix is a
1200    /// shared classifier, which needs a crate both can depend on and is not in
1201    /// this change's scope. Naming it here rather than letting the doc comment
1202    /// imply a guarantee the test does not provide.
1203    #[test]
1204    fn every_named_engine_error_reaches_its_own_class() {
1205        use super::map_engine_error;
1206
1207        for (error, expected) in engine_error_class_table() {
1208            assert_eq!(
1209                map_engine_error(&error).class(),
1210                expected,
1211                "wrong class for {error}"
1212            );
1213        }
1214    }
1215
1216    /// F3: the CLASS is not the whole answer — `ShuttingDown` must also carry
1217    /// its discriminator, because the operator-facing hint keyed off a bare
1218    /// `not_running` tells them the run ended and points at `aion list`, which
1219    /// is false twice (the run is fine; that command fails the same way).
1220    ///
1221    /// The class-table test above is invariant to this: it asserts
1222    /// `"not_running"` and would stay green with the discriminator dropped.
1223    /// That is exactly why this assertion is separate.
1224    ///
1225    /// Killing mutation: revert the arm to `ClientError::not_running(...)`.
1226    /// `error_type` becomes `None` and the first assertion fails.
1227    #[test]
1228    fn shutting_down_carries_its_discriminator_not_only_its_class()
1229    -> Result<(), Box<dyn std::error::Error>> {
1230        let mapped = super::map_engine_error(&aion::EngineError::ShuttingDown);
1231        let ClientError::NotRunning { detail } = &mapped else {
1232            return Err(format!("ShuttingDown must keep the not_running CLASS: {mapped}").into());
1233        };
1234        assert_eq!(
1235            detail.error_type.as_deref(),
1236            Some("ShuttingDown"),
1237            "the wire surface builds `not_running_with_type(\"ShuttingDown\", …)`; an embedded \
1238             caller that loses the discriminator cannot be told anything true about why"
1239        );
1240        // CONTROL: a different error in the same class must NOT claim this
1241        // discriminator, or the assertion above would pass for anything.
1242        let other = super::map_engine_error(&aion::EngineError::Runtime {
1243            reason: "beamr scheduler refused".to_owned(),
1244        });
1245        assert_ne!(
1246            other.class(),
1247            "not_running",
1248            "control: the negative case must not share the class under test"
1249        );
1250        Ok(())
1251    }
1252
1253    /// The table itself, lifted out of the test body so the case list can grow
1254    /// with the taxonomy without the assertion loop growing at all, and split
1255    /// by family so each half stays readable.
1256    fn engine_error_class_table() -> Vec<(aion::EngineError, &'static str)> {
1257        let mut cases = admission_and_run_state_cases();
1258        cases.extend(store_and_query_cases());
1259        cases
1260    }
1261
1262    /// Refusals about the request, the run's state, or the engine's own
1263    /// availability — plus the negative control.
1264    fn admission_and_run_state_cases() -> Vec<(aion::EngineError, &'static str)> {
1265        use aion_core::{RunId, WorkflowId};
1266
1267        let version = aion::ContentHash::from_bytes([7u8; 32]);
1268        vec![
1269            // The two oldest named arms, and until this revision the only two
1270            // with no row: the table asserted every arm reached its own class
1271            // while silently omitting them, so either could have been deleted
1272            // and every assertion here would still have passed.
1273            (
1274                aion::EngineError::WorkflowNotFound {
1275                    workflow_type: "orders".to_owned(),
1276                },
1277                "not_found",
1278            ),
1279            (
1280                aion::EngineError::InvalidState {
1281                    reason: "workflow w run r is Running, not terminal".to_owned(),
1282                },
1283                "invalid_state",
1284            ),
1285            (
1286                aion::EngineError::StartInputRefused {
1287                    workflow_type: "orders".to_owned(),
1288                    version: version.clone(),
1289                    reason: "field `total` is missing".to_owned(),
1290                },
1291                "invalid_input",
1292            ),
1293            (
1294                aion::EngineError::SignalRefused {
1295                    workflow_id: WorkflowId::new_v4(),
1296                    run_id: RunId::new_v4(),
1297                    signal_name: "approve".to_owned(),
1298                    version,
1299                    reason: "undeclared signal".to_owned(),
1300                },
1301                "invalid_input",
1302            ),
1303            (
1304                aion::EngineError::TerminalWriterUnavailable {
1305                    workflow_id: "w".to_owned(),
1306                    run_id: "r".to_owned(),
1307                    holder: "another reservation".to_owned(),
1308                },
1309                "invalid_state",
1310            ),
1311            (
1312                aion::EngineError::TerminalWriterHeld {
1313                    workflow_id: "w".to_owned(),
1314                    run_id: "r".to_owned(),
1315                },
1316                "invalid_state",
1317            ),
1318            (
1319                aion::EngineError::RunIsRecoverable {
1320                    workflow_id: "w".to_owned(),
1321                    run_id: "r".to_owned(),
1322                    version: "abc".to_owned(),
1323                },
1324                "invalid_state",
1325            ),
1326            (
1327                aion::EngineError::NoResidencyVerdict {
1328                    workflow_id: "w".to_owned(),
1329                    run_id: "r".to_owned(),
1330                },
1331                "invalid_state",
1332            ),
1333            (
1334                aion::EngineError::ContractIdentity {
1335                    workflow_type: "orders".to_owned(),
1336                    source: aion::ContractIdentityError::RedeployRequired {
1337                        stored_version: "orders$deadbeef".to_owned(),
1338                    },
1339                },
1340                "invalid_state",
1341            ),
1342            (
1343                aion::EngineError::NoQueueDeclaration {
1344                    workflow_type: "orders".to_owned(),
1345                    version: aion::ContentHash::from_bytes([5u8; 32]),
1346                    activities: "charge_card,send_receipt".to_owned(),
1347                },
1348                "invalid_state",
1349            ),
1350            (aion::EngineError::ShuttingDown, "not_running"),
1351            // The control: a variant this map does NOT name must still land in
1352            // the generic bucket. If it stops doing so, the wildcard has been
1353            // replaced by something that names everything, and the assertions
1354            // above would have passed for the wrong reason.
1355            (
1356                aion::EngineError::Runtime {
1357                    reason: "beamr scheduler refused".to_owned(),
1358                },
1359                "backend",
1360            ),
1361        ]
1362    }
1363
1364    /// The two nested families, each of which reaches this map wrapped in an
1365    /// outer `EngineError` variant rather than as itself.
1366    fn store_and_query_cases() -> Vec<(aion::EngineError, &'static str)> {
1367        use aion_core::WorkflowId;
1368
1369        vec![
1370            (
1371                aion::EngineError::Store(aion_store::StoreError::NotOwner { shard: 3 }),
1372                "not_owner",
1373            ),
1374            // 🔴 The shape a caller actually gets: the fence is raised inside a
1375            // recorded append, so it arrives wrapped in `Durability`.
1376            (
1377                aion::EngineError::Durability(aion::durability::DurabilityError::Store(
1378                    aion_store::StoreError::NotOwner { shard: 3 },
1379                )),
1380                "not_owner",
1381            ),
1382            (
1383                aion::EngineError::Durability(aion::durability::DurabilityError::Store(
1384                    aion_store::StoreError::NotFound {
1385                        workflow_id: WorkflowId::new_v4(),
1386                    },
1387                )),
1388                "not_found",
1389            ),
1390            // The store family's own control: a store fault that is genuinely
1391            // the engine's problem must NOT acquire a caller-facing class just
1392            // because it arrived through the same helper.
1393            (
1394                aion::EngineError::Durability(aion::durability::DurabilityError::Store(
1395                    aion_store::StoreError::SequenceConflict {
1396                        expected: 4,
1397                        found: 7,
1398                    },
1399                )),
1400                "backend",
1401            ),
1402            (
1403                aion::EngineError::Query(aion::QueryError::UnknownQuery("balance".to_owned())),
1404                "unknown_query",
1405            ),
1406            (
1407                aion::EngineError::Query(aion::QueryError::Timeout),
1408                "query_timeout",
1409            ),
1410            (
1411                aion::EngineError::Query(aion::QueryError::NotRunning(WorkflowId::new_v4())),
1412                "not_running",
1413            ),
1414            (
1415                aion::EngineError::Query(aion::QueryError::ReplyDropped),
1416                "not_running",
1417            ),
1418            (
1419                aion::EngineError::Query(aion::QueryError::Unknown(WorkflowId::new_v4())),
1420                "not_found",
1421            ),
1422            (
1423                aion::EngineError::Query(aion::QueryError::HandlerFailed {
1424                    message: "handler panicked".to_owned(),
1425                }),
1426                "query_failed",
1427            ),
1428            // The caller's own arguments were malformed: the request is the
1429            // thing to fix, so this is `invalid_argument` — not the handler's
1430            // failure and not the engine's.
1431            (
1432                aion::EngineError::Query(aion::QueryError::InvalidArguments {
1433                    reason: "arguments payload is not a well-formed JSON document".to_owned(),
1434                }),
1435                "invalid_input",
1436            ),
1437            // The query family's own control, and the eighth of eight arms —
1438            // an earlier revision of this table listed six and left this one
1439            // unmeasured, so a change re-classifying an engine-seam failure as
1440            // a caller-facing one survived green. A seam that could not deliver
1441            // the query is the ENGINE's failure, not the caller's, and must
1442            // land in the generic bucket rather than acquire a remedy the
1443            // caller cannot act on.
1444            (
1445                aion::EngineError::Query(aion::QueryError::Engine(
1446                    aion::engine_seam::EngineSeamError::Delivery {
1447                        reason: "mailbox send failed".to_owned(),
1448                    },
1449                )),
1450                "backend",
1451            ),
1452        ]
1453    }
1454}