Skip to main content

tokeira_engine/
lib.rs

1//! Embeddable Temporal-compatible execution engine and `tokeirad` service bootstrap.
2//!
3//! [`Engine`] is the zero-listener entry point: it owns an in-memory authoritative
4//! store, runtime workers, and the Temporal edge router. Its [`TemporalEndpoint`]
5//! accepts raw protobuf calls and, with the default `temporalio-client` feature,
6//! produces a callback service for `ConnectionOptions::service_override`.
7//!
8//! `tokeirad` also consumes this crate. [`TokeiradHandle::start_in_memory`] and
9//! [`run_from_cli`] attach the optional gRPC, HTTP/JSON, gRPC-Web, and Nexus HTTP
10//! transports after the shared service graph has been constructed. Embedded and
11//! listener-backed operation therefore execute the same edge handlers and runtime
12//! semantics; only their transport and shutdown handles differ.
13
14#![deny(rust_2018_idioms)]
15// The bootstrap facade wires many subsystems and accepts them as parameters.
16#![allow(clippy::too_many_arguments)]
17
18use std::{
19    collections::HashMap,
20    convert::Infallible,
21    future::Future,
22    net::SocketAddr,
23    path::{Path, PathBuf},
24    pin::Pin,
25    sync::{
26        Arc, RwLock, Weak,
27        atomic::{AtomicU64, Ordering},
28    },
29    time::{Duration as StdDuration, Instant},
30};
31
32use anyhow::{Context, Result, anyhow};
33use async_trait::async_trait;
34use clap::Parser;
35use http_body_util::{BodyExt, Full, LengthLimitError, Limited};
36use hyper::{
37    Method, Request, Response, StatusCode,
38    body::{Bytes, Incoming},
39    service::service_fn,
40};
41use hyper_util::rt::TokioIo;
42use tokio::{
43    fs::{self, OpenOptions},
44    io::AsyncWriteExt as _,
45    net::TcpListener,
46    sync::{broadcast, mpsc, oneshot},
47    task::JoinHandle,
48};
49use tokio_stream::wrappers::TcpListenerStream;
50use tokio_util::sync::CancellationToken;
51use tonic::transport::Server;
52use tonic_web::GrpcWebLayer;
53use tower_http::cors::CorsLayer;
54use tracing::{error, info};
55
56#[cfg(feature = "chasm-extensions")]
57pub mod chasm;
58mod chasm_extensions;
59#[cfg(feature = "chasm-extensions")]
60pub use chasm_extensions::EngineBuilder;
61use chasm_extensions::{
62    ChasmExtensions, check_rebuilt_outboxes, check_registered_archetypes,
63    declared_search_attributes, embedded_stack_error, registry_start_error,
64    seed_declared_search_attributes, seed_start_error,
65};
66
67pub mod correlation_format;
68#[cfg(all(test, feature = "dsql-integration"))]
69mod distributed_schema_live_tests;
70#[cfg(test)]
71mod embedded_dsql_tests;
72#[doc(hidden)]
73pub mod harness;
74mod http_api_transport;
75mod listener;
76mod nexus_http_transport;
77pub mod observability;
78
79pub use listener::{EngineListenError, EngineListener, EngineListenerShutdownError};
80
81use http_api_transport::HttpApiLayer;
82use nexus_http_transport::NexusHttpLayer;
83use tokeira_auth::{
84    Authorizer, ClaimMapper, DefaultAuthorizer, GrantRule, GrantRules, JwksKeyProvider,
85    JwtAuthenticator, JwtIssuerProfile, MultiSourceClaimMapper, StsAuthenticator, WorkerScope,
86    WorkerScopeRule, WorkerScopeRules,
87};
88use tokeira_chasm::Library as _;
89use tokeira_config::{Cli, ConfigStorageKind};
90pub use tokeira_config::{
91    DsqlMigrationPolicy, EmbeddedEngineConfig, EmbeddedStorageConfig, SnapshotPolicyConfig,
92    TokeiraConfig,
93};
94use tokeira_edge::{
95    Authenticator, CacheBackedRouter, CallbackResponse, EdgeInterceptors, EdgeRoutingConfig,
96    HistoryNotifyingRepository, HistoryWaitRegistry, HttpApiCatalog, HttpApiPolicy,
97    InMemoryNamespaceCache, InMemoryOperatorApi, InProcessGrpcService, LocalOnlyRouter,
98    LongPollConfig, LongPollGate, NamespaceCache, OperatorService, PendingQueryStore,
99    PollerRegistry, ResolvedNamespace, RoutingCache, ScopedWorkerSessionRegistry,
100    WorkerComputeNamespaceCatalogAdapter, WorkflowExecutionDescription, WorkflowService,
101    conformance::{WireCoverageLayer, WireCoverageRecorder},
102    grpc::{
103        admin_service::AdminServiceGrpc, operator_service::OperatorServiceGrpc,
104        runtime_adapter::RuntimeAdapter, workflow_service::WorkflowServiceGrpc,
105    },
106    handle_nexus_callback,
107    nexus_http::MAX_NEXUS_PAYLOAD_BYTES,
108    operator_service::{ClusterInfo, OperatorApi, SearchAttributeDefinition},
109    run_routing_subscription,
110    translate::to_internal::namespace_id_for,
111    workflow_service::{ExecutionResolver, WorkflowRuntimeApi},
112};
113pub use tokeira_edge::{InProcessGrpcRequest, InProcessGrpcResponse};
114
115struct AuthorizationStack {
116    grpc: Arc<dyn Authenticator>,
117    nexus: Arc<dyn tokeira_edge::nexus_http::NexusHttpAuthorizer>,
118    principal_attribution: bool,
119}
120
121async fn build_authorization_stack(config: &TokeiraConfig) -> Result<AuthorizationStack> {
122    let Some(authorization) = config
123        .policy
124        .authorization
125        .as_ref()
126        .filter(|authorization| authorization.has_identity_source())
127    else {
128        let grpc = harness::installed()
129            .and_then(|hooks| hooks.fallback_grpc_authenticator.clone())
130            .unwrap_or_else(|| Arc::new(tokeira_edge::AllowAllAuthenticator));
131        let stack = AuthorizationStack {
132            grpc,
133            nexus: Arc::new(tokeira_edge::nexus_http::PermissiveNexusHttpAuthorizer),
134            principal_attribution: false,
135        };
136        return Ok(stack);
137    };
138
139    let mut profiles = Vec::with_capacity(authorization.jwt.issuers.len());
140    for issuer in &authorization.jwt.issuers {
141        let grants = GrantRules::new(
142            issuer
143                .grants
144                .iter()
145                .map(|rule| {
146                    GrantRule::new(
147                        rule.match_sub.clone(),
148                        rule.grant.iter().map(String::as_str),
149                    )
150                    .with_context(|| format!("invalid JWT grants for issuer {}", issuer.name))
151                })
152                .collect::<Result<Vec<_>>>()?,
153        );
154        let worker_scopes = WorkerScopeRules::new(
155            issuer
156                .worker_scopes
157                .iter()
158                .map(|rule| {
159                    let scope = WorkerScope::try_new(
160                        rule.namespace.clone(),
161                        rule.task_queues.clone(),
162                        rule.deployment_name.clone(),
163                        rule.build_id.clone(),
164                    )
165                    .with_context(|| {
166                        format!("invalid JWT Worker scope for issuer {}", issuer.name)
167                    })?;
168                    WorkerScopeRule::new(rule.match_sub.clone(), scope).with_context(|| {
169                        format!(
170                            "invalid JWT Worker-scope subject for issuer {}",
171                            issuer.name
172                        )
173                    })
174                })
175                .collect::<Result<Vec<_>>>()?,
176        );
177        let keys =
178            JwksKeyProvider::start(issuer.jwks_uri.clone(), issuer.refresh_interval_duration())
179                .await;
180        profiles.push(
181            JwtIssuerProfile::new(
182                issuer.name.clone(),
183                issuer.issuer.clone(),
184                issuer.audience.clone(),
185                issuer.permissions_claim.clone(),
186                grants,
187                keys,
188            )
189            .with_worker_scopes(worker_scopes),
190        );
191    }
192    let jwt = (!profiles.is_empty()).then(|| JwtAuthenticator::new(profiles));
193    let sts = authorization
194        .aws_iam
195        .as_ref()
196        .map(|aws_iam| {
197            let grants = aws_iam
198                .grants
199                .iter()
200                .map(|rule| {
201                    GrantRule::new(
202                        rule.match_arn.clone(),
203                        rule.grant.iter().map(String::as_str),
204                    )
205                    .context("invalid AWS IAM grants")
206                })
207                .collect::<Result<Vec<_>>>()
208                .map(GrantRules::new)?;
209            let worker_scopes = WorkerScopeRules::new(
210                aws_iam
211                    .worker_scopes
212                    .iter()
213                    .map(|rule| {
214                        let scope = WorkerScope::try_new(
215                            rule.namespace.clone(),
216                            rule.task_queues.clone(),
217                            rule.deployment_name.clone(),
218                            rule.build_id.clone(),
219                        )
220                        .context("invalid AWS IAM Worker scope")?;
221                        WorkerScopeRule::new(rule.match_arn.clone(), scope)
222                            .context("invalid AWS IAM Worker-scope ARN")
223                    })
224                    .collect::<Result<Vec<_>>>()?,
225            );
226            Ok::<_, anyhow::Error>(StsAuthenticator::new(grants).with_worker_scopes(worker_scopes))
227        })
228        .transpose()?;
229    let mapper: Arc<dyn ClaimMapper> = Arc::new(MultiSourceClaimMapper::new(jwt, sts));
230    let authorizer: Arc<dyn Authorizer> = Arc::new(DefaultAuthorizer);
231    Ok(AuthorizationStack {
232        grpc: Arc::new(tokeira_edge::PolicyAuthenticator::new(
233            mapper.clone(),
234            authorizer.clone(),
235            authorization.expose_authorizer_errors,
236        )),
237        nexus: Arc::new(tokeira_edge::nexus_http::PolicyNexusHttpAuthorizer::new(
238            mapper,
239            authorizer,
240            authorization.expose_authorizer_errors,
241        )),
242        principal_attribution: authorization.principal_attribution,
243    })
244}
245use tokeira_kernel::{Link, LoadedRun};
246use tokeira_managed_dsql::{
247    AwsDsqlControlPlane, CanonicalClusterIdentity, ClusterAction, CreateOrRecoverRequest,
248    DsqlControlPlane, LifecycleEnvironment, LocalClusterDescriptorStore, ManagedDsqlLifecycle,
249    Readiness, ResolvedCluster, StartupDeadline, SystemLifecycleEnvironment,
250};
251use tokeira_observability::{
252    ClusterStatusLabel, DbClassLabel, EmbeddedOperationLabel, EmbeddedStorageModeLabel,
253    ErrorClassLabel, OwnershipOutcomeLabel, SchemaOutcomeLabel, record_embedded_lifecycle,
254};
255use tokeira_projection::{
256    DsqlVisibilityStore, InMemoryVisibilityStore, ProjectionSink, ProjectionWorker, SearchAttrType,
257    VisibilityQueryService, VisibilitySink, VisibilityStore,
258};
259use tokeira_runtime::{
260    ChannelDemandObservationSink, ChannelWorkerComputeReconcileSink,
261    CompletionCallbackScannerConfig, CompletionDeliveryOutcome, ConnectionBudgetApplier,
262    HttpNexusClient, HttpNexusCompletionClient, InMemoryNexusEndpointStore, MembershipConfig,
263    NEXUS_CALLBACK_PATH, NEXUS_OPERATION_STATE_HEADER, NEXUS_OPERATION_TOKEN_HEADER,
264    NexusCompletion, NexusCompletionClient, NexusCompletionDeps, NexusCompletionRuntimeConfig,
265    NexusEndpointRegistry, NexusEndpointSpec, NexusEndpointSpecTarget, NexusEndpointStore,
266    NexusNamespaceResolver, NexusWorkerComputeProvider, OBSERVATION_CHANNEL_CAPACITY,
267    RECONCILE_CHANNEL_CAPACITY, RepositoryBackedTaskQueueConfigStore, RuntimeConfig,
268    RuntimeShutdownHandle, ScheduleEngineConfig, ScheduleStore, SweepResult,
269    SystemWorkerComputeClock, TEMPORAL_CALLBACK_TOKEN_HEADER, TokeiraRuntime,
270    WorkerComputeControllerService, WorkerComputeOutbox, WorkerComputeReconciler,
271    WorkflowTaskReportedProblem, nexus_payload_to_body, reported_problem_from_state,
272    run_schedule_engine, system_callback_post_url,
273};
274use tokeira_storage::{
275    ConnectionDirector, DbClass, InMemoryStore, InMemoryWorkerComputeRepository, LeaseOutcome,
276    LeaseRepository, ProjectionLog, RunRepository, TaskQueueConfigRepository,
277    WorkerComputeRepository, WorkerDeploymentRepository, WorkerTaskProvenanceStore,
278    dsql::{
279        ConnectionControlLeaseRepository, ControlLeaseAcquireOutcome, ControlLeaseAcquireRequest,
280        ControlLeaseClusterIdentity, ControlLeaseError, ControlLeaseGuard, DsqlAuthConfig,
281        DsqlConnectionDirector, DsqlCoordinationConfig, DsqlPoolConfig, DsqlStore,
282        EmbeddedDsqlPoolConfig, MigrationRunner, OWNER_ADMISSION_MARGIN, OWNER_LEASE_DURATION,
283        OWNER_RENEW_INTERVAL, OwnershipAdmissionGate, OwnershipAdmissionState,
284        SCHEMA_MIGRATION_ADMISSION_MARGIN, SCHEMA_MIGRATION_LEASE_DURATION,
285        SchemaCompatibilityError, SchemaDecision, SchemaIncompatibilityCategory,
286        SchemaMigrationPolicy, WarmupDeadline,
287    },
288};
289use tokeira_types::{
290    ExecutionRef, IncarnationId, NamespaceId, NodeEndpoint, PlacementConfig, ProjectionCursor,
291    SearchAttrValue, ShardEpoch, ShardId, WorkflowId,
292};
293
294/// Nexus-endpoint bootstrap target used by the dev startup path. Kept in the
295/// library so both the CLI wrapper and the `start_in_memory` facade build the
296/// same initial registry shape.
297#[allow(dead_code)]
298#[derive(Clone, Debug)]
299pub enum BootstrapNexusEndpointTarget {
300    External {
301        address: String,
302    },
303    Worker {
304        namespace_name: String,
305        task_queue: String,
306    },
307}
308
309#[derive(Debug)]
310struct VisibilityRegistryOperatorApi<V> {
311    inner: InMemoryOperatorApi,
312    visibility_store: V,
313    declared_search_attributes: Vec<(String, SearchAttrType)>,
314}
315
316impl<V> VisibilityRegistryOperatorApi<V> {
317    fn new(inner: InMemoryOperatorApi, visibility_store: V) -> Self {
318        Self {
319            inner,
320            visibility_store,
321            declared_search_attributes: Vec::new(),
322        }
323    }
324
325    fn with_declared_search_attributes(mut self, declared: Vec<(String, SearchAttrType)>) -> Self {
326        self.declared_search_attributes = declared;
327        self
328    }
329}
330
331#[async_trait]
332impl<V> OperatorApi for VisibilityRegistryOperatorApi<V>
333where
334    V: VisibilityStore + Clone + 'static,
335{
336    async fn cluster_info(&self) -> Result<ClusterInfo> {
337        self.inner.cluster_info().await
338    }
339
340    async fn list_search_attributes(
341        &self,
342        namespace: Option<&str>,
343    ) -> Result<Vec<SearchAttributeDefinition>> {
344        self.inner.list_search_attributes(namespace).await
345    }
346
347    async fn upsert_search_attribute(
348        &self,
349        namespace: &str,
350        attr: SearchAttributeDefinition,
351    ) -> Result<()> {
352        let attr_type = visibility_search_attr_type(&attr.attr_type)?;
353        // OperatorService mutates the user-visible catalog, but projection is
354        // the authority that makes a field queryable. Register first so a
355        // successful AddSearchAttributes response cannot expose an attribute
356        // that the visibility compiler still rejects.
357        self.visibility_store
358            .register_attr(namespace_id_for(namespace), attr.name.clone(), attr_type)
359            .await?;
360        self.inner.upsert_search_attribute(namespace, attr).await
361    }
362
363    async fn remove_search_attribute(&self, namespace: &str, attr_name: &str) -> Result<()> {
364        self.inner
365            .remove_search_attribute(namespace, attr_name)
366            .await
367    }
368
369    /// Register the predefined search-attribute set into the visibility store for
370    /// this namespace (idempotent upserts), so visibility queries in a runtime-
371    /// created namespace resolve predefined fields exactly as the bootstrapped
372    /// `default` namespace does. Routed to the store, not `self.inner`, so the
373    /// predefined set stays out of the user-visible catalog.
374    async fn seed_predefined_search_attributes(&self, namespace: &str) -> Result<()> {
375        tokeira_projection::seed_predefined_search_attributes(
376            &self.visibility_store,
377            namespace_id_for(namespace),
378        )
379        .await?;
380        // Resolve before register: the projection's idempotent registration does
381        // not reject an existing key of another type. New namespaces get the same
382        // declarations as namespaces present at startup.
383        seed_declared_search_attributes(
384            &self.visibility_store,
385            namespace_id_for(namespace),
386            &self.declared_search_attributes,
387        )
388        .await?;
389        Ok(())
390    }
391}
392
393/// [`NexusNamespaceResolver`] backed by the edge namespace cache.
394///
395/// The runtime publisher needs a namespace's *name* where it holds only its
396/// [`NamespaceId`]: to tag the External-endpoint outbound Nexus metric with the
397/// originator's namespace, and to name the target namespace in the event that resolves an
398/// external signal or cancel request whose command named none. tokeira namespace ids are a
399/// non-invertible function of the name, so resolution scans the (small) registered set and
400/// matches `namespace_id_for(name)` — the inverse the runtime needs. Returns `None` when no
401/// registered namespace hashes to the id, leaving the metric unrecorded rather than
402/// mistagged, and the resolving event's name empty.
403struct CacheNexusNamespaceResolver {
404    namespaces: Arc<dyn NamespaceCache>,
405}
406
407#[async_trait]
408impl NexusNamespaceResolver for CacheNexusNamespaceResolver {
409    async fn name_for_id(&self, namespace_id: NamespaceId) -> Option<String> {
410        self.namespaces
411            .list_all()
412            .await
413            .ok()?
414            .into_iter()
415            .map(|namespace| namespace.name)
416            .find(|name| namespace_id_for(name) == namespace_id)
417    }
418}
419
420fn visibility_search_attr_type(value: &str) -> Result<SearchAttrType> {
421    match value {
422        "keyword" => Ok(SearchAttrType::Keyword),
423        "keyword_list" => Ok(SearchAttrType::KeywordList),
424        "int" => Ok(SearchAttrType::Int),
425        "bool" => Ok(SearchAttrType::Bool),
426        "double" => Ok(SearchAttrType::Double),
427        "datetime" => Ok(SearchAttrType::Datetime),
428        "text" => Ok(SearchAttrType::Text),
429        other => Err(anyhow!("unsupported search attribute type `{other}`")),
430    }
431}
432
433/// Companion to [`BootstrapNexusEndpointTarget`] carrying per-endpoint config
434/// during bootstrap.
435#[allow(dead_code)]
436#[derive(Clone, Debug)]
437pub struct BootstrapNexusEndpointConfig {
438    pub target: BootstrapNexusEndpointTarget,
439}
440
441/// A running Tokeira engine that binds no listener at startup.
442///
443/// The engine owns the authoritative store, runtime workers, and edge
444/// services. [`Self::endpoint`] is cloneable and can be adapted directly to the
445/// Temporal Rust SDK's callback transport; [`Self::listen`] optionally serves the
446/// same services on a host-chosen TCP address. Dropping the engine cancels
447/// background work, stops every attached listener, and makes every endpoint
448/// clone reject new calls. When snapshot policy is configured, call
449/// [`Self::shutdown`] for the final graceful-shutdown snapshot; `Drop` cannot
450/// perform asynchronous file I/O.
451#[derive(Debug)]
452pub struct Engine {
453    // Shared bootstrap state exists in every deployment; typed access is opt-in.
454    #[cfg_attr(not(feature = "chasm-extensions"), allow(dead_code))]
455    component_visibility_store: chasm_extensions::VisibilityQueryStore,
456    // Owned in every deployment; typed access is an opt-in public surface.
457    #[cfg_attr(not(feature = "chasm-extensions"), allow(dead_code))]
458    chasm_engine: Arc<tokeira_runtime::chasm::ChasmEngine>,
459    #[cfg_attr(not(feature = "chasm-extensions"), allow(dead_code))]
460    registry: Arc<tokeira_chasm::Registry>,
461    endpoint: TemporalEndpoint,
462    background_cancel: CancellationToken,
463    log_broadcast: broadcast::Sender<LogEvent>,
464    snapshot_policy: Option<EngineSnapshotPolicy>,
465    recovery_task: Option<JoinHandle<Result<()>>>,
466    startup_report: EngineStartupReport,
467    shutdown_coordinator: Option<EmbeddedShutdownCoordinator>,
468    listeners: listener::ListenerRegistry,
469}
470
471/// Storage mode selected by the explicit embedded startup boundary.
472#[derive(Clone, Copy, Debug, PartialEq, Eq)]
473pub enum EmbeddedStorageMode {
474    /// Ephemeral process-local authoritative storage.
475    InMemory,
476    /// Dedicated cluster created or recovered by the managed lifecycle.
477    ManagedDsql,
478    /// Canonical operator-supplied DSQL identity.
479    ExistingDsql,
480}
481
482/// Complete redacted report returned by a successful embedded startup.
483#[derive(Clone, Debug, PartialEq, Eq)]
484pub struct EngineStartupReport {
485    /// Explicit storage mode that reached admission-open.
486    pub storage_mode: EmbeddedStorageMode,
487    /// Canonical cluster observation for a durable mode.
488    pub cluster: Option<ClusterStartupReport>,
489    /// Release-pinned schema outcome for a durable mode.
490    pub schema: Option<SchemaStartupReport>,
491    /// Exclusive embedded ownership outcome for a durable mode.
492    pub ownership: Option<OwnershipStartupReport>,
493}
494
495impl EngineStartupReport {
496    fn in_memory() -> Self {
497        Self {
498            storage_mode: EmbeddedStorageMode::InMemory,
499            cluster: None,
500            schema: None,
501            ownership: None,
502        }
503    }
504}
505
506/// Canonical cluster fields safe for host diagnostics.
507#[derive(Clone, Debug, PartialEq, Eq)]
508pub struct ClusterStartupReport {
509    /// AWS Region used by lifecycle and IAM signing.
510    pub region: String,
511    /// Canonical AWS DSQL cluster ID.
512    pub cluster_id: String,
513    /// Canonical AWS DSQL cluster ARN.
514    pub cluster_arn: String,
515    /// Locator supplied to this generation's database connection factory.
516    ///
517    /// Existing mode uses the configured endpoint; managed mode uses the AWS
518    /// endpoint observed before pool warmup. Later wake observations do not
519    /// retarget the pool or change this report field. Never used as identity.
520    pub endpoint: String,
521    /// Whether startup created, recovered, or validated the cluster.
522    pub action: ClusterAction,
523}
524
525/// Schema migration work completed during startup.
526#[derive(Clone, Copy, Debug, PartialEq, Eq)]
527pub enum SchemaStartupOutcome {
528    /// The database already satisfied the release contract.
529    Compatible,
530    /// A previously empty database received the complete Tokeira schema.
531    Initialized,
532    /// A verified older schema advanced to the target version.
533    Migrated,
534    /// Legacy compatibility metadata was backfilled without schema DDL.
535    MetadataBackfilled,
536}
537
538/// Redacted release/schema compatibility evidence.
539#[derive(Clone, Debug, PartialEq, Eq)]
540pub struct SchemaStartupReport {
541    /// Highest observed migration version after policy application.
542    pub observed_version: u32,
543    /// Oldest schema version this release may read.
544    pub minimum_supported_version: u32,
545    /// Automatic migration target for this release.
546    pub target_version: u32,
547    /// Newest schema version this release may read.
548    pub maximum_readable_version: u32,
549    /// Release-pinned migration-set digest; never migration SQL.
550    pub migration_set_digest: String,
551    /// Startup migration outcome.
552    pub outcome: SchemaStartupOutcome,
553}
554
555/// Exclusive ownership acquisition evidence.
556#[derive(Clone, Copy, Debug, PartialEq, Eq)]
557pub struct OwnershipStartupReport {
558    /// Whether the claim was clean or replaced an expired owner.
559    pub outcome: ControlLeaseAcquireOutcome,
560    /// Database-issued monotonic fence token.
561    pub fence_token: i64,
562}
563
564/// Ordered startup phase used by typed, redacted failures.
565#[derive(Clone, Copy, Debug, PartialEq, Eq)]
566pub enum EmbeddedStartupPhase {
567    /// Validate explicit storage intent and ordinary server policy.
568    Configuration,
569    /// Create, recover, or validate the canonical cluster identity.
570    ClusterResolution,
571    /// Establish and warm the bounded process-local DSQL pool.
572    ConnectionWarmup,
573    /// Assess and apply the release-pinned schema policy.
574    Schema,
575    /// Acquire the exclusive embedded owner claim.
576    Ownership,
577    /// Reconstruct runtime state and acquire self-assigned shard leases.
578    RuntimeRestore,
579}
580
581impl std::fmt::Display for EmbeddedStartupPhase {
582    fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
583        formatter.write_str(match self {
584            Self::Configuration => "configuration validation",
585            Self::ClusterResolution => "cluster resolution",
586            Self::ConnectionWarmup => "DSQL connection warmup",
587            Self::Schema => "schema compatibility",
588            Self::Ownership => "embedded ownership",
589            Self::RuntimeRestore => "runtime restoration",
590        })
591    }
592}
593
594/// Redacted failure from explicit embedded startup.
595#[derive(Debug)]
596pub enum EmbeddedEngineStartError {
597    /// A library or executor conflicts with the immutable registry.
598    Registry(tokeira_chasm::ChasmError),
599    /// Persisted executions require a library absent from this deployment.
600    UnregisteredArchetype {
601        /// Persisted root component type.
602        archetype_id: u32,
603        /// Stored executions requiring it.
604        executions: u64,
605    },
606    /// A declared search attribute conflicts with the namespace's registered type.
607    SearchAttributeType {
608        /// Namespace whose registry conflicts.
609        namespace: NamespaceId,
610        /// Declared attribute name.
611        key: String,
612        /// Type required by the library.
613        declared: SearchAttrType,
614        /// Type already registered in projection storage.
615        existing: SearchAttrType,
616    },
617    /// Startup rebuild isolated executions this deployment cannot serve.
618    UnserviceableOutbox {
619        /// Archetype of the first failing execution in scan order.
620        archetype_id: u32,
621        /// Missing data, handler or undecodable payload encountered first.
622        cause: tokeira_runtime::chasm::RebuildFailure,
623        /// Number of executions isolated during the pass.
624        executions: usize,
625    },
626    /// Configuration failed before any external resource was touched.
627    InvalidConfiguration(tokeira_config::EmbeddedConfigError),
628    /// One startup phase failed; the nested cause is deliberately discarded.
629    Phase {
630        /// Phase that did not complete.
631        phase: EmbeddedStartupPhase,
632    },
633    /// The one host-configured startup budget elapsed in the named phase.
634    DeadlineExceeded {
635        /// Phase active when the shared deadline elapsed.
636        phase: EmbeddedStartupPhase,
637    },
638}
639
640impl std::fmt::Display for EmbeddedEngineStartError {
641    fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
642        match self {
643            Self::Registry(error) => write!(
644                formatter,
645                "CHASM registration failed: {error}; correct the conflicting library or executor before restarting"
646            ),
647            Self::UnregisteredArchetype {
648                archetype_id,
649                executions,
650            } => write!(
651                formatter,
652                "CHASM archetype {archetype_id} is not registered but has {executions} stored executions; load its library before restarting"
653            ),
654            Self::SearchAttributeType {
655                namespace,
656                key,
657                declared,
658                existing,
659            } => write!(
660                formatter,
661                "namespace {} search attribute `{key}` declares {declared:?} but storage has {existing:?}; restore the matching declaration or correct the registered type before restarting",
662                namespace.0
663            ),
664            Self::UnserviceableOutbox {
665                archetype_id,
666                cause,
667                executions,
668            } => write!(
669                formatter,
670                "CHASM rebuild cannot serve {executions} executions; first failure in archetype {archetype_id}: {cause}; restore the matching library or repair its persisted data before restarting"
671            ),
672            Self::InvalidConfiguration(error) => {
673                write!(formatter, "invalid embedded engine configuration: {error}")
674            }
675            Self::Phase { phase } => {
676                write!(formatter, "embedded engine startup failed during {phase}")
677            }
678            Self::DeadlineExceeded { phase } => {
679                write!(
680                    formatter,
681                    "embedded engine startup deadline exceeded during {phase}"
682                )
683            }
684        }
685    }
686}
687
688impl std::error::Error for EmbeddedEngineStartError {}
689
690impl From<tokeira_config::EmbeddedConfigError> for EmbeddedEngineStartError {
691    fn from(error: tokeira_config::EmbeddedConfigError) -> Self {
692        Self::InvalidConfiguration(error)
693    }
694}
695
696/// Independent cleanup stage that failed during explicit shutdown.
697#[derive(Clone, Copy, Debug, PartialEq, Eq)]
698pub enum EmbeddedShutdownFailure {
699    /// In-process handlers did not drain by the deadline.
700    RpcDrain,
701    /// Runtime or engine-owned tasks did not join by the deadline.
702    RuntimeTasks,
703    /// A self-assigned shard lease could not be relinquished.
704    ShardRelease,
705    /// The conditional embedded-owner release failed.
706    OwnershipRelease,
707    /// Embedded DSQL pool drain/closure failed.
708    Storage,
709    /// Final in-memory snapshot persistence failed.
710    Snapshot,
711    /// An attached listener did not stop and join by the deadline.
712    ListenerDrain,
713}
714
715/// Aggregated explicit-shutdown failure.
716#[derive(Debug)]
717pub struct EmbeddedEngineShutdownError {
718    failures: Vec<EmbeddedShutdownFailure>,
719}
720
721impl std::fmt::Display for EmbeddedEngineShutdownError {
722    fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
723        write!(
724            formatter,
725            "embedded engine shutdown completed with {} cleanup failure(s)",
726            self.failures.len()
727        )
728    }
729}
730
731impl std::error::Error for EmbeddedEngineShutdownError {}
732
733impl EmbeddedEngineShutdownError {
734    /// Ordered cleanup stages that failed; later cleanup was still attempted.
735    pub fn failures(&self) -> &[EmbeddedShutdownFailure] {
736        &self.failures
737    }
738}
739
740type CleanupFuture = Pin<Box<dyn Future<Output = Result<()>> + Send>>;
741type CleanupAction = Box<dyn FnOnce(Instant) -> CleanupFuture + Send>;
742
743#[derive(Clone)]
744struct EmbeddedOwnership {
745    repository: ConnectionControlLeaseRepository,
746    guard: Arc<tokio::sync::Mutex<Option<ControlLeaseGuard>>>,
747    gate: OwnershipAdmissionGate,
748}
749
750impl std::fmt::Debug for EmbeddedOwnership {
751    fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
752        formatter
753            .debug_struct("EmbeddedOwnership")
754            .field("gate", &self.gate)
755            .finish_non_exhaustive()
756    }
757}
758
759struct EmbeddedShutdownCoordinator {
760    service: InProcessGrpcService,
761    runtime_tasks: RuntimeShutdownHandle,
762    engine_tasks: RuntimeShutdownHandle,
763    shard_cleanup: Option<CleanupAction>,
764    director: Option<Arc<DsqlConnectionDirector>>,
765    ownership: Option<EmbeddedOwnership>,
766}
767
768impl std::fmt::Debug for EmbeddedShutdownCoordinator {
769    fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
770        formatter
771            .debug_struct("EmbeddedShutdownCoordinator")
772            .field("service", &self.service)
773            .field("runtime_tasks", &self.runtime_tasks)
774            .field("engine_tasks", &self.engine_tasks)
775            .field("has_shard_cleanup", &self.shard_cleanup.is_some())
776            .field("has_director", &self.director.is_some())
777            .field("ownership", &self.ownership)
778            .finish()
779    }
780}
781
782#[derive(Debug)]
783struct EngineSnapshotPolicy {
784    store: InMemoryStore,
785    location: PathBuf,
786    periodic_task: Option<JoinHandle<()>>,
787}
788
789static SNAPSHOT_TEMP_SEQUENCE: AtomicU64 = AtomicU64::new(0);
790
791impl EngineSnapshotPolicy {
792    fn start(
793        store: InMemoryStore,
794        config: SnapshotPolicyConfig,
795        cancel: CancellationToken,
796    ) -> Self {
797        let interval_duration = std::time::Duration::from_millis(config.interval_ms);
798        let location = config.location;
799        let periodic_store = store.clone();
800        let periodic_location = location.clone();
801        // Anchor the cadence before spawning. A busy executor may not poll the
802        // task immediately, but elapsed startup time still counts toward the
803        // operator-configured interval.
804        let first_tick = tokio::time::Instant::now() + interval_duration;
805        let periodic_task = tokio::spawn(async move {
806            // The first capture belongs one complete interval after startup. An
807            // immediate tick would turn "every N milliseconds" into an
808            // undocumented snapshot-at-boot policy and add I/O before any state
809            // could have changed.
810            let mut interval = tokio::time::interval_at(first_tick, interval_duration);
811            interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
812            loop {
813                tokio::select! {
814                    biased;
815                    _ = cancel.cancelled() => break,
816                    _ = interval.tick() => {
817                        if let Err(error) = persist_snapshot(&periodic_store, &periodic_location).await {
818                            error!(
819                                snapshot_path = %periodic_location.display(),
820                                error = %error,
821                                "failed to persist periodic engine snapshot"
822                            );
823                        }
824                    }
825                }
826            }
827        });
828        Self {
829            store,
830            location,
831            periodic_task: Some(periodic_task),
832        }
833    }
834}
835
836/// Cloneable raw-protobuf endpoint for an embedded [`Engine`].
837///
838/// This is intentionally SDK-neutral: its request and response are the same wire
839/// fields carried by `temporalio-client::callback_based::GrpcRequest` and
840/// `GrpcSuccessResponse`, but the core endpoint does not expose the SDK's tonic
841/// generation in its public types.
842#[derive(Clone, Debug)]
843pub struct TemporalEndpoint {
844    service: InProcessGrpcService,
845    shutdown: CancellationToken,
846}
847
848impl Engine {
849    /// Discover projected executions for a registered root in one namespace.
850    ///
851    /// The embedder must authorize namespace access before obtaining this handle;
852    /// it bypasses the Temporal transport and its interceptors. Results are
853    /// eventually consistent. Use [`Self::chasm`] for authoritative state/commands.
854    #[cfg(feature = "chasm-extensions")]
855    pub fn chasm_visibility<C>(
856        &self,
857        namespace_id: tokeira_types::NamespaceId,
858    ) -> Result<tokeira_projection::ComponentVisibility, tokeira_chasm::ChasmError>
859    where
860        C: tokeira_chasm::EngineComponent
861            + tokeira_chasm::RootComponent
862            + tokeira_chasm::SearchAttributeProvider
863            + tokeira_chasm::VisibilityContributor,
864    {
865        let archetype_id = self.registry.archetype_id(C::FQN).ok_or_else(|| {
866            tokeira_chasm::ChasmError::Internal(format!("archetype `{}` is not registered", C::FQN))
867        })?;
868        Ok(tokeira_projection::ComponentVisibility::new(
869            self.component_visibility_store.0.clone(),
870            namespace_id,
871            tokeira_types::ArchetypeId(archetype_id),
872        ))
873    }
874    /// Configure unstable CHASM libraries, executors and time before startup.
875    #[cfg(feature = "chasm-extensions")]
876    pub fn builder(config: EmbeddedEngineConfig) -> EngineBuilder {
877        EngineBuilder {
878            config,
879            extensions: ChasmExtensions::default(),
880        }
881    }
882
883    /// Return a typed handle only for a registered component. The built-in
884    /// activity archetype is always reachable, including with no extension libraries.
885    #[cfg(feature = "chasm-extensions")]
886    pub fn chasm<C>(
887        &self,
888    ) -> Result<tokeira_runtime::chasm::TypedEngine<C>, tokeira_chasm::ChasmError>
889    where
890        C: tokeira_chasm::EngineComponent
891            + tokeira_chasm::RootComponent
892            + tokeira_chasm::SearchAttributeProvider
893            + tokeira_chasm::VisibilityContributor,
894    {
895        if self.registry.archetype_id(C::FQN).is_none() {
896            return Err(tokeira_chasm::ChasmError::Internal(format!(
897                "archetype `{}` is not registered",
898                C::FQN
899            )));
900        }
901        Ok(tokeira_runtime::chasm::TypedEngine::new(
902            self.chasm_engine.clone(),
903        ))
904    }
905
906    /// Start a zero-listener engine with the default in-memory configuration.
907    pub async fn start() -> Result<Self> {
908        Self::start_with_config(TokeiraConfig::default()).await
909    }
910
911    /// Alias for [`Self::start`] that makes the operating mode explicit at call sites.
912    pub async fn embedded() -> Result<Self> {
913        Self::start().await
914    }
915
916    /// Start a zero-listener engine with an explicit validated configuration.
917    ///
918    /// Storage is always forced to in-memory: durable external storage and cluster
919    /// placement are daemon concerns, while this facade guarantees process-local
920    /// ownership and no bound sockets.
921    pub async fn start_with_config(config: TokeiraConfig) -> Result<Self> {
922        Self::start_in_memory_with_extensions(config, ChasmExtensions::default()).await
923    }
924
925    async fn start_in_memory_with_extensions(
926        config: TokeiraConfig,
927        extensions: ChasmExtensions,
928    ) -> Result<Self> {
929        let config = embedded_config(config)?;
930        let snapshot_config = config.policy.snapshot.clone();
931        let (store, restored) = restore_snapshot_store(&config).await?;
932        let stack = build_embedded(Arc::new(config), store.clone(), restored, extensions).await?;
933        let snapshot_policy = snapshot_config.map(|config| {
934            EngineSnapshotPolicy::start(store, config, stack.background_cancel.clone())
935        });
936        stack.engine_tasks.close_registration();
937        let shutdown_coordinator = EmbeddedShutdownCoordinator {
938            service: stack.service.clone(),
939            runtime_tasks: stack.runtime_tasks,
940            engine_tasks: stack.engine_tasks,
941            shard_cleanup: stack.shard_cleanup,
942            director: None,
943            ownership: None,
944        };
945        let engine = Self {
946            component_visibility_store: stack.component_visibility_store,
947            chasm_engine: stack.chasm_engine,
948            registry: stack.registry,
949            endpoint: TemporalEndpoint {
950                service: stack.service,
951                shutdown: stack.background_cancel.clone(),
952            },
953            background_cancel: stack.background_cancel,
954            log_broadcast: stack.log_broadcast,
955            snapshot_policy,
956            recovery_task: stack.recovery_task,
957            startup_report: EngineStartupReport::in_memory(),
958            shutdown_coordinator: Some(shutdown_coordinator),
959            listeners: listener::ListenerRegistry::default(),
960        };
961        record_embedded_startup(&engine.startup_report);
962        Ok(engine)
963    }
964
965    /// Start a zero-listener engine from an explicit embedded storage decision.
966    ///
967    /// The legacy [`Self::start`] and [`Self::start_with_config`] methods remain
968    /// in-memory only. This is the sole engine boundary that may create or
969    /// recover managed Aurora DSQL infrastructure.
970    pub async fn start_with_embedded_config(
971        config: EmbeddedEngineConfig,
972    ) -> Result<Self, EmbeddedEngineStartError> {
973        Self::start_with_extensions(config, ChasmExtensions::default()).await
974    }
975
976    async fn start_with_extensions(
977        config: EmbeddedEngineConfig,
978        extensions: ChasmExtensions,
979    ) -> Result<Self, EmbeddedEngineStartError> {
980        if !matches!(config.storage, EmbeddedStorageConfig::InMemory)
981            && config.server.policy.snapshot.is_some()
982        {
983            return Err(EmbeddedEngineStartError::Phase {
984                phase: EmbeddedStartupPhase::Configuration,
985            });
986        }
987        config.validate()?;
988        if matches!(config.storage, EmbeddedStorageConfig::InMemory) {
989            return Self::start_in_memory_with_extensions(config.server, extensions)
990                .await
991                .map_err(embedded_stack_error);
992        }
993        start_embedded_dsql(config, extensions).await
994    }
995
996    /// Return the complete redacted report for the startup that produced this engine.
997    pub fn startup_report(&self) -> &EngineStartupReport {
998        &self.startup_report
999    }
1000
1001    /// Return a cloneable raw-protobuf endpoint for this engine.
1002    pub fn endpoint(&self) -> TemporalEndpoint {
1003        self.endpoint.clone()
1004    }
1005
1006    /// Subscribe to the same narrow RPC-event stream exposed by [`TokeiradHandle`].
1007    pub fn log_sink(&self) -> broadcast::Receiver<LogEvent> {
1008        self.log_broadcast.subscribe()
1009    }
1010
1011    /// Adapt this engine to `ConnectionOptions::service_override`.
1012    ///
1013    /// DNS load balancing must be disabled on the SDK connection because the
1014    /// override performs no name resolution or network I/O.
1015    #[cfg(feature = "temporalio-client")]
1016    pub fn service_override(&self) -> temporalio_client::callback_based::CallbackBasedGrpcService {
1017        self.endpoint.service_override()
1018    }
1019
1020    /// Stop accepting calls, cancel background work, and persist the final snapshot.
1021    ///
1022    /// Attached listeners are stopped and joined first, after in-process
1023    /// admission has closed and the runtime has been signalled but before the
1024    /// in-process drain: from then on no network handler can be admitted, so
1025    /// the coordinator's drain, task join, lease release, ownership release,
1026    /// and storage close run in the same world they did before listeners
1027    /// existed. A listener that does not join by the deadline is reported as
1028    /// [`EmbeddedShutdownFailure::ListenerDrain`] and shutdown continues.
1029    ///
1030    /// When snapshot policy is disabled this retains the previous cancellation-only
1031    /// behaviour. A configured final snapshot failure is returned to the caller: a
1032    /// graceful shutdown must not silently claim durability it did not achieve.
1033    pub async fn shutdown(mut self) -> Result<(), EmbeddedEngineShutdownError> {
1034        let deadline = Instant::now() + StdDuration::from_secs(30);
1035        let mut failures = Vec::new();
1036        if let Some(mut coordinator) = self.shutdown_coordinator.take() {
1037            coordinator.begin_shutdown();
1038            self.listeners.stop_all(deadline, &mut failures).await;
1039            coordinator.shutdown(deadline, &mut failures).await;
1040        } else {
1041            self.listeners.stop_all(deadline, &mut failures).await;
1042            self.background_cancel.cancel();
1043        }
1044        if let Some(task) = self.recovery_task.take()
1045            && task.await.is_err()
1046        {
1047            failures.push(EmbeddedShutdownFailure::ShardRelease);
1048        }
1049        if let Some(policy) = self.snapshot_policy.take()
1050            && finish_snapshot_policy(policy).await.is_err()
1051        {
1052            failures.push(EmbeddedShutdownFailure::Snapshot);
1053        }
1054        record_embedded_shutdown(&self.startup_report, &failures);
1055        if failures.is_empty() {
1056            Ok(())
1057        } else {
1058            Err(EmbeddedEngineShutdownError { failures })
1059        }
1060    }
1061}
1062
1063impl EmbeddedShutdownCoordinator {
1064    fn begin_shutdown(&self) {
1065        self.service.begin_shutdown();
1066        self.engine_tasks.begin_shutdown();
1067        self.runtime_tasks.begin_shutdown();
1068    }
1069
1070    async fn shutdown(&mut self, deadline: Instant, failures: &mut Vec<EmbeddedShutdownFailure>) {
1071        if self.service.drain(deadline).await.is_err() {
1072            failures.push(EmbeddedShutdownFailure::RpcDrain);
1073        }
1074        if self.engine_tasks.wait(deadline).await.is_err() {
1075            failures.push(EmbeddedShutdownFailure::RuntimeTasks);
1076        }
1077        if self.runtime_tasks.wait(deadline).await.is_err() {
1078            failures.push(EmbeddedShutdownFailure::RuntimeTasks);
1079        }
1080        if let Some(cleanup) = self.shard_cleanup.take()
1081            && cleanup(deadline).await.is_err()
1082        {
1083            failures.push(EmbeddedShutdownFailure::ShardRelease);
1084        }
1085        // RPC admission is already closed and both task groups have joined, so
1086        // shard release is the last operation allowed through ordinary owner
1087        // admission. Closing earlier makes that conditional DSQL release unable
1088        // to acquire its bounded Control connection and also prevents already
1089        // admitted RPCs from draining gracefully.
1090        if let Some(ownership) = &self.ownership {
1091            ownership.gate.begin_closing();
1092        }
1093        if let (Some(ownership), Some(director)) = (&self.ownership, &self.director) {
1094            let now = Instant::now();
1095            let ownership_drained = now < deadline
1096                && tokio::time::timeout(
1097                    deadline.saturating_duration_since(now),
1098                    ownership.gate.wait_for_drain(),
1099                )
1100                .await
1101                .is_ok();
1102            if ownership_drained {
1103                let now = Instant::now();
1104                let release_result = if now < deadline {
1105                    tokio::time::timeout(
1106                        deadline.saturating_duration_since(now),
1107                        release_embedded_ownership(ownership, director),
1108                    )
1109                    .await
1110                    .map_err(|_| anyhow!("embedded ownership release deadline elapsed"))
1111                    .and_then(|result| result)
1112                } else {
1113                    Err(anyhow!("embedded ownership release deadline elapsed"))
1114                };
1115                if release_result.is_err() {
1116                    failures.push(EmbeddedShutdownFailure::OwnershipRelease);
1117                }
1118            } else {
1119                // Releasing while a checkout still uses the claim would let a
1120                // clean takeover overlap prior database work. Leave the claim
1121                // to expire instead and report the incomplete cleanup.
1122                failures.push(EmbeddedShutdownFailure::OwnershipRelease);
1123            }
1124        }
1125        if let Some(director) = &self.director
1126            && director.shutdown_with_deadline(deadline).await.is_err()
1127        {
1128            failures.push(EmbeddedShutdownFailure::Storage);
1129        }
1130    }
1131}
1132
1133async fn start_embedded_dsql(
1134    config: EmbeddedEngineConfig,
1135    extensions: ChasmExtensions,
1136) -> Result<Engine, EmbeddedEngineStartError> {
1137    let deadline = Instant::now() + StdDuration::from_millis(config.startup_timeout_ms);
1138    let storage_mode = match &config.storage {
1139        EmbeddedStorageConfig::ManagedDsql(_) => EmbeddedStorageMode::ManagedDsql,
1140        EmbeddedStorageConfig::ExistingDsql(_) => EmbeddedStorageMode::ExistingDsql,
1141        EmbeddedStorageConfig::InMemory => unreachable!("in-memory handled by caller"),
1142    };
1143    let region = match &config.storage {
1144        EmbeddedStorageConfig::ManagedDsql(managed) => managed.region.clone(),
1145        EmbeddedStorageConfig::ExistingDsql(existing) => existing.region.clone(),
1146        EmbeddedStorageConfig::InMemory => unreachable!("in-memory handled by caller"),
1147    };
1148    let control = startup_infallible_phase(
1149        deadline,
1150        EmbeddedStartupPhase::ClusterResolution,
1151        AwsDsqlControlPlane::from_region(region.clone()),
1152    )
1153    .await?;
1154    let migration_policy = config
1155        .effective_migration_policy()
1156        .expect("durable embedded mode always has a migration policy");
1157    let storage = config.storage.clone();
1158    let ConnectedEmbeddedDsql {
1159        server,
1160        cluster,
1161        connection_endpoint,
1162        store: dsql_store,
1163        wake_required,
1164    } = connect_embedded_dsql(
1165        config,
1166        control.clone(),
1167        SystemLifecycleEnvironment,
1168        deadline,
1169        DsqlStore::connect_embedded,
1170    )
1171    .await?;
1172    let lifecycle_deadline = StartupDeadline::at(deadline);
1173    let director = dsql_store.connection_director_arc();
1174
1175    let cluster = if wake_required {
1176        match refresh_cluster_after_wake(
1177            &storage,
1178            control,
1179            SystemLifecycleEnvironment,
1180            cluster,
1181            lifecycle_deadline,
1182            deadline,
1183        )
1184        .await
1185        {
1186            Ok(cluster) => cluster,
1187            Err(error) => {
1188                let _ = director.shutdown_with_deadline(deadline).await;
1189                return Err(error);
1190            }
1191        }
1192    } else {
1193        cluster
1194    };
1195
1196    let schema = match startup_phase(
1197        deadline,
1198        EmbeddedStartupPhase::Schema,
1199        apply_embedded_schema(
1200            director.clone(),
1201            &cluster,
1202            schema_migration_policy(migration_policy),
1203            deadline,
1204        ),
1205    )
1206    .await
1207    {
1208        Ok(report) => report,
1209        Err(error) => {
1210            let _ = director.shutdown_with_deadline(deadline).await;
1211            return Err(error);
1212        }
1213    };
1214
1215    let (ownership, ownership_report) = match startup_phase(
1216        deadline,
1217        EmbeddedStartupPhase::Ownership,
1218        acquire_embedded_ownership(director.clone(), &cluster, deadline),
1219    )
1220    .await
1221    {
1222        Ok(ownership) => ownership,
1223        Err(error) => {
1224            let _ = director.shutdown_with_deadline(deadline).await;
1225            return Err(error);
1226        }
1227    };
1228    if director
1229        .install_ownership_gate(ownership.gate.clone())
1230        .is_err()
1231    {
1232        rollback_embedded_dsql(&ownership, &director, deadline).await;
1233        return Err(EmbeddedEngineStartError::Phase {
1234            phase: EmbeddedStartupPhase::Ownership,
1235        });
1236    }
1237
1238    let stack =
1239        match startup_infallible_phase(deadline, EmbeddedStartupPhase::RuntimeRestore, async {
1240            build_dsql_stack(
1241                StackTransport::Embedded,
1242                Arc::new(server),
1243                dsql_store,
1244                connection_endpoint.clone(),
1245                extensions,
1246            )
1247            .await
1248            .map_err(embedded_stack_error)
1249        })
1250        .await
1251        .and_then(|result| result)
1252        {
1253            Ok(ConstructedStack::Embedded(stack)) => stack,
1254            Ok(ConstructedStack::Network(_)) => {
1255                rollback_embedded_dsql(&ownership, &director, deadline).await;
1256                return Err(EmbeddedEngineStartError::Phase {
1257                    phase: EmbeddedStartupPhase::RuntimeRestore,
1258                });
1259            }
1260            Err(error) => {
1261                rollback_embedded_dsql(&ownership, &director, deadline).await;
1262                return Err(error);
1263            }
1264        };
1265
1266    spawn_ownership_renewal(
1267        &stack.engine_tasks,
1268        director.clone(),
1269        ownership.clone(),
1270        stack.service.clone(),
1271    );
1272    stack.engine_tasks.close_registration();
1273    let report = EngineStartupReport {
1274        storage_mode,
1275        cluster: Some(cluster_startup_report(&cluster, &connection_endpoint)),
1276        schema: Some(schema),
1277        ownership: Some(ownership_report),
1278    };
1279    let coordinator = EmbeddedShutdownCoordinator {
1280        service: stack.service.clone(),
1281        runtime_tasks: stack.runtime_tasks,
1282        engine_tasks: stack.engine_tasks,
1283        shard_cleanup: stack.shard_cleanup,
1284        director: Some(director),
1285        ownership: Some(ownership),
1286    };
1287    let engine = Engine {
1288        component_visibility_store: stack.component_visibility_store,
1289        chasm_engine: stack.chasm_engine,
1290        registry: stack.registry,
1291        endpoint: TemporalEndpoint {
1292            service: stack.service,
1293            shutdown: stack.background_cancel.clone(),
1294        },
1295        background_cancel: stack.background_cancel,
1296        log_broadcast: stack.log_broadcast,
1297        snapshot_policy: None,
1298        recovery_task: stack.recovery_task,
1299        startup_report: report,
1300        shutdown_coordinator: Some(coordinator),
1301        listeners: listener::ListenerRegistry::default(),
1302    };
1303    record_embedded_startup(&engine.startup_report);
1304    Ok(engine)
1305}
1306
1307// Keep the AWS observation separate from the locator captured by the pool's
1308// connection factory. Readiness and wake polling may refresh that observation;
1309// they cannot retarget connections already established by this generation.
1310#[derive(Debug)]
1311struct ConnectedEmbeddedDsql<S> {
1312    server: TokeiraConfig,
1313    cluster: ResolvedCluster,
1314    connection_endpoint: String,
1315    store: S,
1316    wake_required: bool,
1317}
1318
1319// The injected control plane, clock, and connector exercise the actual startup
1320// wiring offline without replacing identity validation or readiness decisions.
1321async fn connect_embedded_dsql<C, T, F, Fut, S>(
1322    config: EmbeddedEngineConfig,
1323    control: C,
1324    time: T,
1325    deadline: Instant,
1326    connect: F,
1327) -> Result<ConnectedEmbeddedDsql<S>, EmbeddedEngineStartError>
1328where
1329    C: DsqlControlPlane + Clone,
1330    T: LifecycleEnvironment + Clone,
1331    F: FnOnce(DsqlAuthConfig, EmbeddedDsqlPoolConfig, WarmupDeadline) -> Fut,
1332    Fut: Future<Output = Result<S>>,
1333{
1334    let lifecycle_deadline = StartupDeadline::at(deadline);
1335    let resolved = match &config.storage {
1336        EmbeddedStorageConfig::ManagedDsql(managed) => {
1337            let lifecycle = ManagedDsqlLifecycle::new(
1338                control.clone(),
1339                LocalClusterDescriptorStore::new(&managed.descriptor_path),
1340                time.clone(),
1341            );
1342            startup_phase(
1343                deadline,
1344                EmbeddedStartupPhase::ClusterResolution,
1345                lifecycle.create_or_recover(
1346                    CreateOrRecoverRequest {
1347                        region: managed.region.clone(),
1348                        tags: managed.tags.clone(),
1349                    },
1350                    lifecycle_deadline,
1351                ),
1352            )
1353            .await?
1354        }
1355        EmbeddedStorageConfig::ExistingDsql(existing) => {
1356            // `resolve_existing` cannot touch the descriptor seam. The inert
1357            // path is never opened and exists only to satisfy the lifecycle's
1358            // generic state-store type without adding another public adapter.
1359            let unused_descriptor = LocalClusterDescriptorStore::new(PathBuf::new());
1360            let lifecycle =
1361                ManagedDsqlLifecycle::new(control.clone(), unused_descriptor, time.clone());
1362            let identity = CanonicalClusterIdentity::new(
1363                &existing.region,
1364                &existing.cluster_id,
1365                &existing.cluster_arn,
1366            )
1367            .map_err(|_| EmbeddedEngineStartError::Phase {
1368                phase: EmbeddedStartupPhase::ClusterResolution,
1369            })?;
1370            startup_phase(
1371                deadline,
1372                EmbeddedStartupPhase::ClusterResolution,
1373                lifecycle.resolve_existing(identity, lifecycle_deadline),
1374            )
1375            .await?
1376        }
1377        EmbeddedStorageConfig::InMemory => unreachable!("in-memory handled by caller"),
1378    };
1379    let readiness = refresh_cluster_until_storage_handoff(
1380        &config.storage,
1381        control.clone(),
1382        time,
1383        resolved,
1384        lifecycle_deadline,
1385        deadline,
1386    )
1387    .await?;
1388    let cluster = match &readiness {
1389        Readiness::Active(usable) => usable.resolved().clone(),
1390        Readiness::WakeRequired(cluster) => cluster.clone(),
1391    };
1392
1393    // GetCluster validates the canonical resource and status, but its public
1394    // endpoint cannot select the caller's network path. In particular, AWS's
1395    // PrivateLink database hostname is distinct from the management observation
1396    // (Aurora DSQL User Guide, "Managing and connecting ... using AWS PrivateLink").
1397    // Capture the locator once: the connector uses it for IAM signing, TLS, and
1398    // every replenished connection, including the initial scale-to-zero wake.
1399    let connection_endpoint = match &config.storage {
1400        EmbeddedStorageConfig::ExistingDsql(existing) => existing.endpoint.clone(),
1401        EmbeddedStorageConfig::ManagedDsql(_) => cluster.endpoint.clone(),
1402        EmbeddedStorageConfig::InMemory => unreachable!("in-memory handled by caller"),
1403    };
1404    let mut server = config.server;
1405    server.infrastructure.storage = ConfigStorageKind::Dsql;
1406    server.infrastructure.placement.controller_endpoint = None;
1407    server.infrastructure.dsql.endpoint = Some(connection_endpoint.clone());
1408    server.infrastructure.dsql.region = Some(cluster.identity.region.clone());
1409    server
1410        .validate()
1411        .map_err(|_| EmbeddedEngineStartError::Phase {
1412            phase: EmbeddedStartupPhase::Configuration,
1413        })?;
1414
1415    let limits = match &config.storage {
1416        EmbeddedStorageConfig::ManagedDsql(managed) => &managed.limits,
1417        EmbeddedStorageConfig::ExistingDsql(existing) => &existing.limits,
1418        EmbeddedStorageConfig::InMemory => unreachable!("in-memory handled by caller"),
1419    };
1420    let mut pool_config = EmbeddedDsqlPoolConfig::with_limits(
1421        limits.max_connections,
1422        limits.concurrent_connection_creations,
1423        limits.connection_rate_per_second,
1424        limits.connection_burst,
1425    );
1426    pool_config.shard_count = server.infrastructure.placement.shard_count;
1427    pool_config.projection_partition_count = server.infrastructure.placement.partition_count;
1428    let auth = DsqlAuthConfig {
1429        endpoint: connection_endpoint.clone(),
1430        region: Some(cluster.identity.region.clone()),
1431        admin_role_arn: server.infrastructure.dsql.admin_role_arn.clone(),
1432        runtime_role_arn: server.infrastructure.dsql.runtime_role_arn.clone(),
1433        readonly_role_arn: server.infrastructure.dsql.readonly_role_arn.clone(),
1434    };
1435    let store = startup_phase(
1436        deadline,
1437        EmbeddedStartupPhase::ConnectionWarmup,
1438        connect(auth, pool_config, WarmupDeadline::new(deadline)),
1439    )
1440    .await?;
1441    Ok(ConnectedEmbeddedDsql {
1442        server,
1443        cluster,
1444        connection_endpoint,
1445        store,
1446        wake_required: matches!(readiness, Readiness::WakeRequired(_)),
1447    })
1448}
1449
1450fn cluster_startup_report(
1451    cluster: &ResolvedCluster,
1452    connection_endpoint: &str,
1453) -> ClusterStartupReport {
1454    ClusterStartupReport {
1455        region: cluster.identity.region.clone(),
1456        cluster_id: cluster.identity.cluster_id.clone(),
1457        cluster_arn: cluster.identity.cluster_arn.clone(),
1458        endpoint: connection_endpoint.to_owned(),
1459        action: cluster.action,
1460    }
1461}
1462
1463fn storage_mode_label(mode: EmbeddedStorageMode) -> EmbeddedStorageModeLabel {
1464    match mode {
1465        EmbeddedStorageMode::InMemory => EmbeddedStorageModeLabel::InMemory,
1466        EmbeddedStorageMode::ManagedDsql => EmbeddedStorageModeLabel::ManagedDsql,
1467        EmbeddedStorageMode::ExistingDsql => EmbeddedStorageModeLabel::ExistingDsql,
1468    }
1469}
1470
1471fn schema_outcome_label(report: &EngineStartupReport) -> SchemaOutcomeLabel {
1472    match report.schema.as_ref().map(|schema| schema.outcome) {
1473        None => SchemaOutcomeLabel::NotApplicable,
1474        Some(SchemaStartupOutcome::Compatible) => SchemaOutcomeLabel::Compatible,
1475        Some(SchemaStartupOutcome::Initialized) => SchemaOutcomeLabel::Initialized,
1476        Some(SchemaStartupOutcome::Migrated) => SchemaOutcomeLabel::Migrated,
1477        Some(SchemaStartupOutcome::MetadataBackfilled) => SchemaOutcomeLabel::MetadataBackfilled,
1478    }
1479}
1480
1481fn ownership_outcome_label(report: &EngineStartupReport) -> OwnershipOutcomeLabel {
1482    match report.ownership.map(|ownership| ownership.outcome) {
1483        None => OwnershipOutcomeLabel::NotApplicable,
1484        Some(ControlLeaseAcquireOutcome::Clean) => OwnershipOutcomeLabel::AcquiredClean,
1485        Some(ControlLeaseAcquireOutcome::ExpiredTakeover) => OwnershipOutcomeLabel::AcquiredExpired,
1486    }
1487}
1488
1489/// Emit only bounded lifecycle dimensions through the host's current recorder.
1490///
1491/// This is deliberately downstream of successful construction. The metrics
1492/// facade is observational: it does not install a recorder, hold an exporter,
1493/// bind a listener, or participate in the startup decision.
1494fn record_embedded_startup(report: &EngineStartupReport) {
1495    record_embedded_lifecycle(
1496        storage_mode_label(report.storage_mode),
1497        if report.cluster.is_some() {
1498            ClusterStatusLabel::Active
1499        } else {
1500            ClusterStatusLabel::NotApplicable
1501        },
1502        schema_outcome_label(report),
1503        ownership_outcome_label(report),
1504        DbClassLabel::Control,
1505        EmbeddedOperationLabel::Startup,
1506        ErrorClassLabel::None,
1507    );
1508}
1509
1510fn record_embedded_shutdown(report: &EngineStartupReport, failures: &[EmbeddedShutdownFailure]) {
1511    let ownership = if report.ownership.is_some() && failures.is_empty() {
1512        OwnershipOutcomeLabel::Released
1513    } else {
1514        ownership_outcome_label(report)
1515    };
1516    record_embedded_lifecycle(
1517        storage_mode_label(report.storage_mode),
1518        if report.cluster.is_some() {
1519            ClusterStatusLabel::Active
1520        } else {
1521            ClusterStatusLabel::NotApplicable
1522        },
1523        schema_outcome_label(report),
1524        ownership,
1525        DbClassLabel::Control,
1526        EmbeddedOperationLabel::Shutdown,
1527        if failures.is_empty() {
1528            ErrorClassLabel::None
1529        } else {
1530            ErrorClassLabel::Internal
1531        },
1532    );
1533}
1534
1535async fn refresh_cluster_until_storage_handoff<C, T>(
1536    storage: &EmbeddedStorageConfig,
1537    control: C,
1538    time: T,
1539    cluster: ResolvedCluster,
1540    lifecycle_deadline: StartupDeadline,
1541    deadline: Instant,
1542) -> Result<Readiness, EmbeddedEngineStartError>
1543where
1544    C: DsqlControlPlane,
1545    T: LifecycleEnvironment,
1546{
1547    match storage {
1548        EmbeddedStorageConfig::ManagedDsql(managed) => {
1549            let lifecycle = ManagedDsqlLifecycle::new(
1550                control,
1551                LocalClusterDescriptorStore::new(&managed.descriptor_path),
1552                time,
1553            );
1554            startup_phase(
1555                deadline,
1556                EmbeddedStartupPhase::ClusterResolution,
1557                lifecycle.refresh_until_usable(cluster, lifecycle_deadline),
1558            )
1559            .await
1560        }
1561        EmbeddedStorageConfig::ExistingDsql(_) => {
1562            let lifecycle = ManagedDsqlLifecycle::new(
1563                control,
1564                LocalClusterDescriptorStore::new(PathBuf::new()),
1565                time,
1566            );
1567            startup_phase(
1568                deadline,
1569                EmbeddedStartupPhase::ClusterResolution,
1570                lifecycle.refresh_until_usable(cluster, lifecycle_deadline),
1571            )
1572            .await
1573        }
1574        EmbeddedStorageConfig::InMemory => unreachable!("in-memory handled by caller"),
1575    }
1576}
1577
1578async fn refresh_cluster_after_wake<C, T>(
1579    storage: &EmbeddedStorageConfig,
1580    control: C,
1581    time: T,
1582    cluster: ResolvedCluster,
1583    lifecycle_deadline: StartupDeadline,
1584    deadline: Instant,
1585) -> Result<ResolvedCluster, EmbeddedEngineStartError>
1586where
1587    C: DsqlControlPlane,
1588    T: LifecycleEnvironment,
1589{
1590    let usable = match storage {
1591        EmbeddedStorageConfig::ManagedDsql(managed) => {
1592            let lifecycle = ManagedDsqlLifecycle::new(
1593                control,
1594                LocalClusterDescriptorStore::new(&managed.descriptor_path),
1595                time,
1596            );
1597            startup_phase(
1598                deadline,
1599                EmbeddedStartupPhase::ConnectionWarmup,
1600                lifecycle.refresh_after_wake(cluster, lifecycle_deadline),
1601            )
1602            .await?
1603        }
1604        EmbeddedStorageConfig::ExistingDsql(_) => {
1605            let lifecycle = ManagedDsqlLifecycle::new(
1606                control,
1607                LocalClusterDescriptorStore::new(PathBuf::new()),
1608                time,
1609            );
1610            startup_phase(
1611                deadline,
1612                EmbeddedStartupPhase::ConnectionWarmup,
1613                lifecycle.refresh_after_wake(cluster, lifecycle_deadline),
1614            )
1615            .await?
1616        }
1617        EmbeddedStorageConfig::InMemory => unreachable!("in-memory handled by caller"),
1618    };
1619    Ok(usable.into_resolved())
1620}
1621
1622fn schema_migration_policy(policy: DsqlMigrationPolicy) -> SchemaMigrationPolicy {
1623    match policy {
1624        DsqlMigrationPolicy::Automatic => SchemaMigrationPolicy::Automatic,
1625        DsqlMigrationPolicy::ValidateOnly => SchemaMigrationPolicy::ValidateOnly,
1626    }
1627}
1628
1629async fn apply_embedded_schema(
1630    director: Arc<DsqlConnectionDirector>,
1631    cluster: &ResolvedCluster,
1632    policy: SchemaMigrationPolicy,
1633    deadline: Instant,
1634) -> Result<SchemaStartupReport> {
1635    let runner = MigrationRunner::embedded();
1636    let contract = MigrationRunner::compatibility_contract();
1637    let leases = ConnectionControlLeaseRepository::new();
1638    let mut permit = director.acquire(DbClass::Control).await?;
1639    let decision = runner
1640        .assess_connection(permit.connection()?, &contract, policy)
1641        .await?;
1642    let outcome = match &decision {
1643        SchemaDecision::Compatible {
1644            legacy_backfill: false,
1645            ..
1646        } => SchemaStartupOutcome::Compatible,
1647        SchemaDecision::Compatible {
1648            legacy_backfill: true,
1649            ..
1650        } => SchemaStartupOutcome::MetadataBackfilled,
1651        SchemaDecision::Initialize { .. } => SchemaStartupOutcome::Initialized,
1652        SchemaDecision::Migrate { .. } => SchemaStartupOutcome::Migrated,
1653        SchemaDecision::MigrationRequired { current, target } => {
1654            return Err(anyhow!(
1655                "schema migration required from V{current} to V{target}"
1656            ));
1657        }
1658        SchemaDecision::Reject(_) => return Err(anyhow!("schema compatibility rejected")),
1659    };
1660    if !matches!(outcome, SchemaStartupOutcome::Compatible) {
1661        runner
1662            .bootstrap_migration_coordination(permit.connection()?, &decision)
1663            .await?;
1664        leases.bootstrap(permit.connection()?).await?;
1665        let mut migration_guard = leases
1666            .acquire(
1667                permit.connection()?,
1668                &ControlLeaseAcquireRequest {
1669                    claim_name: "schema-migration".to_owned(),
1670                    cluster: control_lease_identity(cluster),
1671                    owner_id: format!("schema-{}", IncarnationId::new()),
1672                    lease_duration: SCHEMA_MIGRATION_LEASE_DURATION,
1673                    admission_margin: SCHEMA_MIGRATION_ADMISSION_MARGIN,
1674                    acquire_deadline: deadline,
1675                },
1676            )
1677            .await?;
1678        let migration_gate = OwnershipAdmissionGate::for_guard(&migration_guard);
1679        let apply_result = runner
1680            .apply_decision(
1681                permit.connection()?,
1682                &decision,
1683                &leases,
1684                &mut migration_guard,
1685                &migration_gate,
1686            )
1687            .await;
1688        let release_result = leases
1689            .release(permit.connection()?, &migration_guard, &migration_gate)
1690            .await;
1691        apply_result?;
1692        release_result?;
1693    }
1694    let final_decision = runner
1695        .assess_connection(
1696            permit.connection()?,
1697            &contract,
1698            SchemaMigrationPolicy::ValidateOnly,
1699        )
1700        .await?;
1701    let observed_version = match final_decision {
1702        SchemaDecision::Compatible { current, .. } => current,
1703        _ => return Err(anyhow!("schema did not converge to a readable version")),
1704    };
1705    Ok(SchemaStartupReport {
1706        observed_version,
1707        minimum_supported_version: contract.minimum_supported_version,
1708        target_version: contract.target_version,
1709        maximum_readable_version: contract.maximum_readable_version,
1710        migration_set_digest: contract.migration_set_digest,
1711        outcome,
1712    })
1713}
1714
1715async fn acquire_embedded_ownership(
1716    director: Arc<DsqlConnectionDirector>,
1717    cluster: &ResolvedCluster,
1718    deadline: Instant,
1719) -> Result<(EmbeddedOwnership, OwnershipStartupReport)> {
1720    let repository = ConnectionControlLeaseRepository::new();
1721    let mut permit = director.acquire(DbClass::Control).await?;
1722    repository.bootstrap(permit.connection()?).await?;
1723    let guard = repository
1724        .acquire(
1725            permit.connection()?,
1726            &ControlLeaseAcquireRequest {
1727                claim_name: "embedded-owner".to_owned(),
1728                cluster: control_lease_identity(cluster),
1729                owner_id: format!("engine-{}", IncarnationId::new()),
1730                lease_duration: OWNER_LEASE_DURATION,
1731                admission_margin: OWNER_ADMISSION_MARGIN,
1732                acquire_deadline: deadline,
1733            },
1734        )
1735        .await?;
1736    let gate = OwnershipAdmissionGate::for_guard(&guard);
1737    if let Some(quiescence_deadline) = guard.quiescence_deadline() {
1738        let now = Instant::now();
1739        if now >= deadline || quiescence_deadline > deadline {
1740            return Err(anyhow!("expired owner quiescence exceeds startup deadline"));
1741        }
1742        tokio::time::sleep_until(tokio::time::Instant::from_std(quiescence_deadline)).await;
1743        gate.finish_quiescence(&guard, Instant::now())?;
1744    }
1745    let report = OwnershipStartupReport {
1746        outcome: guard.outcome(),
1747        fence_token: guard.fence_token(),
1748    };
1749    Ok((
1750        EmbeddedOwnership {
1751            repository,
1752            guard: Arc::new(tokio::sync::Mutex::new(Some(guard))),
1753            gate,
1754        },
1755        report,
1756    ))
1757}
1758
1759fn spawn_ownership_renewal(
1760    tasks: &RuntimeShutdownHandle,
1761    director: Arc<DsqlConnectionDirector>,
1762    ownership: EmbeddedOwnership,
1763    service: InProcessGrpcService,
1764) {
1765    let cancel = tasks.cancellation_token();
1766    let _renewal = tasks.spawn(async move {
1767        let interval_duration =
1768            StdDuration::try_from(OWNER_RENEW_INTERVAL).expect("positive owner renewal interval");
1769        let mut interval = tokio::time::interval(interval_duration);
1770        interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
1771        interval.tick().await;
1772        loop {
1773            tokio::select! {
1774                _ = cancel.cancelled() => break,
1775                _ = interval.tick() => {
1776                    let mut guard_slot = ownership.guard.lock().await;
1777                    let Some(guard) = guard_slot.as_mut() else {
1778                        break;
1779                    };
1780                    let result = match director.acquire(DbClass::Control).await {
1781                        Ok(mut permit) => match permit.connection() {
1782                            Ok(connection) => ownership.repository.renew(
1783                                    connection,
1784                                    guard,
1785                                    OWNER_LEASE_DURATION,
1786                                    OWNER_ADMISSION_MARGIN,
1787                                    &ownership.gate,
1788                                )
1789                                .await
1790                                .map_err(anyhow::Error::from),
1791                            Err(error) => Err(error),
1792                        },
1793                        Err(error) => Err(error),
1794                    };
1795                    if let Err(error) = result {
1796                        if matches!(
1797                            error.downcast_ref::<ControlLeaseError>(),
1798                            Some(ControlLeaseError::Fenced)
1799                        ) {
1800                            ownership.gate.fence();
1801                        } else {
1802                            guard.enforce_admission_deadline(Instant::now(), &ownership.gate);
1803                        }
1804                    }
1805                    if ownership.gate.state() != OwnershipAdmissionState::Open {
1806                        service.begin_shutdown();
1807                        break;
1808                    }
1809                }
1810            }
1811        }
1812    });
1813}
1814
1815fn control_lease_identity(cluster: &ResolvedCluster) -> ControlLeaseClusterIdentity {
1816    ControlLeaseClusterIdentity {
1817        cluster_id: cluster.identity.cluster_id.clone(),
1818        cluster_arn: cluster.identity.cluster_arn.clone(),
1819    }
1820}
1821
1822async fn rollback_embedded_dsql(
1823    ownership: &EmbeddedOwnership,
1824    director: &Arc<DsqlConnectionDirector>,
1825    deadline: Instant,
1826) {
1827    ownership.gate.begin_closing();
1828    let now = Instant::now();
1829    if now < deadline {
1830        let _ = tokio::time::timeout(
1831            deadline.saturating_duration_since(now),
1832            release_embedded_ownership(ownership, director),
1833        )
1834        .await;
1835    }
1836    let _ = director.shutdown_with_deadline(deadline).await;
1837}
1838
1839async fn release_embedded_ownership(
1840    ownership: &EmbeddedOwnership,
1841    director: &Arc<DsqlConnectionDirector>,
1842) -> Result<()> {
1843    // Ordinary owner admission is closed here. This shutdown-only director
1844    // seam still consumes the Control budget and one bounded physical slot.
1845    let mut permit = director.acquire_shutdown_control().await?;
1846    let mut guard = ownership.guard.lock().await;
1847    if let Some(guard) = guard.take() {
1848        ownership
1849            .repository
1850            .release(permit.connection()?, &guard, &ownership.gate)
1851            .await?;
1852    }
1853    Ok(())
1854}
1855
1856async fn startup_phase<T, E, F>(
1857    deadline: Instant,
1858    phase: EmbeddedStartupPhase,
1859    future: F,
1860) -> Result<T, EmbeddedEngineStartError>
1861where
1862    F: Future<Output = Result<T, E>>,
1863{
1864    let now = Instant::now();
1865    if now >= deadline {
1866        return Err(EmbeddedEngineStartError::DeadlineExceeded { phase });
1867    }
1868    match tokio::time::timeout(deadline.saturating_duration_since(now), future).await {
1869        Ok(Ok(value)) => Ok(value),
1870        Ok(Err(_)) => Err(EmbeddedEngineStartError::Phase { phase }),
1871        Err(_) => Err(EmbeddedEngineStartError::DeadlineExceeded { phase }),
1872    }
1873}
1874
1875async fn startup_infallible_phase<T, F>(
1876    deadline: Instant,
1877    phase: EmbeddedStartupPhase,
1878    future: F,
1879) -> Result<T, EmbeddedEngineStartError>
1880where
1881    F: Future<Output = T>,
1882{
1883    let now = Instant::now();
1884    if now >= deadline {
1885        return Err(EmbeddedEngineStartError::DeadlineExceeded { phase });
1886    }
1887    tokio::time::timeout(deadline.saturating_duration_since(now), future)
1888        .await
1889        .map_err(|_| EmbeddedEngineStartError::DeadlineExceeded { phase })
1890}
1891
1892/// Drain a cancelled snapshot policy: await its periodic task, then persist
1893/// the final consistent cut.
1894///
1895/// Shared by the embedded facade and the listener-backed daemon so both
1896/// serving modes end with the identical durability sequence. The caller must
1897/// already have cancelled the policy's cancellation token.
1898async fn finish_snapshot_policy(mut policy: EngineSnapshotPolicy) -> Result<()> {
1899    if let Some(task) = policy.periodic_task.take() {
1900        task.await
1901            .context("engine snapshot task panicked during shutdown")?;
1902    }
1903    // Background workers observe the same token. Yield once so
1904    // cancellation-aware workers that are already runnable can leave their
1905    // select loops before the final consistent cut and before the caller
1906    // tears down its Tokio runtime.
1907    tokio::task::yield_now().await;
1908    persist_snapshot(&policy.store, &policy.location)
1909        .await
1910        .with_context(|| {
1911            format!(
1912                "failed to persist final engine snapshot to {}",
1913                policy.location.display()
1914            )
1915        })
1916}
1917
1918async fn restore_snapshot_store(config: &TokeiraConfig) -> Result<(InMemoryStore, bool)> {
1919    let Some(snapshot) = config.policy.snapshot.as_ref() else {
1920        return Ok((InMemoryStore::default(), false));
1921    };
1922    match fs::read(&snapshot.location).await {
1923        Ok(bytes) => {
1924            let store = InMemoryStore::from_snapshot(&bytes).with_context(|| {
1925                format!(
1926                    "failed to restore engine snapshot from {}",
1927                    snapshot.location.display()
1928                )
1929            })?;
1930            retire_snapshot_leases(&store).await.with_context(|| {
1931                format!(
1932                    "failed to retire prior embedded-engine leases from {}",
1933                    snapshot.location.display()
1934                )
1935            })?;
1936            info!(
1937                snapshot_path = %snapshot.location.display(),
1938                "restored engine snapshot"
1939            );
1940            Ok((store, true))
1941        }
1942        Err(error) if error.kind() == std::io::ErrorKind::NotFound => {
1943            Ok((InMemoryStore::default(), false))
1944        }
1945        Err(error) => Err(error).with_context(|| {
1946            format!(
1947                "failed to read engine snapshot from {}",
1948                snapshot.location.display()
1949            )
1950        }),
1951    }
1952}
1953
1954async fn retire_snapshot_leases(store: &InMemoryStore) -> Result<()> {
1955    for lease in store
1956        .list_bundle_leases()
1957        .await
1958        .context("failed to list engine snapshot leases")?
1959    {
1960        let Some(owner) = lease.owner_node_id else {
1961            continue;
1962        };
1963        match store
1964            .relinquish_bundle(lease.bundle_id, owner, lease.epoch)
1965            .await
1966            .context("failed to relinquish an engine snapshot lease")?
1967        {
1968            LeaseOutcome::Acquired { .. } => {}
1969            LeaseOutcome::Rejected {
1970                current_owner,
1971                current_epoch,
1972            } => {
1973                return Err(anyhow!(
1974                    "engine snapshot lease retirement was fenced by owner {current_owner} at epoch {}",
1975                    current_epoch.0
1976                ));
1977            }
1978            LeaseOutcome::Renewed { epoch } => {
1979                return Err(anyhow!(
1980                    "engine snapshot lease retirement unexpectedly renewed epoch {}",
1981                    epoch.0
1982                ));
1983            }
1984        }
1985    }
1986    Ok(())
1987}
1988
1989async fn persist_snapshot(store: &InMemoryStore, location: &Path) -> Result<()> {
1990    let bytes = store
1991        .snapshot()
1992        .await
1993        .context("failed to capture engine snapshot")?;
1994    write_snapshot_atomically(location, &bytes).await
1995}
1996
1997async fn write_snapshot_atomically(location: &Path, bytes: &[u8]) -> Result<()> {
1998    if let Some(parent) = location
1999        .parent()
2000        .filter(|parent| !parent.as_os_str().is_empty())
2001    {
2002        fs::create_dir_all(parent).await.with_context(|| {
2003            format!(
2004                "failed to create engine snapshot directory {}",
2005                parent.display()
2006            )
2007        })?;
2008    }
2009
2010    // A same-directory temporary keeps rename atomic on the target filesystem.
2011    // The process/sequence suffix prevents two embedded engines in one test
2012    // process from sharing a staging file even if they target the same snapshot.
2013    let sequence = SNAPSHOT_TEMP_SEQUENCE.fetch_add(1, Ordering::Relaxed);
2014    let mut temporary_name = location.as_os_str().to_os_string();
2015    temporary_name.push(format!(".tmp-{}-{sequence}", std::process::id()));
2016    let temporary = PathBuf::from(temporary_name);
2017
2018    let write_result: Result<()> = async {
2019        let mut file = OpenOptions::new()
2020            .create(true)
2021            .truncate(true)
2022            .write(true)
2023            .open(&temporary)
2024            .await
2025            .with_context(|| {
2026                format!(
2027                    "failed to create temporary engine snapshot {}",
2028                    temporary.display()
2029                )
2030            })?;
2031        file.write_all(bytes).await.with_context(|| {
2032            format!(
2033                "failed to write temporary engine snapshot {}",
2034                temporary.display()
2035            )
2036        })?;
2037        file.sync_all().await.with_context(|| {
2038            format!(
2039                "failed to sync temporary engine snapshot {}",
2040                temporary.display()
2041            )
2042        })?;
2043        drop(file);
2044        fs::rename(&temporary, location)
2045            .await
2046            .with_context(|| format!("failed to replace engine snapshot {}", location.display()))?;
2047        Ok(())
2048    }
2049    .await;
2050
2051    if write_result.is_err() {
2052        // Cleanup is best-effort and intentionally targets only the unique
2053        // staging file created above; the prior complete snapshot stays intact.
2054        let _ = fs::remove_file(&temporary).await;
2055    }
2056    write_result
2057}
2058
2059fn embedded_config(mut config: TokeiraConfig) -> Result<TokeiraConfig> {
2060    config.infrastructure.storage = ConfigStorageKind::InMemory;
2061    // An embedded engine owns its only runtime node. Retaining a controller
2062    // endpoint would spawn membership and routing clients even though there is
2063    // no advertised listener for another node to reach.
2064    config.infrastructure.placement.controller_endpoint = None;
2065    config
2066        .validate()
2067        .context("invalid embedded engine config")?;
2068    Ok(config)
2069}
2070
2071impl Drop for Engine {
2072    fn drop(&mut self) {
2073        if let Some(coordinator) = &self.shutdown_coordinator {
2074            coordinator.begin_shutdown();
2075        }
2076        self.background_cancel.cancel();
2077    }
2078}
2079
2080impl TemporalEndpoint {
2081    /// Dispatch one raw protobuf request through the Temporal service router.
2082    pub async fn call(
2083        &self,
2084        request: InProcessGrpcRequest,
2085    ) -> Result<InProcessGrpcResponse, tonic::Status> {
2086        tokio::select! {
2087            biased;
2088            _ = self.shutdown.cancelled() => {
2089                Err(tonic::Status::unavailable("embedded Tokeira engine is shut down"))
2090            }
2091            response = self.service.call(request) => response,
2092        }
2093    }
2094
2095    /// Whether the engine that owns this endpoint has begun shutdown.
2096    pub fn is_shutdown(&self) -> bool {
2097        self.shutdown.is_cancelled()
2098    }
2099
2100    /// Build the Temporal Rust SDK callback service for this endpoint.
2101    #[cfg(feature = "temporalio-client")]
2102    pub fn service_override(&self) -> temporalio_client::callback_based::CallbackBasedGrpcService {
2103        let endpoint = self.clone();
2104        temporalio_client::callback_based::CallbackBasedGrpcService {
2105            callback: Arc::new(move |request| {
2106                let endpoint = endpoint.clone();
2107                Box::pin(async move {
2108                    endpoint
2109                        .call(InProcessGrpcRequest {
2110                            service: request.service,
2111                            rpc: request.rpc,
2112                            headers: request.headers,
2113                            proto: request.proto,
2114                        })
2115                        .await
2116                        // The edge and the SDK share one tonic, so the status
2117                        // crosses the seam untouched: code, message, details,
2118                        // and metadata exactly as the edge produced them.
2119                        .map(
2120                            |response| temporalio_client::callback_based::GrpcSuccessResponse {
2121                                headers: response.headers,
2122                                proto: response.proto,
2123                            },
2124                        )
2125                })
2126            }),
2127        }
2128    }
2129}
2130
2131const EMBEDDED_NEXUS_CALLBACK_BASE: &str = "http://tokeira-engine.invalid";
2132
2133/// Completion client that keeps `temporal://system` callbacks inside the engine
2134/// while retaining the normal HTTP client for explicitly external callback URLs.
2135struct InProcessNexusCompletionClient {
2136    runtime: RwLock<Option<Weak<dyn WorkflowRuntimeApi>>>,
2137    http: HttpNexusCompletionClient,
2138}
2139
2140impl std::fmt::Debug for InProcessNexusCompletionClient {
2141    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
2142        f.debug_struct("InProcessNexusCompletionClient")
2143            .finish_non_exhaustive()
2144    }
2145}
2146
2147impl InProcessNexusCompletionClient {
2148    fn new() -> Self {
2149        Self {
2150            runtime: RwLock::new(None),
2151            http: HttpNexusCompletionClient::new(),
2152        }
2153    }
2154
2155    fn attach(&self, runtime: &Arc<dyn WorkflowRuntimeApi>) -> Result<()> {
2156        let mut slot = self
2157            .runtime
2158            .write()
2159            .map_err(|_| anyhow!("embedded Nexus completion runtime lock poisoned"))?;
2160        *slot = Some(Arc::downgrade(runtime));
2161        Ok(())
2162    }
2163}
2164
2165#[async_trait]
2166impl NexusCompletionClient for InProcessNexusCompletionClient {
2167    async fn complete_operation(
2168        &self,
2169        url: &str,
2170        token: &str,
2171        operation_token: &str,
2172        completion: NexusCompletion,
2173        links: &[Link],
2174    ) -> Result<CompletionDeliveryOutcome> {
2175        if url != system_callback_post_url(EMBEDDED_NEXUS_CALLBACK_BASE) {
2176            return self
2177                .http
2178                .complete_operation(url, token, operation_token, completion, links)
2179                .await;
2180        }
2181
2182        let runtime = self
2183            .runtime
2184            .read()
2185            .ok()
2186            .and_then(|slot| slot.as_ref().and_then(Weak::upgrade));
2187        let Some(runtime) = runtime else {
2188            return Ok(CompletionDeliveryOutcome::RetryableError {
2189                detail: "embedded Nexus completion runtime is unavailable".to_owned(),
2190            });
2191        };
2192
2193        let state = completion.operation_state();
2194        let (body, content_type) = match completion {
2195            NexusCompletion::Succeeded(payloads) => nexus_payload_to_body(&payloads),
2196            NexusCompletion::Failed(body) | NexusCompletion::Canceled(body) => {
2197                (body, Some("application/json".to_owned()))
2198            }
2199        };
2200        let response = handle_nexus_callback(
2201            runtime.as_ref(),
2202            Some(token),
2203            Some(operation_token),
2204            Some(state),
2205            content_type.as_deref(),
2206            &body,
2207        )
2208        .await;
2209        let detail = response
2210            .body
2211            .map(|body| String::from_utf8_lossy(&body).into_owned())
2212            .unwrap_or_else(|| format!("Nexus callback returned status {}", response.status));
2213        Ok(match response.status {
2214            200..=299 => CompletionDeliveryOutcome::Delivered,
2215            400 | 401 | 403 | 404 | 409 | 501 => {
2216                CompletionDeliveryOutcome::NonRetryableError { detail }
2217            }
2218            _ => CompletionDeliveryOutcome::RetryableError { detail },
2219        })
2220    }
2221}
2222
2223/// Handle returned by [`TokeiradHandle::start_in_memory`].
2224///
2225/// Drop-safe: the inner server task receives a shutdown signal when the handle
2226/// is dropped, after which the task completes on its own. For deterministic
2227/// teardown (tests that want to assert a clean exit) call [`Self::shutdown`]
2228/// explicitly.
2229///
2230/// The handle also exposes a [`log_sink`](Self::log_sink) that integration
2231/// tests use to observe SDK-facing RPC activity (specifically the per-call
2232/// `tracing::debug!` line that `record_worker_heartbeat` emits). The stream is
2233/// a broadcast receiver over structured tracing events captured inside the
2234/// facade; consumers not subscribed to it see no behaviour change.
2235#[derive(Debug)]
2236pub struct TokeiradHandle {
2237    bound_addr: SocketAddr,
2238    shutdown_tx: Option<oneshot::Sender<()>>,
2239    server_task: Option<JoinHandle<Result<()>>>,
2240    /// Cancellation token for background workers spawned by the facade.
2241    ///
2242    /// Kept alongside the server task so that dropping the handle cancels the
2243    /// membership client, routing subscription, projection workers, and
2244    /// schedule engine in lockstep with the gRPC server shutdown.
2245    background_cancel: CancellationToken,
2246    log_broadcast: broadcast::Sender<LogEvent>,
2247    /// In-memory snapshot policy, when configured. The final consistent cut
2248    /// runs in [`Self::shutdown`]; a plain drop cancels only (an async write
2249    /// cannot run in `Drop`), so callers needing durability call `shutdown`.
2250    snapshot_policy: Option<EngineSnapshotPolicy>,
2251}
2252
2253/// One observable RPC event streamed through [`TokeiradHandle::log_sink`].
2254///
2255/// This is deliberately narrow: it captures only the fields integration tests
2256/// need to distinguish one RPC from another. The full tracing-subscriber
2257/// configuration still applies — this channel is a convenience fan-out, not a
2258/// replacement for the log pipeline.
2259#[derive(Clone, Debug)]
2260pub struct LogEvent {
2261    pub rpc: String,
2262    pub namespace: String,
2263    pub heartbeat_count: usize,
2264}
2265
2266impl TokeiradHandle {
2267    /// Start an in-memory `tokeirad` bound to the caller-provided socket.
2268    ///
2269    /// `addr` may be `127.0.0.1:0` to request an ephemeral port; the bound
2270    /// port is then available via [`Self::bound_addr`]. All other server
2271    /// configuration uses `TokeiraConfig::default()`, which selects the
2272    /// in-memory storage path.
2273    pub async fn start_in_memory(addr: SocketAddr) -> Result<Self> {
2274        Self::start_in_memory_with_config(addr, TokeiraConfig::default()).await
2275    }
2276
2277    /// Start an in-memory `tokeirad` with an explicit validated configuration.
2278    ///
2279    /// This is the production bootstrap path with only storage selection and
2280    /// listener addresses fixed for process-local integration tests. In
2281    /// particular, configured authentication, authorization, and Worker-scope
2282    /// policy are constructed exactly as they are by the binary.
2283    pub async fn start_in_memory_with_config(
2284        addr: SocketAddr,
2285        mut config: TokeiraConfig,
2286    ) -> Result<Self> {
2287        config.infrastructure.storage = ConfigStorageKind::InMemory;
2288        // Bind the inbound Nexus completion listener on an ephemeral loopback port so
2289        // parallel in-memory servers (tests, multi-node harnesses) never collide on the
2290        // fixed default port; the runtime resolves `temporal://system` to the bound port.
2291        config.policy.nexus_completion.http_addr = "127.0.0.1:0".to_string();
2292        config.policy.nexus_completion.system_callback_url = "http://127.0.0.1:0".to_string();
2293        config
2294            .validate()
2295            .context("invalid in-memory server config")?;
2296        let effective_config = Arc::new(config);
2297        let (
2298            (server_task, bound_addr, shutdown_tx, background_cancel, log_broadcast, _recorder),
2299            snapshot_policy,
2300        ) = build_and_serve(addr, effective_config).await?;
2301        Ok(Self {
2302            bound_addr,
2303            shutdown_tx: Some(shutdown_tx),
2304            server_task: Some(server_task),
2305            background_cancel,
2306            log_broadcast,
2307            snapshot_policy,
2308        })
2309    }
2310
2311    /// Parse TOML and start the in-memory integration facade.
2312    ///
2313    /// Keeping parsing inside this crate lets cross-crate integration tests
2314    /// exercise the public configuration surface without depending directly on
2315    /// the configuration implementation crate.
2316    pub async fn start_in_memory_with_toml(addr: SocketAddr, config: &str) -> Result<Self> {
2317        let config = toml::from_str(config).context("parse in-memory server config")?;
2318        Self::start_in_memory_with_config(addr, config).await
2319    }
2320
2321    /// The socket address the server is bound to after startup.
2322    pub fn bound_addr(&self) -> SocketAddr {
2323        self.bound_addr
2324    }
2325
2326    /// Subscribe to the in-process log-event broadcast. Each subscriber gets a
2327    /// fresh receiver; events emitted before subscription are not replayed.
2328    pub fn log_sink(&self) -> broadcast::Receiver<LogEvent> {
2329        self.log_broadcast.subscribe()
2330    }
2331
2332    /// Signal the server to shut down, wait for the task to exit, and persist
2333    /// the final snapshot when a snapshot policy is configured.
2334    pub async fn shutdown(mut self) -> Result<()> {
2335        if let Some(tx) = self.shutdown_tx.take() {
2336            let _ = tx.send(());
2337        }
2338        self.background_cancel.cancel();
2339        if let Some(task) = self.server_task.take() {
2340            task.await
2341                .context("tokeirad server task panicked")?
2342                .context("tokeirad server task returned an error")?;
2343        }
2344        match self.snapshot_policy.take() {
2345            Some(policy) => finish_snapshot_policy(policy).await,
2346            None => Ok(()),
2347        }
2348    }
2349}
2350
2351impl Drop for TokeiradHandle {
2352    fn drop(&mut self) {
2353        if let Some(tx) = self.shutdown_tx.take() {
2354            let _ = tx.send(());
2355        }
2356        self.background_cancel.cancel();
2357        // Server task is not awaited here; callers who need deterministic
2358        // teardown call `shutdown` explicitly. Tokio will reap the task.
2359    }
2360}
2361
2362fn dsql_auth_config(config: &TokeiraConfig) -> Result<DsqlAuthConfig> {
2363    let dsql = &config.infrastructure.dsql;
2364    let endpoint = dsql
2365        .endpoint
2366        .clone()
2367        .filter(|endpoint| !endpoint.trim().is_empty())
2368        .ok_or_else(|| {
2369            anyhow!("infrastructure.dsql.endpoint must be set when infrastructure.storage is dsql")
2370        })?;
2371    Ok(DsqlAuthConfig {
2372        endpoint,
2373        region: dsql.region.clone(),
2374        admin_role_arn: dsql.admin_role_arn.clone(),
2375        runtime_role_arn: dsql.runtime_role_arn.clone(),
2376        readonly_role_arn: dsql.readonly_role_arn.clone(),
2377    })
2378}
2379
2380fn dsql_pool_config_with_client(
2381    config: &TokeiraConfig,
2382    ddb_client: aws_sdk_dynamodb::Client,
2383) -> (DsqlPoolConfig, aws_sdk_dynamodb::Client) {
2384    let (rate_limiter_table, conn_lease_table) = dsql_coordination_table_names(config);
2385    let pool_config = DsqlPoolConfig {
2386        coordination: DsqlCoordinationConfig {
2387            rate_limiter_table,
2388            conn_lease_table,
2389        },
2390        shard_count: config.infrastructure.placement.shard_count,
2391        projection_partition_count: config.infrastructure.placement.partition_count,
2392        ..DsqlPoolConfig::default()
2393    };
2394    (pool_config, ddb_client)
2395}
2396
2397async fn dsql_pool_config(
2398    config: &TokeiraConfig,
2399    auth: &DsqlAuthConfig,
2400) -> Result<(DsqlPoolConfig, aws_sdk_dynamodb::Client)> {
2401    let region = auth
2402        .resolved_region()
2403        .ok_or_else(|| anyhow!("dsql region must be configured or derivable from endpoint"))?;
2404    let sdk_config = aws_config::defaults(aws_config::BehaviorVersion::latest())
2405        .region(aws_config::Region::new(region))
2406        .load()
2407        .await;
2408    Ok(dsql_pool_config_with_client(
2409        config,
2410        aws_sdk_dynamodb::Client::new(&sdk_config),
2411    ))
2412}
2413
2414fn dsql_coordination_table_names(config: &TokeiraConfig) -> (String, String) {
2415    let dsql = &config.infrastructure.dsql;
2416    let rate_limiter = dsql
2417        .rate_limiter_table
2418        .clone()
2419        .unwrap_or_else(|| format!("{}-dsql-rate-limiter", config.infrastructure.cluster_name));
2420    let conn_lease = dsql
2421        .conn_lease_table
2422        .clone()
2423        .unwrap_or_else(|| format!("{}-dsql-conn-lease", config.infrastructure.cluster_name));
2424    (rate_limiter, conn_lease)
2425}
2426
2427/// Entrypoint the CLI delegates to.
2428///
2429/// Parses `TokeiraConfig` from the CLI arguments, handles `--dump-config`, and
2430/// serves the gRPC stack on `infrastructure.network.grpc_addr` until ctrl-C
2431/// or SIGTERM, then drains gracefully: in-flight requests finish, background
2432/// work is cancelled, and (for in-memory storage with a snapshot policy) the
2433/// final consistent cut is persisted before exit.
2434// `--version` / `--dump-config` are CLI outputs to stdout by contract; daemon
2435// logging elsewhere speaks `tracing` only.
2436#[allow(clippy::print_stdout)]
2437pub async fn run_from_cli(cli: Cli) -> Result<()> {
2438    if cli.version {
2439        println!("{}", render_build_info(cli.verbose, cli.json));
2440        return Ok(());
2441    }
2442
2443    let (effective_config, config_source) = TokeiraConfig::resolve(cli.config.as_deref())?;
2444    if cli.dump_config {
2445        // A TOML comment so the dump stays parseable while naming the winner.
2446        println!("# config source: {config_source}");
2447        println!("{}", effective_config.to_toml()?);
2448        return Ok(());
2449    }
2450    let observability = observability::ObservabilityConfig::from_tokeira_config(&effective_config)?;
2451    let effective_config = Arc::new(effective_config);
2452    let readiness = observability::TokeiradReadiness::new();
2453    let _observability_runtime = observability::install_process_observability(
2454        &observability,
2455        effective_config.clone(),
2456        readiness.registry.clone(),
2457    )
2458    .await?;
2459    log_build_info("tokeirad");
2460    for warning in effective_config.emergency_warnings() {
2461        tracing::warn!("{warning}");
2462    }
2463
2464    let addr: SocketAddr = effective_config
2465        .infrastructure
2466        .network
2467        .grpc_addr
2468        .parse()
2469        .with_context(|| {
2470            format!(
2471                "invalid infrastructure.network.grpc_addr value: {}",
2472                effective_config.infrastructure.network.grpc_addr
2473            )
2474        })?;
2475    info!(config_source = %config_source, "loaded tokeirad configuration");
2476
2477    let (
2478        (
2479            mut server_task,
2480            bound_addr,
2481            shutdown_tx,
2482            background_cancel,
2483            _log_broadcast,
2484            wire_recorder,
2485        ),
2486        snapshot_policy,
2487    ) = build_and_serve(addr, effective_config).await?;
2488    readiness.mark_started();
2489    info!("tokeirad gRPC server listening on {bound_addr}");
2490
2491    // Daemon-only signal handling. The embedded facade (`Engine`) deliberately
2492    // installs no handlers and no process observability: an embedded host owns
2493    // its process lifecycle, and a library that grabs SIGTERM would race the
2494    // host's own shutdown sequence. Only this served entrypoint reacts to
2495    // ctrl-C/SIGTERM, and the reaction is the graceful drain: stop accepting
2496    // connections via the listener's shutdown oneshot, await in-flight
2497    // requests, cancel background work, then take the final snapshot below.
2498    let serve_result = {
2499        let drain_signal = async {
2500            #[cfg(unix)]
2501            {
2502                let mut sigterm =
2503                    tokio::signal::unix::signal(tokio::signal::unix::SignalKind::terminate())
2504                        .context("failed to install the SIGTERM handler")?;
2505                tokio::select! {
2506                    result = tokio::signal::ctrl_c() => result
2507                        .context("failed to listen for ctrl-c")
2508                        .map(|()| "SIGINT"),
2509                    _ = sigterm.recv() => Ok("SIGTERM"),
2510                }
2511            }
2512            #[cfg(not(unix))]
2513            {
2514                tokio::signal::ctrl_c()
2515                    .await
2516                    .context("failed to listen for ctrl-c")
2517                    .map(|()| "ctrl-c")
2518            }
2519        };
2520        tokio::select! {
2521            result = &mut server_task => result
2522                .context("tokeirad server task panicked")?
2523                .context("tokeirad server task returned an error"),
2524            signal = drain_signal => {
2525                match signal {
2526                    Ok(signal) => info!(signal, "shutdown signal received; draining tokeirad"),
2527                    Err(error) => {
2528                        // Signal delivery is broken; a blunt stop is the only
2529                        // honest option left, but say why.
2530                        tracing::warn!(?error, "signal handling failed; shutting down");
2531                    }
2532                }
2533                let _ = shutdown_tx.send(());
2534                (&mut server_task)
2535                    .await
2536                    .context("tokeirad server task panicked")?
2537                    .context("tokeirad server task returned an error")
2538            }
2539        }
2540    };
2541    background_cancel.cancel();
2542    // The final consistent cut runs after the drain regardless of how the
2543    // server exited, so an error exit still leaves the freshest snapshot. A
2544    // snapshot failure must not mask the server's own exit status; it is
2545    // surfaced only after that status is known good.
2546    let snapshot_result = match snapshot_policy {
2547        Some(policy) => finish_snapshot_policy(policy).await,
2548        None => Ok(()),
2549    };
2550
2551    // Tier-2 conformance evidence export. When the wire-coverage recorder is present (the
2552    // conformance flag was set), snapshot the observed `(wire_method, status_code)` set
2553    // after the server has stopped serving and write it as pretty JSON for the Rust report
2554    // (see `.kiro/specs/temporal-functional-conformance`, task 9.x). This runs regardless
2555    // of how the server exited so a clean shutdown after a run still produces evidence; a
2556    // failed export is logged but MUST NOT mask the server's own exit status, so the
2557    // server result is returned after the export attempt. When the recorder is `None`
2558    // nothing is written and there is zero behavioural change.
2559    if let Some(recorder) = wire_recorder {
2560        let out_path = wire_coverage_out_path();
2561        if let Err(error) = export_wire_coverage(recorder.as_ref(), &out_path) {
2562            tracing::warn!(
2563                path = %out_path.display(),
2564                ?error,
2565                "failed to export conformance wire-coverage evidence"
2566            );
2567        }
2568    }
2569
2570    serve_result?;
2571    snapshot_result?;
2572    Ok(())
2573}
2574
2575/// Snapshot the recorder and write the wire-coverage record to `path` as pretty JSON.
2576///
2577/// Pretty (rather than compact) JSON is deliberate: the evidence file is read by humans
2578/// triaging conformance runs and is diffed across runs, so a stable, line-oriented layout
2579/// is worth the few extra bytes. The recorder's `snapshot()` already sorts rows
2580/// deterministically, so two runs observing the same calls produce byte-identical output.
2581///
2582/// Errors are surfaced with context rather than panicking: a failed export is a loss of
2583/// *evidence*, never a correctness fault, and the caller logs it without disturbing the
2584/// server's exit status.
2585fn export_wire_coverage(recorder: &WireCoverageRecorder, path: &Path) -> Result<()> {
2586    let record = recorder.snapshot();
2587    let row_count = record.rows.len();
2588    let json = serde_json::to_string_pretty(&record)
2589        .context("failed to serialize wire-coverage record to JSON")?;
2590    std::fs::write(path, json).with_context(|| {
2591        format!(
2592            "failed to write wire-coverage evidence to {}",
2593            path.display()
2594        )
2595    })?;
2596    info!(
2597        path = %path.display(),
2598        rows = row_count,
2599        "wrote conformance wire-coverage evidence"
2600    );
2601    Ok(())
2602}
2603
2604fn log_build_info(process: &'static str) {
2605    let info = tokeira_build_info::summary();
2606    tracing::info!(
2607        process,
2608        server_version = tokeira_build_info::SERVER_VERSION,
2609        tokeira_version = info.tokeira_version,
2610        tokeira_git_sha = info.tokeira_git_sha,
2611        tokeira_source_revision = info.tokeira_source_revision,
2612        temporal_proto_version = info.temporal_proto_version,
2613        temporal_server_compat = info.temporal_server_compat,
2614        rust_toolchain = info.rust_toolchain,
2615        source_tree_hash = info.source_tree_hash,
2616        feature_matrix_digest = info.feature_matrix_digest,
2617        sdk_matrix_digest = info.sdk_matrix_digest,
2618        build_mode = info.build_mode,
2619        "tokeira build provenance"
2620    );
2621}
2622
2623fn render_build_info(verbose: bool, json: bool) -> String {
2624    let info = tokeira_build_info::summary();
2625    if json {
2626        return serde_json::json!({
2627            "server_version": tokeira_build_info::SERVER_VERSION,
2628            "tokeira_version": info.tokeira_version,
2629            "tokeira_git_sha": info.tokeira_git_sha,
2630            "source_revision": info.tokeira_source_revision,
2631            "temporal_proto_version": info.temporal_proto_version,
2632            "temporal_server_compat": info.temporal_server_compat,
2633            "rust_toolchain": info.rust_toolchain,
2634            "source_tree_hash": info.source_tree_hash,
2635            "feature_matrix_digest": info.feature_matrix_digest,
2636            "sdk_matrix_digest": info.sdk_matrix_digest,
2637            "build_mode": info.build_mode,
2638        })
2639        .to_string();
2640    }
2641
2642    if verbose {
2643        return [
2644            format!("server_version: {}", tokeira_build_info::SERVER_VERSION),
2645            format!("tokeira_version: {}", info.tokeira_version),
2646            format!("tokeira_git_sha: {}", info.tokeira_git_sha),
2647            format!("tokeira_source_revision: {}", info.tokeira_source_revision),
2648            format!("temporal_proto_version: {}", info.temporal_proto_version),
2649            format!("temporal_server_compat: {}", info.temporal_server_compat),
2650            format!("rust_toolchain: {}", info.rust_toolchain),
2651            format!("source_tree_hash: {}", info.source_tree_hash),
2652            format!("feature_matrix_digest: {}", info.feature_matrix_digest),
2653            format!("sdk_matrix_digest: {}", info.sdk_matrix_digest),
2654            format!("build_mode: {}", info.build_mode),
2655        ]
2656        .join("\n");
2657    }
2658
2659    format!(
2660        "tokeira {}\ngit {}\ntemporal_proto {}\ntemporal_server {}",
2661        tokeira_build_info::SERVER_VERSION,
2662        info.tokeira_git_sha,
2663        info.temporal_proto_version,
2664        info.temporal_server_compat
2665    )
2666}
2667
2668/// What [`build_and_serve`] hands back to the
2669/// caller after the gRPC stack is spawned.
2670///
2671/// The final element — the optional `Arc<WireCoverageRecorder>` — is the Tier-2
2672/// conformance recorder handle. It is `Some` only when the conformance flag is set (see
2673/// [`wire_coverage_enabled`]); the caller snapshots it after the server task completes to
2674/// export wire-coverage evidence. It is carried as a tuple element rather than threaded
2675/// into [`TokeiradHandle`] because only the CLI entrypoint exports the evidence — the
2676/// in-memory test facade ignores it.
2677type ServerStack = (
2678    JoinHandle<Result<()>>,
2679    SocketAddr,
2680    oneshot::Sender<()>,
2681    CancellationToken,
2682    broadcast::Sender<LogEvent>,
2683    Option<Arc<WireCoverageRecorder>>,
2684);
2685
2686#[derive(Clone, Copy, Debug)]
2687enum StackTransport {
2688    Network(SocketAddr),
2689    Embedded,
2690}
2691
2692#[derive(Debug)]
2693enum ConstructedStack {
2694    Network(ServerStack),
2695    Embedded(EmbeddedStack),
2696}
2697
2698struct EmbeddedStack {
2699    component_visibility_store: chasm_extensions::VisibilityQueryStore,
2700    chasm_engine: Arc<tokeira_runtime::chasm::ChasmEngine>,
2701    registry: Arc<tokeira_chasm::Registry>,
2702    service: InProcessGrpcService,
2703    background_cancel: CancellationToken,
2704    log_broadcast: broadcast::Sender<LogEvent>,
2705    recovery_task: Option<JoinHandle<Result<()>>>,
2706    runtime_tasks: RuntimeShutdownHandle,
2707    engine_tasks: RuntimeShutdownHandle,
2708    shard_cleanup: Option<CleanupAction>,
2709}
2710
2711struct StackStartupGuard {
2712    runtime_tasks: RuntimeShutdownHandle,
2713    engine_tasks: RuntimeShutdownHandle,
2714    armed: bool,
2715}
2716
2717impl StackStartupGuard {
2718    fn new(runtime_tasks: RuntimeShutdownHandle, engine_tasks: RuntimeShutdownHandle) -> Self {
2719        Self {
2720            runtime_tasks,
2721            engine_tasks,
2722            armed: true,
2723        }
2724    }
2725
2726    fn disarm(&mut self) {
2727        self.armed = false;
2728    }
2729}
2730
2731impl Drop for StackStartupGuard {
2732    fn drop(&mut self) {
2733        if self.armed {
2734            // Async cleanup is owned by the caller's pool rollback, but every
2735            // background loop must first observe cancellation so it cannot
2736            // retain the partially constructed stack indefinitely.
2737            self.engine_tasks.begin_shutdown();
2738            self.runtime_tasks.begin_shutdown();
2739        }
2740    }
2741}
2742
2743#[derive(Clone, Debug)]
2744struct SelfAssignedShardLease {
2745    shard_id: ShardId,
2746    owner: String,
2747    epoch: ShardEpoch,
2748}
2749
2750impl std::fmt::Debug for EmbeddedStack {
2751    fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
2752        formatter
2753            .debug_struct("EmbeddedStack")
2754            .field("service", &self.service)
2755            .field("background_cancel", &self.background_cancel)
2756            .field("runtime_tasks", &self.runtime_tasks)
2757            .field("engine_tasks", &self.engine_tasks)
2758            .field("has_shard_cleanup", &self.shard_cleanup.is_some())
2759            .finish_non_exhaustive()
2760    }
2761}
2762
2763/// Default maximum identifier length for standalone-activity ids, mirroring
2764/// Temporal's `MaxIDLengthLimit` default of `1000` (`common/dynamicconfig` @
2765/// v1.31.0). Used to validate standalone-activity ids at the edge.
2766const DEFAULT_MAX_ID_LENGTH: usize = 1000;
2767
2768/// Refuse a migration policy a listener-backed distributed node doesn't
2769/// honour, before it connects to anything.
2770///
2771/// A distributed node never migrates the schema it shares with every other
2772/// node: migrating stays the operator's explicit `tkr schema setup` (root
2773/// `AGENTS.md` §4). So an absent or `validate_only` policy selects the startup
2774/// check, and `automatic` is refused rather than quietly read as
2775/// `validate_only`.
2776fn refuse_distributed_migration_policy(config: &TokeiraConfig) -> Result<()> {
2777    match config.infrastructure.dsql.migration_policy {
2778        None | Some(DsqlMigrationPolicy::ValidateOnly) => Ok(()),
2779        Some(DsqlMigrationPolicy::Automatic) => Err(anyhow!(
2780            "infrastructure.dsql.migration_policy = \"automatic\" is not supported by \
2781             listener-backed distributed DSQL: a distributed tokeirad never migrates the \
2782             schema it shares with other nodes. Apply migrations with `tkr schema setup`, \
2783             then set the policy to \"validate_only\" or remove it"
2784        )),
2785    }
2786}
2787
2788/// Check a listener-backed distributed node's DSQL schema against its
2789/// release's schema contract, without changing it, and refuse unless the
2790/// release can run on it.
2791///
2792/// The assessment is the embedded engine's (`apply_embedded_schema`) under the
2793/// validate-only policy. It uses the runner over the compiled-in migrations,
2794/// from which the release's contract is computed; the store's own runner reads
2795/// a migrations directory. Its one connection comes from the store's
2796/// coordinated reservoir, so the check keeps to DSQL's connection limits like
2797/// every other read.
2798async fn check_distributed_schema(dsql_store: &DsqlStore) -> Result<()> {
2799    let contract = MigrationRunner::compatibility_contract();
2800    let decision = {
2801        let mut permit = dsql_store
2802            .connection_director()
2803            .acquire(DbClass::Control)
2804            .await?;
2805        MigrationRunner::embedded()
2806            .assess_connection(
2807                permit.connection()?,
2808                &contract,
2809                SchemaMigrationPolicy::ValidateOnly,
2810            )
2811            .await
2812            .context("failed to read the DSQL schema's migration ledger")?
2813    };
2814    let current = distributed_schema_admission(decision)?;
2815    info!(
2816        schema_version = current,
2817        target_version = contract.target_version,
2818        maximum_readable_version = contract.maximum_readable_version,
2819        "DSQL schema check passed"
2820    );
2821    Ok(())
2822}
2823
2824/// Admit a listener-backed distributed node only to a schema that a
2825/// validate-only assessment found compatible, returning its version.
2826/// Otherwise return the refusal the node exits with: what it found and what
2827/// to do about it.
2828fn distributed_schema_admission(decision: SchemaDecision) -> Result<u32> {
2829    match decision {
2830        SchemaDecision::Compatible { current, .. } => Ok(current),
2831        SchemaDecision::MigrationRequired { current: 0, target } => Err(anyhow!(
2832            "tokeirad refuses to start: the DSQL database has no Tokeira schema. Apply this \
2833             release's migrations, through V{target}, with `tkr schema setup`, then start \
2834             tokeirad"
2835        )),
2836        SchemaDecision::MigrationRequired { current, target } => Err(anyhow!(
2837            "tokeirad refuses to start: the DSQL schema is at V{current}, behind this \
2838             release's target V{target}. Apply the missing migrations with `tkr schema \
2839             setup`, then start tokeirad"
2840        )),
2841        SchemaDecision::Reject(incompatibility) => {
2842            let advice = match incompatibility.category {
2843                SchemaIncompatibilityCategory::FutureSchema => {
2844                    "a later release migrated it; run that release or a later one"
2845                }
2846                SchemaIncompatibilityCategory::BelowMinimum => {
2847                    "it is older than this release migrates from; upgrade it through an \
2848                     earlier release first"
2849                }
2850                _ => {
2851                    "its migration ledger doesn't match this release's migrations; find out \
2852                     what changed it before any node runs on it"
2853                }
2854            };
2855            Err(
2856                anyhow::Error::new(SchemaCompatibilityError::Incompatible(incompatibility))
2857                    .context(format!(
2858                        "tokeirad refuses to start: this release can't run on the DSQL \
2859                         schema, because {advice}"
2860                    )),
2861            )
2862        }
2863        // An assessment under the validate-only policy never initializes or
2864        // migrates; only the automatic policy decides either.
2865        decision @ (SchemaDecision::Initialize { .. } | SchemaDecision::Migrate { .. }) => {
2866            Err(anyhow!(
2867                "tokeirad refuses to start: a validate-only schema assessment decided \
2868                 {decision:?}"
2869            ))
2870        }
2871    }
2872}
2873
2874/// Build the full server stack and start serving on the given address.
2875///
2876/// Storage selection is driven only by `infrastructure.storage`: endpoint
2877/// presence alone is not treated as intent. This keeps failed DSQL writeback
2878/// and explicit in-memory deployments from being conflated. A DSQL node checks
2879/// its schema before it builds anything ([`check_distributed_schema`]).
2880///
2881/// Factored out so both the CLI entrypoint and the in-memory facade share one
2882/// bootstrap path. Deliberately long and sequential: each block mirrors the
2883/// startup dependency order.
2884async fn build_and_serve(
2885    addr: SocketAddr,
2886    effective_config: Arc<TokeiraConfig>,
2887) -> Result<(ServerStack, Option<EngineSnapshotPolicy>)> {
2888    let mut snapshot_inputs = None;
2889    let stack = match effective_config.infrastructure.storage {
2890        ConfigStorageKind::InMemory => {
2891            // The listener-backed in-memory server shares the embedded
2892            // facade's snapshot mechanism exactly: restore at boot, periodic
2893            // persistence, and a final cut at shutdown all operate on the one
2894            // in-memory store.
2895            let (store, restored) = restore_snapshot_store(&effective_config).await?;
2896            snapshot_inputs = effective_config
2897                .policy
2898                .snapshot
2899                .clone()
2900                .map(|config| (store.clone(), config));
2901            build_in_memory_stack(
2902                StackTransport::Network(addr),
2903                effective_config,
2904                store,
2905                restored,
2906                ChasmExtensions::default(),
2907            )
2908            .await
2909        }
2910        ConfigStorageKind::Dsql => {
2911            if effective_config.policy.snapshot.is_some() {
2912                return Err(anyhow!(
2913                    "policy.snapshot is configured but infrastructure.storage is \"dsql\": \
2914                     snapshots are the in-memory store's persistence mechanism, and DSQL \
2915                     storage is already durable — remove policy.snapshot or switch \
2916                     infrastructure.storage to \"memory\""
2917                ));
2918            }
2919            refuse_distributed_migration_policy(&effective_config)?;
2920            let auth = dsql_auth_config(&effective_config)?;
2921            let endpoint = auth.endpoint.clone();
2922            let (pool_config, ddb_client) = dsql_pool_config(&effective_config, &auth).await?;
2923            let dsql_store = DsqlStore::connect(auth, pool_config, ddb_client)
2924                .await
2925                .context("failed to connect DSQL storage backend")?;
2926            // The check runs before anything is built, so a refusal leaves no
2927            // listener, runtime or shard lease behind. Shutting the store down
2928            // closes its connections and releases the DynamoDB connection-slot
2929            // blocks its reservoir took, which dropping it would leave owned.
2930            if let Err(refusal) = check_distributed_schema(&dsql_store).await {
2931                if let Err(error) = dsql_store.shutdown().await {
2932                    tracing::warn!(
2933                        error = %format!("{error:#}"),
2934                        "failed to shut DSQL storage down after refusing its schema"
2935                    );
2936                }
2937                return Err(refusal);
2938            }
2939            build_dsql_stack(
2940                StackTransport::Network(addr),
2941                effective_config,
2942                dsql_store,
2943                endpoint,
2944                ChasmExtensions::default(),
2945            )
2946            .await
2947        }
2948    }?;
2949    match stack {
2950        ConstructedStack::Network(stack) => {
2951            // The policy shares the stack's cancellation token so the periodic
2952            // task dies with the server's other background work; the final
2953            // consistent cut stays with the caller's shutdown sequence.
2954            let snapshot_policy = snapshot_inputs
2955                .map(|(store, config)| EngineSnapshotPolicy::start(store, config, stack.3.clone()));
2956            Ok((stack, snapshot_policy))
2957        }
2958        ConstructedStack::Embedded(_) => Err(anyhow!(
2959            "network service construction returned an embedded stack"
2960        )),
2961    }
2962}
2963
2964async fn build_dsql_stack(
2965    transport: StackTransport,
2966    effective_config: Arc<TokeiraConfig>,
2967    dsql_store: DsqlStore,
2968    endpoint: String,
2969    extensions: ChasmExtensions,
2970) -> Result<ConstructedStack> {
2971    let (director, run_repository, projection_log, worker_deployment_repository, _migration_runner) =
2972        dsql_store.into_parts();
2973    // Every repository and visibility surface shares this exact director. The
2974    // embedded caller retains another Arc solely for ordered pool shutdown;
2975    // no second pool or DynamoDB coordinator is constructed here.
2976    let budget_applier: Arc<dyn ConnectionBudgetApplier> = Arc::new(DsqlConnectionBudgetApplier {
2977        director: director.clone(),
2978    });
2979    let chasm_node_repo = Arc::new(tokeira_storage::dsql::DsqlChasmNodeRepository::new(
2980        director.clone(),
2981    ));
2982    let task_queue_config_repository: Arc<dyn TaskQueueConfigRepository> = Arc::new(
2983        tokeira_storage::dsql::DsqlTaskQueueConfigRepository::new(director.clone()),
2984    );
2985    let worker_compute_repository = effective_config.policy.worker_compute.enabled.then(|| {
2986        Arc::new(tokeira_storage::dsql::DsqlWorkerComputeRepository::new(
2987            director.clone(),
2988        )) as Arc<dyn WorkerComputeRepository>
2989    });
2990    let worker_task_provenance: Arc<dyn WorkerTaskProvenanceStore> = Arc::new(
2991        tokeira_storage::dsql::DsqlWorkerTaskProvenanceStore::new(director.clone()),
2992    );
2993    let visibility_store = DsqlVisibilityStore::new(director);
2994    let worker_deployment_repository: Arc<dyn WorkerDeploymentRepository> =
2995        Arc::new(worker_deployment_repository);
2996    build_service_stack_with_storage(
2997        transport,
2998        effective_config,
2999        Arc::new(run_repository),
3000        worker_deployment_repository,
3001        worker_compute_repository,
3002        task_queue_config_repository,
3003        worker_task_provenance,
3004        projection_log,
3005        visibility_store.clone(),
3006        {
3007            let visibility_store = visibility_store.clone();
3008            move || visibility_store.clone()
3009        },
3010        Some(endpoint),
3011        chasm_node_repo,
3012        false,
3013        budget_applier,
3014        extensions,
3015    )
3016    .await
3017}
3018
3019async fn build_embedded(
3020    effective_config: Arc<TokeiraConfig>,
3021    store: InMemoryStore,
3022    recover_self_assigned_shard: bool,
3023    extensions: ChasmExtensions,
3024) -> Result<EmbeddedStack> {
3025    match build_in_memory_stack(
3026        StackTransport::Embedded,
3027        effective_config,
3028        store,
3029        recover_self_assigned_shard,
3030        extensions,
3031    )
3032    .await?
3033    {
3034        ConstructedStack::Embedded(stack) => Ok(stack),
3035        ConstructedStack::Network(_) => Err(anyhow!(
3036            "embedded service construction returned a network stack"
3037        )),
3038    }
3039}
3040
3041async fn build_in_memory_stack(
3042    transport: StackTransport,
3043    effective_config: Arc<TokeiraConfig>,
3044    store: InMemoryStore,
3045    recover_self_assigned_shard: bool,
3046    extensions: ChasmExtensions,
3047) -> Result<ConstructedStack> {
3048    let visibility_store = InMemoryVisibilityStore::default();
3049    let chasm_node_repo: Arc<dyn tokeira_storage::ChasmNodeRepository> =
3050        Arc::new(tokeira_storage::InMemoryChasmNodeStore::new());
3051    #[cfg(test)]
3052    let (chasm_node_repo, visibility_store) = extensions
3053        .test_storage
3054        .clone()
3055        .unwrap_or((chasm_node_repo, visibility_store));
3056    let worker_deployment_repository: Arc<dyn WorkerDeploymentRepository> = Arc::new(store.clone());
3057    let worker_task_provenance: Arc<dyn WorkerTaskProvenanceStore> = Arc::new(store.clone());
3058    let worker_compute_repository = effective_config.policy.worker_compute.enabled.then(|| {
3059        Arc::new(InMemoryWorkerComputeRepository::default()) as Arc<dyn WorkerComputeRepository>
3060    });
3061    build_service_stack_with_storage(
3062        transport,
3063        effective_config,
3064        Arc::new(store.clone()),
3065        worker_deployment_repository,
3066        worker_compute_repository,
3067        Arc::new(store.clone()),
3068        worker_task_provenance,
3069        store,
3070        visibility_store.clone(),
3071        {
3072            let visibility_store = visibility_store.clone();
3073            move || VisibilitySink::new(visibility_store.clone())
3074        },
3075        None,
3076        chasm_node_repo,
3077        recover_self_assigned_shard,
3078        // The in-memory store has no connection reservoir to budget.
3079        Arc::new(NoopConnectionBudgetApplier),
3080        extensions,
3081    )
3082    .await
3083}
3084
3085/// How often the visibility repair scanner sweeps committed executions to repair any
3086/// projection lost by the best-effort post-commit write (Req 10.11).
3087const VISIBILITY_REPAIR_INTERVAL: std::time::Duration = std::time::Duration::from_secs(30);
3088
3089/// Recovery cadence for disposable CHASM timers and dispatch state.
3090const CHASM_REBUILD_INTERVAL: std::time::Duration = std::time::Duration::from_secs(30);
3091
3092/// Authorization evidence is derived from task deadlines, so cleanup needs no
3093/// operator TTL. One bounded batch per tick prevents expired rows from
3094/// monopolizing the shared DSQL connection budget.
3095const WORKER_TASK_PROVENANCE_CLEANUP_INTERVAL: std::time::Duration =
3096    std::time::Duration::from_secs(30);
3097const WORKER_TASK_PROVENANCE_CLEANUP_BATCH: usize = 256;
3098
3099fn spawn_worker_task_provenance_cleanup(
3100    tasks: &RuntimeShutdownHandle,
3101    store: Arc<dyn WorkerTaskProvenanceStore>,
3102    cancel: CancellationToken,
3103) -> JoinHandle<()> {
3104    tasks.spawn(async move {
3105        let mut interval = tokio::time::interval(WORKER_TASK_PROVENANCE_CLEANUP_INTERVAL);
3106        interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
3107        loop {
3108            tokio::select! {
3109                _ = cancel.cancelled() => break,
3110                _ = interval.tick() => {
3111                    if let Err(error) = store
3112                        .delete_expired(
3113                            time::OffsetDateTime::now_utc(),
3114                            WORKER_TASK_PROVENANCE_CLEANUP_BATCH,
3115                        )
3116                        .await
3117                    {
3118                        tracing::warn!(
3119                            error_kind = worker_task_provenance_cleanup_error_kind(&error),
3120                            "Worker task authorization-evidence cleanup pass failed"
3121                        );
3122                    }
3123                }
3124            }
3125        }
3126    })
3127}
3128
3129fn worker_task_provenance_cleanup_error_kind(
3130    error: &tokeira_storage::WorkerTaskProvenanceError,
3131) -> &'static str {
3132    match error {
3133        tokeira_storage::WorkerTaskProvenanceError::Unavailable { .. } => "unavailable",
3134        tokeira_storage::WorkerTaskProvenanceError::DigestConflict => "conflict",
3135        tokeira_storage::WorkerTaskProvenanceError::Corrupt { .. } => "corrupt",
3136    }
3137}
3138
3139/// Spawn the background visibility repair scanner: it reconstructs each committed
3140/// execution's snapshot from authoritative node state and re-applies it iff-newer, so
3141/// a committed transition can never permanently lack a projection (Req 10.11). The
3142/// snapshot rebuild uses the registered root adapters for every archetype. Runs immediately, then on an interval, until
3143/// `cancel` fires.
3144fn spawn_visibility_repair(
3145    tasks: &RuntimeShutdownHandle,
3146    nodes: Arc<dyn tokeira_storage::ChasmNodeRepository>,
3147    sink: Arc<dyn tokeira_projection::ProjectionSink>,
3148    registry: Arc<tokeira_chasm::Registry>,
3149    partition_count: u32,
3150    cancel: CancellationToken,
3151) {
3152    let rebuild: tokeira_runtime::chasm::SnapshotRebuilder =
3153        Arc::new(move |archetype_id, bytes| {
3154            match registry.visibility_snapshot(archetype_id, bytes) {
3155                Ok(snapshot) => snapshot,
3156                Err(error) => {
3157                    tracing::warn!(
3158                        ?error,
3159                        archetype_id,
3160                        "CHASM visibility rebuild decode failed"
3161                    );
3162                    None
3163                }
3164            }
3165        });
3166    let scanner =
3167        tokeira_runtime::chasm::VisibilityRepairScanner::new(nodes, sink, rebuild, partition_count);
3168    let _repair = tasks.spawn(async move {
3169        let mut interval = tokio::time::interval(VISIBILITY_REPAIR_INTERVAL);
3170        interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
3171        loop {
3172            tokio::select! {
3173                _ = cancel.cancelled() => break,
3174                _ = interval.tick() => {
3175                    if let Err(error) = scanner.repair_once().await {
3176                        tracing::warn!(?error, "visibility repair pass failed");
3177                    }
3178                }
3179            }
3180        }
3181    });
3182}
3183
3184/// Repeat the startup recovery pass so losing an in-memory delivery hint costs
3185/// at most one scan interval. Shutdown cancels any in-flight derived scan.
3186fn spawn_outbox_rebuild(
3187    tasks: &RuntimeShutdownHandle,
3188    scanner: tokeira_runtime::chasm::OutboxRebuildScanner,
3189    cancel: CancellationToken,
3190) {
3191    let _rebuild = tasks.spawn(async move {
3192        let mut interval = tokio::time::interval(CHASM_REBUILD_INTERVAL);
3193        interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
3194        // Startup already ran the first pass before opening admission.
3195        interval.tick().await;
3196        loop {
3197            tokio::select! {
3198                _ = cancel.cancelled() => break,
3199                _ = interval.tick() => {
3200                    tokio::select! {
3201                        _ = cancel.cancelled() => break,
3202                        result = scanner.rebuild_once() => {
3203                            match result {
3204                                Ok(stats) if stats.unserviceable > 0 => tracing::error!(unserviceable = stats.unserviceable, first = ?stats.first_unserviceable, "CHASM rebuild isolated executions; healthy executions were rebuilt"),
3205                                Ok(_) => {},
3206                                Err(error) => tracing::warn!(?error, "CHASM outbox rebuild pass failed"),
3207                            }
3208                        }
3209                    }
3210                }
3211            }
3212        }
3213    });
3214}
3215
3216/// The interval the CHASM timer sweeper ticks at. Activity timeouts (schedule-to-
3217/// start/close, start-to-close, heartbeat) fire within roughly this cadence of their
3218/// deadline; ~200ms is well below the smallest conformance timeout while keeping the
3219/// sweep cheap (a snapshot of the armed-timer map plus a fenced update per due timer).
3220const CHASM_TIMER_SWEEP_INTERVAL: std::time::Duration = std::time::Duration::from_millis(200);
3221
3222/// Spawn the CHASM timer sweeper: it fires armed activity timeouts on a tick by
3223/// delegating to the [`TimeoutEvaluator`](tokeira_runtime::chasm::TimeoutEvaluator)
3224/// (the activity bridge), then re-arms each execution to its state-derived next
3225/// deadline. Runtime-only (clock + loop); all timeout/retry semantics are pure
3226/// (`tokeira-chasm-activity`) behind the evaluator, so the kernel-purity and
3227/// history-authority invariants hold. Runs until `cancel` fires. Gated on standalone
3228/// activities or extension libraries being enabled; generic registered handlers
3229/// drive extension timers while the activity evaluator keeps its existing path.
3230fn spawn_chasm_timer_sweeper(
3231    tasks: &RuntimeShutdownHandle,
3232    engine: Arc<tokeira_runtime::chasm::ChasmEngine>,
3233    evaluator: Arc<dyn tokeira_runtime::chasm::TimeoutEvaluator>,
3234    archetype_id: u32,
3235    cancel: CancellationToken,
3236) {
3237    let sweeper = tokeira_runtime::chasm::ChasmTimerSweeper::new(engine)
3238        .with_evaluator(archetype_id, evaluator);
3239    let _sweeper = tasks.spawn(async move {
3240        let mut interval = tokio::time::interval(CHASM_TIMER_SWEEP_INTERVAL);
3241        interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
3242        loop {
3243            tokio::select! {
3244                _ = cancel.cancelled() => break,
3245                _ = interval.tick() => {
3246                    sweeper.sweep_once().await;
3247                }
3248            }
3249        }
3250    });
3251}
3252
3253async fn build_service_stack_with_storage<R, L, S, V, F>(
3254    transport: StackTransport,
3255    effective_config: Arc<TokeiraConfig>,
3256    run_repository: Arc<R>,
3257    worker_deployment_repository: Arc<dyn WorkerDeploymentRepository>,
3258    worker_compute_repository: Option<Arc<dyn WorkerComputeRepository>>,
3259    task_queue_config_repository: Arc<dyn TaskQueueConfigRepository>,
3260    worker_task_provenance: Arc<dyn WorkerTaskProvenanceStore>,
3261    projection_log: L,
3262    visibility_query_store: V,
3263    projection_sink: F,
3264    dsql_endpoint: Option<String>,
3265    chasm_node_repo: Arc<dyn tokeira_storage::ChasmNodeRepository>,
3266    recover_self_assigned_shard: bool,
3267    budget_applier: Arc<dyn ConnectionBudgetApplier>,
3268    extensions: ChasmExtensions,
3269) -> Result<ConstructedStack>
3270where
3271    R: LeaseRepository + RunRepository + 'static,
3272    L: ProjectionLog + Clone + 'static,
3273    S: ProjectionSink + VisibilityStore + 'static,
3274    V: VisibilityStore + Clone + 'static,
3275    F: Fn() -> S + Clone + Send + Sync + 'static,
3276{
3277    // Construction spawns scanners and may seed an Active home. No runtime
3278    // may exist until every physical legacy row has reached its execution home.
3279    tokeira_storage::prepare_execution_placement(run_repository.as_ref()).await?;
3280    // Freeze declarations before constructing the operator wrapper, so later
3281    // namespaces receive the same set as startup. Stateful CHASM admission stays
3282    // in its existing block below, shared by both storage paths.
3283    let has_extensions = !extensions.libraries.is_empty();
3284    let mut registry_builder = tokeira_chasm::Registry::builder();
3285    tokeira_chasm_activity::ActivityLibrary::register(&mut registry_builder)
3286        .map_err(registry_start_error)?;
3287    registry_builder.seal_built_ins();
3288    for library in extensions.libraries {
3289        library(&mut registry_builder).map_err(registry_start_error)?;
3290    }
3291    let registry = Arc::new(registry_builder.build());
3292    let declared = declared_search_attributes(&registry);
3293    // Build the authoritative store first, then wrap it with the
3294    // history-notifying repository used by edge long-poll.
3295    let node_id = IncarnationId::new();
3296    let advertised_addr = match transport {
3297        StackTransport::Network(addr) => addr,
3298        StackTransport::Embedded => SocketAddr::from(([127, 0, 0, 1], 0)),
3299    };
3300    let node_endpoint = configured_node_endpoint(&effective_config, advertised_addr);
3301    let history_waits = HistoryWaitRegistry::default();
3302    let repo = Arc::new(HistoryNotifyingRepository::new(
3303        run_repository,
3304        history_waits.clone(),
3305    ));
3306    let default_namespace = ResolvedNamespace::active("default");
3307    let default_namespace_id = namespace_id_for("default");
3308
3309    // Bootstrap edge-facing namespace/operator state.
3310    let namespaces = Arc::new(InMemoryNamespaceCache::new());
3311    namespaces.insert(default_namespace).await?;
3312    // The Nexus endpoint store is the single source of truth shared by runtime
3313    // dispatch resolution and the OperatorService endpoint-admin CRUD. Seeded empty;
3314    // endpoints are created at runtime via `CreateNexusEndpoint`.
3315    let nexus_store = build_nexus_endpoint_store(
3316        namespaces.as_ref(),
3317        HashMap::<String, BootstrapNexusEndpointConfig>::new(),
3318    )
3319    .await?;
3320    let nexus_registry = NexusEndpointRegistry::new(nexus_store.clone());
3321    let nexus_http_client: Arc<dyn tokeira_runtime::NexusHttpClient> =
3322        Arc::new(HttpNexusClient::new());
3323
3324    // The runtime owns execution orchestration, scanners, brokers, and all
3325    // run-local in-memory coordination such as buffered consistent queries.
3326    let schedule_store = Arc::new(ScheduleStore::default());
3327    let task_queue_config_store = Arc::new(RepositoryBackedTaskQueueConfigStore::new(
3328        task_queue_config_repository,
3329    ));
3330    // A server never fabricates an unset/default policy when durable state was
3331    // merely unavailable: hydrate must succeed before polls can be accepted.
3332    task_queue_config_store
3333        .hydrate()
3334        .await
3335        .context("failed to hydrate task-queue policy")?;
3336    // Agreement is a startup prerequisite, before runtime construction or
3337    // membership can acquire a home. A later incompatible snapshot terminates
3338    // serving instead of silently changing the durable placement function.
3339    let initial_routing_cache = if let Some(endpoint) = &effective_config
3340        .infrastructure
3341        .placement
3342        .controller_endpoint
3343    {
3344        let cache = Arc::new(RoutingCache::new(placement_config(&effective_config)));
3345        tokio::time::timeout(
3346            std::time::Duration::from_secs(10),
3347            tokeira_edge::subscribe_routing_once(cache.clone(), endpoint.clone()),
3348        )
3349        .await
3350        .context("timed out waiting for controller placement snapshot")??;
3351        Some(cache)
3352    } else {
3353        None
3354    };
3355    let mut runtime_config = RuntimeConfig::default();
3356    runtime_config.lane.controller_managed_placement = effective_config
3357        .infrastructure
3358        .placement
3359        .controller_endpoint
3360        .is_some();
3361    let seed_default_shard = !recover_self_assigned_shard
3362        && dsql_endpoint.is_none()
3363        && effective_config
3364            .infrastructure
3365            .placement
3366            .controller_endpoint
3367            .is_none();
3368
3369    let nexus_completion_cfg = effective_config.policy.nexus_completion.clone();
3370    let (
3371        nexus_callback_listener,
3372        in_process_nexus_client,
3373        nexus_completion_client,
3374        resolved_system_callback_url,
3375    ): (
3376        Option<TcpListener>,
3377        Option<Arc<InProcessNexusCompletionClient>>,
3378        Arc<dyn NexusCompletionClient>,
3379        String,
3380    ) = match transport {
3381        StackTransport::Network(_) => {
3382            // Bind before runtime construction because the firing client's system
3383            // callback URL must contain the listener's actual ephemeral port.
3384            let listener = TcpListener::bind(&nexus_completion_cfg.http_addr)
3385                .await
3386                .with_context(|| {
3387                    format!(
3388                        "failed to bind nexus completion listener on {}",
3389                        nexus_completion_cfg.http_addr
3390                    )
3391                })?;
3392            let callback_addr = listener
3393                .local_addr()
3394                .context("failed to resolve bound nexus completion listener address")?;
3395            let callback_url = with_loopback_port(
3396                &nexus_completion_cfg.system_callback_url,
3397                callback_addr.port(),
3398            );
3399            info!(
3400                bind = %callback_addr,
3401                loopback = %callback_url,
3402                "nexus completion callback listener bound"
3403            );
3404            (
3405                Some(listener),
3406                None,
3407                Arc::new(HttpNexusCompletionClient::new()),
3408                callback_url,
3409            )
3410        }
3411        StackTransport::Embedded => {
3412            // A weak runtime attachment closes the system-callback loop without a
3413            // socket and without creating a runtime→client→runtime ownership cycle.
3414            let client = Arc::new(InProcessNexusCompletionClient::new());
3415            (
3416                None,
3417                Some(client.clone()),
3418                client,
3419                EMBEDDED_NEXUS_CALLBACK_BASE.to_owned(),
3420            )
3421        }
3422    };
3423    let nexus_completion_deps = NexusCompletionDeps {
3424        client: nexus_completion_client,
3425        config: NexusCompletionRuntimeConfig {
3426            system_callback_url: resolved_system_callback_url,
3427            retry_initial_interval: time::Duration::milliseconds(
3428                nexus_completion_cfg.retry_initial_interval_ms as i64,
3429            ),
3430            retry_max_interval: time::Duration::milliseconds(
3431                nexus_completion_cfg.retry_max_interval_ms as i64,
3432            ),
3433            retry_backoff_coefficient: nexus_completion_cfg.retry_backoff_coefficient,
3434            retry_max_attempts: nexus_completion_cfg.retry_max_attempts,
3435        },
3436        scanner: CompletionCallbackScannerConfig::default(),
3437    };
3438
3439    let chasm_nexus_client = nexus_completion_deps.client.clone();
3440    let chasm_nexus_config = nexus_completion_deps.config.clone();
3441
3442    let worker_compute_deployment_repository = worker_deployment_repository.clone();
3443    let runtime = TokeiraRuntime::new_with_nexus_and_shards_and_endpoint(
3444        repo.clone(),
3445        runtime_config.lane_count,
3446        runtime_config.lane,
3447        runtime_config.timer_scanner,
3448        runtime_config.workflow_timeout_scanner,
3449        runtime_config.backlog,
3450        runtime_config.activity_timeout_scanner,
3451        runtime_config.nexus_timeout_scanner,
3452        nexus_registry.clone(),
3453        nexus_http_client.clone(),
3454        // Listener-backed mode posts through HTTP. Embedded mode intercepts the
3455        // reserved system target and invokes the same callback handler inline.
3456        nexus_completion_deps,
3457        effective_config.infrastructure.placement.shard_count,
3458        node_id.to_string(),
3459        node_endpoint.as_authority(),
3460        seed_default_shard,
3461        // Namespace names for the publisher, resolved through the shared edge
3462        // namespace cache: the External-endpoint outbound Nexus metric's tag, and
3463        // the target namespace in external signal and cancel resolving events.
3464        Some(Arc::new(CacheNexusNamespaceResolver {
3465            namespaces: namespaces.clone(),
3466        }) as Arc<dyn NexusNamespaceResolver>),
3467    )
3468    // The edge always exposes Worker Deployment v2 RPCs. Wiring the
3469    // repository here keeps their registry durable for both in-memory and
3470    // DSQL backends instead of falling back to a detached test registry.
3471    .with_queue_placement(
3472        effective_config.infrastructure.placement.partition_count,
3473        effective_config.infrastructure.placement.bundle_count,
3474    )
3475    .with_worker_deployment_repository(worker_deployment_repository)
3476    .with_task_queue_config_store(task_queue_config_store.clone())
3477    .with_delivery_mode_provider(Arc::new(
3478        tokeira_runtime::ConfiguredDeliveryModeProvider::new(
3479            tokeira_runtime::StaticDeliveryPolicy {
3480                enable_fairness: effective_config.policy.task_queues.enable_fairness,
3481            },
3482        ),
3483    ));
3484    runtime.ready().await?;
3485    let runtime = Arc::new(runtime);
3486    let runtime_tasks = runtime.shutdown_handle();
3487    let engine_tasks = RuntimeShutdownHandle::new();
3488    let mut startup_guard = StackStartupGuard::new(runtime_tasks.clone(), engine_tasks.clone());
3489    let background_cancel = engine_tasks.cancellation_token();
3490
3491    let recovered_lease = if recover_self_assigned_shard {
3492        let shard_id = ShardId(0);
3493        let epoch = runtime
3494            .acquire_shard(shard_id)
3495            .await
3496            .context("failed to recover embedded snapshot shard")?;
3497        Some((shard_id, node_id.to_string(), epoch))
3498    } else {
3499        None
3500    };
3501
3502    if matches!(transport, StackTransport::Network(_))
3503        && dsql_endpoint.is_some()
3504        && effective_config
3505            .infrastructure
3506            .placement
3507            .controller_endpoint
3508            .is_none()
3509    {
3510        let _leases = self_assign_dsql_shards(
3511            runtime.as_ref(),
3512            repo.as_ref(),
3513            effective_config.infrastructure.placement.shard_count,
3514            &node_id,
3515            &node_endpoint,
3516            false,
3517        )
3518        .await?;
3519    }
3520
3521    let recovery_task = recovered_lease.map(|(shard_id, owner, epoch)| {
3522        let runtime = runtime.clone();
3523        let repo = repo.clone();
3524        let cancel = background_cancel.clone();
3525        tokio::spawn(async move {
3526            cancel.cancelled().await;
3527            // Drain first so the shard-scoped token stops the lease renewer,
3528            // then advance the durable epoch before a graceful snapshot. A
3529            // crash may leave an active owner in the last interval snapshot;
3530            // boot retires that captured owner before acquiring the new one.
3531            runtime.relinquish_shard(shard_id).await;
3532            match repo
3533                .relinquish_bundle(shard_id, owner, epoch)
3534                .await
3535                .context("failed to release embedded snapshot recovery lease")?
3536            {
3537                LeaseOutcome::Acquired { .. } => Ok(()),
3538                LeaseOutcome::Rejected {
3539                    current_owner,
3540                    current_epoch,
3541                } => Err(anyhow!(
3542                    "embedded snapshot recovery lease was fenced by owner {current_owner} at epoch {}",
3543                    current_epoch.0
3544                )),
3545                LeaseOutcome::Renewed { epoch } => Err(anyhow!(
3546                    "embedded snapshot recovery lease unexpectedly renewed at epoch {}",
3547                    epoch.0
3548                )),
3549            }
3550        })
3551    });
3552    let task_queue_config_refresh = task_queue_config_store
3553        .clone()
3554        .spawn_refresh(background_cancel.clone());
3555    let _task_queue_config_refresh = engine_tasks.track_join(task_queue_config_refresh);
3556    let _worker_task_provenance_cleanup = spawn_worker_task_provenance_cleanup(
3557        &engine_tasks,
3558        worker_task_provenance.clone(),
3559        background_cancel.clone(),
3560    );
3561
3562    let _worker_compute_service = if effective_config.policy.worker_compute.enabled {
3563        let controller_repository = worker_compute_repository
3564            .clone()
3565            .context("worker-compute enabled without a controller repository")?;
3566        let (observation_sender, observation_receiver) =
3567            mpsc::channel(OBSERVATION_CHANNEL_CAPACITY);
3568        let observation_sink = Arc::new(ChannelDemandObservationSink::new(observation_sender));
3569        runtime
3570            .broker()
3571            .set_demand_observation_sink(observation_sink.clone());
3572        runtime
3573            .activity_broker()
3574            .set_demand_observation_sink(observation_sink.clone());
3575        runtime
3576            .nexus_task_broker()
3577            .set_demand_observation_sink(observation_sink);
3578
3579        let (reconcile_sender, reconcile_receiver) = mpsc::channel(RECONCILE_CHANNEL_CAPACITY);
3580        runtime
3581            .deployment_registry()
3582            .expect("Worker Deployment repository was installed above")
3583            .set_worker_compute_reconcile_sink(Arc::new(ChannelWorkerComputeReconcileSink::new(
3584                reconcile_sender,
3585            )));
3586
3587        let clock = Arc::new(SystemWorkerComputeClock);
3588        let catalog = Arc::new(WorkerComputeNamespaceCatalogAdapter::new(
3589            namespaces.clone(),
3590        ));
3591        let reconciler = WorkerComputeReconciler::new(
3592            worker_compute_deployment_repository,
3593            controller_repository.clone(),
3594            node_id,
3595        );
3596        let sampler = runtime.worker_compute_queue_sampler(controller_repository.clone(), node_id);
3597        let provider = Arc::new(NexusWorkerComputeProvider::new(
3598            nexus_registry,
3599            nexus_http_client,
3600            runtime.nexus_task_broker(),
3601        ));
3602        let outbox = WorkerComputeOutbox::new(
3603            controller_repository.clone(),
3604            provider,
3605            clock.clone(),
3606            node_id,
3607        );
3608        let shard_runtime = runtime.clone();
3609        let active_shards: tokeira_runtime::WorkerComputeActiveShards =
3610            Arc::new(move || shard_runtime.active_shards());
3611        let service = WorkerComputeControllerService::new(
3612            catalog,
3613            controller_repository,
3614            reconciler,
3615            sampler,
3616            outbox,
3617            clock,
3618            active_shards,
3619            observation_receiver,
3620            reconcile_receiver,
3621        );
3622        let service_cancel = background_cancel.clone();
3623        Some(engine_tasks.spawn(async move {
3624            if let Err(error) = service.run(service_cancel).await {
3625                tracing::warn!(?error, "worker-compute controller service exited");
3626            }
3627        }))
3628    } else {
3629        None
3630    };
3631
3632    let membership_client = effective_config
3633        .infrastructure
3634        .placement
3635        .controller_endpoint
3636        .clone()
3637        .map(|controller_endpoint| {
3638            runtime.spawn_membership_client(
3639                membership_config(
3640                    &effective_config,
3641                    node_id,
3642                    node_endpoint.clone(),
3643                    controller_endpoint,
3644                ),
3645                budget_applier,
3646                background_cancel.clone(),
3647            )
3648        });
3649    if let Some(membership_client) = membership_client {
3650        let _membership_client = engine_tasks.track_join(membership_client);
3651    }
3652    let schedule_engine = run_schedule_engine(
3653        schedule_store.clone(),
3654        runtime.clone(),
3655        ScheduleEngineConfig::default(),
3656        background_cancel.clone(),
3657    );
3658    let _schedule_engine = engine_tasks.track_join(schedule_engine);
3659
3660    let authorization = build_authorization_stack(&effective_config).await?;
3661    let interceptors = Arc::new(EdgeInterceptors::configured(
3662        namespaces.clone(),
3663        authorization.grpc,
3664        authorization.principal_attribution,
3665    ));
3666    let router: Arc<dyn tokeira_edge::EdgeRouter> = if let Some(controller_endpoint) =
3667        effective_config
3668            .infrastructure
3669            .placement
3670            .controller_endpoint
3671            .clone()
3672    {
3673        let routing_cache = initial_routing_cache
3674            .expect("controller snapshot validated before runtime construction");
3675        let routing_config = EdgeRoutingConfig {
3676            controller_endpoint,
3677            max_retries: effective_config
3678                .infrastructure
3679                .placement
3680                .routing_max_retries,
3681        };
3682        let subscription_cache = routing_cache.clone();
3683        let subscription_cancel = background_cancel.clone();
3684        let routing_shutdown = runtime.shutdown_handle();
3685        let routing_cancel = background_cancel.clone();
3686        let _routing_subscription = engine_tasks.spawn(async move {
3687            if let Err(error) =
3688                run_routing_subscription(subscription_cache, routing_config, subscription_cancel)
3689                    .await
3690            {
3691                tracing::error!(
3692                    ?error,
3693                    "routing placement became incompatible; stopping node"
3694                );
3695                routing_shutdown.begin_shutdown();
3696                routing_cancel.cancel();
3697            }
3698        });
3699        runtime.set_workflow_task_starter(Arc::new(
3700            tokeira_edge::grpc::workflow_dispatch::RemoteWorkflowTaskStarter::new(
3701                routing_cache.clone(),
3702            ),
3703        ));
3704        Arc::new(CacheBackedRouter::new(routing_cache, node_id))
3705    } else {
3706        Arc::new(LocalOnlyRouter)
3707    };
3708    let workflow_broker = runtime.broker();
3709    let buffered_queries = runtime.buffered_queries();
3710    let worker_registry = runtime.worker_registry();
3711    let nexus_task_broker = runtime.nexus_task_broker();
3712    let runtime_adapter = Arc::new(RuntimeAdapter::new(runtime.clone()));
3713    let callback_runtime = runtime_adapter.clone() as Arc<dyn WorkflowRuntimeApi>;
3714    if let Some(client) = in_process_nexus_client {
3715        client.attach(&callback_runtime)?;
3716    }
3717    if let Some(listener) = nexus_callback_listener {
3718        // Listener-backed and embedded completion both delegate to this same
3719        // `WorkflowRuntimeApi`; only the transport around the callback differs.
3720        spawn_nexus_callback_server(listener, callback_runtime, background_cancel.clone());
3721    }
3722    let resolver = Arc::new(StoreExecutionResolver::new(repo.clone()));
3723    tokeira_projection::seed_predefined_search_attributes(
3724        &visibility_query_store,
3725        default_namespace_id,
3726    )
3727    .await
3728    .context("failed to seed Temporal predefined search attributes")?;
3729    let operator_visibility_store = visibility_query_store.clone();
3730    let visibility = Arc::new(VisibilityQueryService::new(visibility_query_store.clone()));
3731    let long_polls = LongPollGate::new(LongPollConfig::default());
3732    let operator_api = Arc::new(
3733        VisibilityRegistryOperatorApi::new(
3734            InMemoryOperatorApi::new("tokeira-local", tokeira_build_info::SERVER_VERSION),
3735            operator_visibility_store,
3736        )
3737        .with_declared_search_attributes(declared.clone()),
3738    );
3739
3740    for partition_id in 0..effective_config.infrastructure.placement.partition_count {
3741        let projection_worker = ProjectionWorker {
3742            log: projection_log.clone(),
3743            sink: projection_sink(),
3744            batch_size: 256,
3745        };
3746        let projection_cancel = background_cancel.clone();
3747        let _projection_worker = engine_tasks.spawn(async move {
3748            if let Err(error) = projection_worker
3749                .run_from_cursor(
3750                    projection_cancel,
3751                    ProjectionCursor::beginning(partition_id, 1),
3752                )
3753                .await
3754            {
3755                tracing::warn!(?error, partition_id, "projection worker exited");
3756            }
3757        });
3758    }
3759
3760    let nexus_http_waiters = tokeira_edge::nexus_http::NexusHttpWaiterRegistry::default();
3761    let workflow_service =
3762        WorkflowService::new_with_stores_and_buffered_queries_and_history_wait_registry(
3763            runtime_adapter.clone(),
3764            resolver,
3765            visibility,
3766            repo.clone(),
3767            operator_api.clone(),
3768            namespaces.clone(),
3769            interceptors.clone(),
3770            PollerRegistry::default(),
3771            PendingQueryStore::default(),
3772            buffered_queries,
3773            workflow_broker,
3774            nexus_task_broker.clone(),
3775            long_polls,
3776            router,
3777            history_waits,
3778            worker_registry,
3779            runtime.heartbeat_store(),
3780            schedule_store,
3781            task_queue_config_store,
3782            Arc::new(tokeira_runtime::BatchOperationStore::default()),
3783        )
3784        .with_nexus_http_waiters(nexus_http_waiters.clone())
3785        .with_worker_task_provenance(worker_task_provenance)
3786        .with_scoped_worker_sessions(ScopedWorkerSessionRegistry::default())
3787        .with_worker_deployment_runtime(runtime_adapter);
3788    // The Nexus endpoint admin shares the dispatch store, gated through the operator
3789    // interceptor (create/update/delete = OperatorWrite, get/list = OperatorRead).
3790    // Limits come from config (raise, never hardcode); the namespace resolver backs
3791    // the Worker-target existence check.
3792    let nexus_endpoint_limits = {
3793        let cfg = &effective_config.policy.nexus_endpoint_limits;
3794        tokeira_edge::nexus_endpoint::NexusEndpointLimits {
3795            name_max_length: cfg.name_max_length,
3796            external_url_max_length: cfg.external_url_max_length,
3797            description_max_size: cfg.description_max_size,
3798            task_queue_max_length: cfg.task_queue_max_length,
3799            list_default_page_size: cfg.list_default_page_size,
3800            list_max_page_size: cfg.list_max_page_size,
3801        }
3802    };
3803    let nexus_endpoint_admin = Arc::new(tokeira_edge::nexus_endpoint::NexusEndpointAdmin::new(
3804        nexus_store.clone(),
3805        Arc::new(tokeira_edge::nexus_endpoint::CacheNamespaceResolver::new(
3806            namespaces.clone(),
3807        )),
3808        nexus_endpoint_limits,
3809    ));
3810    let dispatch_grpc = tokeira_edge::grpc::workflow_dispatch::WorkflowTaskDispatchGrpc::new(
3811        runtime.clone(),
3812        interceptors.clone(),
3813        workflow_service.clone(),
3814    );
3815    let operator_service = OperatorService::new(operator_api, interceptors)
3816        .with_nexus_endpoints(nexus_endpoint_admin)
3817        .with_namespace_deletion(namespaces.clone(), Arc::new(workflow_service.clone()));
3818    let production_nexus_authorizer = authorization.nexus;
3819    let nexus_http_authorizer: Arc<dyn tokeira_edge::nexus_http::NexusHttpAuthorizer> =
3820        match harness::installed().and_then(|hooks| hooks.wrap_nexus_http_authorizer.as_ref()) {
3821            Some(wrap) => wrap(production_nexus_authorizer),
3822            None => production_nexus_authorizer,
3823        };
3824    let nexus_http_layer =
3825        NexusHttpLayer::new(Arc::new(tokeira_edge::nexus_http::NexusHttpHandler::new(
3826            namespaces.clone(),
3827            nexus_store,
3828            nexus_task_broker,
3829            nexus_http_waiters,
3830            nexus_http_authorizer,
3831        )));
3832    let http_api_layer = HttpApiLayer::new(
3833        HttpApiCatalog::pinned().context("failed to build Temporal HTTP API route catalog")?,
3834        HttpApiPolicy::new(
3835            effective_config.policy.http_api.allowed_hosts.clone(),
3836            effective_config
3837                .policy
3838                .http_api
3839                .additional_forwarded_headers
3840                .clone(),
3841        )
3842        .context("failed to compile Temporal HTTP API policy")?,
3843    );
3844
3845    // Wire the standalone-activity (CHASM) bridge onto the gRPC adapter: a CHASM
3846    // engine over the backend's node repository, an activity-library registry, and
3847    // a shared dispatch queue the engine routes committed dispatch tasks into and a
3848    // worker poll drains. The enable gate is operator config — off by default, so an
3849    // unconfigured server matches the `v1.31.0` baseline (RPCs answer
3850    // `UNIMPLEMENTED`); enabling it is a declared deviation (`AGENTS §8`).
3851    let (workflow_grpc, chasm_engine) = {
3852        // Capture what the visibility repair scanner needs before the engine consumes
3853        // the node repo + registry: the activity id selects the legacy evaluator;
3854        // registered root adapters rebuild visibility for every archetype.
3855        let repair_archetype = registry.archetype_id(
3856            <tokeira_chasm_activity::ActivityExecution as tokeira_chasm::Component>::FQN,
3857        );
3858        // Finish pointer migration before the CHASM engine can admit a start. The
3859        // driver skips completed backfills and preserves any newer scoped pointer.
3860        tokeira_storage::run_current_execution_backfill(
3861            chasm_node_repo.as_ref(),
3862            repair_archetype.context("activity CHASM archetype is not registered")?,
3863            500,
3864        )
3865        .await
3866        .context("failed to backfill CHASM current executions")?;
3867        check_registered_archetypes(&chasm_node_repo.distinct_archetypes().await?, &registry)
3868            .map_err(registry_start_error)?;
3869        let repair_nodes = chasm_node_repo.clone();
3870        let dispatch_queue = Arc::new(tokeira_edge::chasm_activity::ActivityDispatchQueue::new());
3871        let multiplexer = Arc::new(tokeira_runtime::chasm::DispatchMultiplexer::default());
3872        // Standalone activities flow into the shared visibility index via the
3873        // engine→projection adapter, post-commit and off the correctness path
3874        // (spec task 24.2). It reuses the same projection apply path as the workflow
3875        // visibility worker, so both archetypes land in one logical index.
3876        let chasm_visibility_sink =
3877            Arc::new(tokeira_runtime::chasm::ProjectionVisibilitySink::new(
3878                Arc::new(projection_sink()),
3879                effective_config.infrastructure.placement.partition_count,
3880            ));
3881        let mut chasm_engine = tokeira_runtime::chasm::ChasmEngine::new(
3882            chasm_node_repo.clone(),
3883            registry.clone(),
3884            multiplexer.clone(),
3885            chasm_visibility_sink,
3886        );
3887        if let Some(clock) = extensions.clock {
3888            chasm_engine = chasm_engine.with_clock(clock);
3889        }
3890        let chasm_engine = Arc::new(chasm_engine);
3891        let activity_config = tokeira_chasm_activity::ActivityConfig {
3892            enable_standalone: effective_config
3893                .policy
3894                .compatibility
3895                .enable_standalone_activities,
3896            enable_callbacks: effective_config
3897                .policy
3898                .compatibility
3899                .enable_standalone_activity_callbacks,
3900            ..tokeira_chasm_activity::ActivityConfig::default()
3901        };
3902        let standalone_enabled = activity_config.enable_standalone;
3903        // Keep an engine handle for the timer sweeper before the bridge takes it.
3904        let sweeper_engine = chasm_engine.clone();
3905        let dispatch_executor = Arc::new(
3906            tokeira_edge::chasm_executors::ActivityDispatchExecutor::new(
3907                Arc::downgrade(&chasm_engine),
3908                dispatch_queue,
3909            ),
3910        );
3911        let activity_bridge = Arc::new(
3912            tokeira_edge::chasm_activity::ActivityBridge::new(
3913                chasm_engine.clone(),
3914                activity_config.clone(),
3915                DEFAULT_MAX_ID_LENGTH,
3916            )
3917            .with_dispatch_executor(dispatch_executor.clone()),
3918        );
3919        // Weak engine handles break the ownership cycle. Register every role before
3920        // rebuild/serve; extension executors join the same multiplexer below.
3921        multiplexer
3922            .register(dispatch_executor)
3923            .map_err(registry_start_error)?;
3924        multiplexer
3925            .register(Arc::new(
3926                tokeira_edge::chasm_executors::StartActivityExecutor::new(
3927                    Arc::downgrade(&chasm_engine),
3928                    activity_config,
3929                    DEFAULT_MAX_ID_LENGTH,
3930                ),
3931            ))
3932            .map_err(registry_start_error)?;
3933        multiplexer
3934            .register(Arc::new(
3935                tokeira_edge::chasm_executors::DeliverCallbackExecutor::new(
3936                    Arc::downgrade(&chasm_engine),
3937                    chasm_nexus_client,
3938                    chasm_nexus_config,
3939                    namespaces.clone(),
3940                ),
3941            ))
3942            .map_err(registry_start_error)?;
3943        for executor in extensions.executors {
3944            multiplexer
3945                .register(executor)
3946                .map_err(registry_start_error)?;
3947        }
3948        // Resolve before registering: projection registration alone silently
3949        // preserves an existing id even if its type differs from the declaration.
3950        for namespace in namespaces.list_all().await? {
3951            seed_declared_search_attributes(
3952                &visibility_query_store,
3953                namespace_id_for(&namespace.name),
3954                &declared,
3955            )
3956            .await
3957            .map_err(seed_start_error)?;
3958        }
3959        let rebuild = tokeira_runtime::chasm::OutboxRebuildScanner::new(
3960            chasm_node_repo,
3961            chasm_engine.clone(),
3962            multiplexer.clone(),
3963        );
3964        let rebuild_stats = rebuild
3965            .rebuild_once()
3966            .await
3967            .context("failed to rebuild CHASM outboxes before serving")?;
3968        check_rebuilt_outboxes(rebuild_stats)?;
3969        spawn_outbox_rebuild(&engine_tasks, rebuild, background_cancel.clone());
3970        // Spawn the visibility repair scanner (Req 10.11): a committed transition can
3971        // never permanently lack a projection — the scanner rebuilds each execution's
3972        // snapshot from authoritative node state and re-applies it iff-newer. This is
3973        // the AGENTS-3 sweeper shape (derived effect reconstructed from authoritative
3974        // state), the durability backstop for the best-effort post-commit write (24.2).
3975        spawn_visibility_repair(
3976            &engine_tasks,
3977            repair_nodes,
3978            Arc::new(projection_sink()),
3979            registry.clone(),
3980            effective_config.infrastructure.placement.partition_count,
3981            background_cancel.clone(),
3982        );
3983        // Spawn the CHASM timer sweeper (`chasm-activity-timeouts-and-retry`): nothing
3984        // else fires armed activity timeouts. A conformance harness must keep it ready
3985        // even when the boot-time default is off, because the corpus enables
3986        // `activity.enableStandalone` live after server startup. Production retains
3987        // the configured gate; registered extension libraries also need generic timers.
3988        let sweeper_forced =
3989            harness::installed().is_some_and(|hooks| hooks.force_chasm_timer_sweeper);
3990        if standalone_enabled || sweeper_forced || has_extensions {
3991            spawn_chasm_timer_sweeper(
3992                &engine_tasks,
3993                sweeper_engine,
3994                activity_bridge.clone(),
3995                repair_archetype.context("activity CHASM archetype is not registered")?,
3996                background_cancel.clone(),
3997            );
3998        }
3999        (
4000            WorkflowServiceGrpc::new(workflow_service.clone()).with_chasm_activity(activity_bridge),
4001            chasm_engine,
4002        )
4003    };
4004    // Minimal AdminService (DescribeMutableState) shares the WorkflowService's
4005    // run repository — the reset conformance suite reads a run's ResetRunId/status.
4006    let admin_grpc = AdminServiceGrpc::new(workflow_service);
4007    let operator_grpc = OperatorServiceGrpc::new(operator_service);
4008
4009    // Embedded startup defers self-assignment until every other fallible stack
4010    // construction step has completed. Admission is not opened until all
4011    // configured leases are held, and the returned cleanup owns the exact
4012    // `(shard, owner, epoch)` tuples needed for conditional release.
4013    let shard_cleanup = if matches!(transport, StackTransport::Embedded)
4014        && dsql_endpoint.is_some()
4015        && effective_config
4016            .infrastructure
4017            .placement
4018            .controller_endpoint
4019            .is_none()
4020    {
4021        let leases = self_assign_dsql_shards(
4022            runtime.as_ref(),
4023            repo.as_ref(),
4024            effective_config.infrastructure.placement.shard_count,
4025            &node_id,
4026            &node_endpoint,
4027            true,
4028        )
4029        .await?;
4030        Some(self_assigned_shard_cleanup(
4031            runtime.clone(),
4032            repo.clone(),
4033            leases,
4034        ))
4035    } else {
4036        None
4037    };
4038
4039    match dsql_endpoint {
4040        Some(endpoint) => info!(%endpoint, "storage backend: dsql"),
4041        None => info!("storage backend: in-memory"),
4042    }
4043
4044    let log_broadcast = broadcast::Sender::<LogEvent>::new(LOG_BROADCAST_CAPACITY);
4045    let addr = match transport {
4046        StackTransport::Embedded => {
4047            startup_guard.disarm();
4048            return Ok(ConstructedStack::Embedded(EmbeddedStack {
4049                component_visibility_store: chasm_extensions::VisibilityQueryStore(Arc::new(
4050                    visibility_query_store,
4051                )),
4052                chasm_engine,
4053                registry,
4054                service: InProcessGrpcService::new(workflow_grpc, operator_grpc, admin_grpc),
4055                background_cancel,
4056                log_broadcast,
4057                recovery_task,
4058                runtime_tasks,
4059                engine_tasks,
4060                shard_cleanup,
4061            }));
4062        }
4063        StackTransport::Network(addr) => addr,
4064    };
4065
4066    // Both reflection protocols are served: `v1alpha` is what the previous
4067    // stack offered and older tooling still asks for, `v1` is the current
4068    // protocol newer tooling tries first.
4069    let reflection_v1 = tonic_reflection::server::Builder::configure()
4070        .register_encoded_file_descriptor_set(tokeira_proto::public::FILE_DESCRIPTOR_SET)
4071        .build_v1()
4072        .context("failed to build gRPC reflection service")?;
4073    let reflection_v1alpha = tonic_reflection::server::Builder::configure()
4074        .register_encoded_file_descriptor_set(tokeira_proto::public::FILE_DESCRIPTOR_SET)
4075        .build_v1alpha()
4076        .context("failed to build gRPC reflection service")?;
4077
4078    let listener = TcpListener::bind(addr)
4079        .await
4080        .with_context(|| format!("failed to bind tokeirad gRPC listener on {addr}"))?;
4081    let bound_addr = listener
4082        .local_addr()
4083        .context("failed to resolve bound local address for tokeirad gRPC listener")?;
4084
4085    let (shutdown_tx, shutdown_rx) = oneshot::channel();
4086
4087    // Tier-2 functional conformance (see `.kiro/specs/temporal-functional-conformance`)
4088    // captures every served `(wire_method, status_code)` at the transport boundary. The
4089    // capturing tower layer is mounted ONLY when the conformance flag is set, so a normal
4090    // production server never installs it and pays zero per-call cost.
4091    //
4092    // The recorder is constructed here, BEFORE the server task is spawned, and an
4093    // `Arc::clone` is moved into the task for the layer while the original handle is
4094    // returned to the caller. This is what lets the caller snapshot the observed coverage
4095    // after the server has shut down (the layer's clone is dropped with the task, but the
4096    // returned `Arc` keeps the counts alive) and export it as JSON evidence — see
4097    // [`run_from_cli`]. When the flag is off the handle is `None`: nothing is mounted,
4098    // nothing is snapshotted, and there is zero behavioural change.
4099    let wire_coverage_recorder = wire_coverage_enabled().then(|| {
4100        info!("conformance wire-coverage recorder enabled");
4101        Arc::new(WireCoverageRecorder::new())
4102    });
4103    let server_recorder = wire_coverage_recorder.clone();
4104
4105    let server_cancel = background_cancel.clone();
4106    let server_task = tokio::spawn(async move {
4107        let shutdown_signal = async move {
4108            tokio::select! {
4109                _ = shutdown_rx => {},
4110                _ = server_cancel.cancelled() => {},
4111            }
4112        };
4113
4114        // The conformance layer changes the server's tower-stack type, so the flagged and
4115        // unflagged paths are distinct server builds rather than a conditionally-mutated
4116        // builder. They are otherwise identical; only the extra `.layer()` differs.
4117        match server_recorder {
4118            Some(recorder) => {
4119                Server::builder()
4120                    .accept_http1(true)
4121                    .layer(nexus_http_layer.clone())
4122                    .layer(http_api_layer.clone())
4123                    .layer(CorsLayer::permissive())
4124                    .layer(GrpcWebLayer::new())
4125                    .layer(WireCoverageLayer::new(recorder))
4126                    .add_service(workflow_grpc.into_service())
4127                    .add_service(operator_grpc.into_service())
4128                    .add_service(admin_grpc.into_service())
4129                    .add_service(dispatch_grpc.into_service())
4130                    .add_service(reflection_v1)
4131                    .add_service(reflection_v1alpha)
4132                    .serve_with_incoming_shutdown(TcpListenerStream::new(listener), shutdown_signal)
4133                    .await
4134            }
4135            None => {
4136                Server::builder()
4137                    .accept_http1(true)
4138                    .layer(nexus_http_layer)
4139                    .layer(http_api_layer)
4140                    .layer(CorsLayer::permissive())
4141                    .layer(GrpcWebLayer::new())
4142                    .add_service(workflow_grpc.into_service())
4143                    .add_service(operator_grpc.into_service())
4144                    .add_service(admin_grpc.into_service())
4145                    .add_service(dispatch_grpc.into_service())
4146                    .add_service(reflection_v1)
4147                    .add_service(reflection_v1alpha)
4148                    .serve_with_incoming_shutdown(TcpListenerStream::new(listener), shutdown_signal)
4149                    .await
4150            }
4151        }
4152        .with_context(|| format!("failed to serve gRPC transport on {bound_addr}"))?;
4153        Ok::<(), anyhow::Error>(())
4154    });
4155
4156    // Harness-owned background work (the conformance dynamic-config control
4157    // listener, spec `.kiro/specs/conformance-config-override/`) spawns at the
4158    // same point the stack's own background tasks do, tied to the same
4159    // cancellation token so a graceful drain also stops the harness side.
4160    // Production installs no hooks, so this is a no-op outside a harness.
4161    if let Some(task) = harness::installed().and_then(|hooks| hooks.background_task.as_ref()) {
4162        task(background_cancel.clone());
4163    }
4164
4165    startup_guard.disarm();
4166    Ok(ConstructedStack::Network((
4167        server_task,
4168        bound_addr,
4169        shutdown_tx,
4170        background_cancel,
4171        log_broadcast,
4172        wire_coverage_recorder,
4173    )))
4174}
4175
4176/// Replace the port in a `scheme://host[:port]` loopback base URL with the actually-bound
4177/// listener `port`. This keeps `system_callback_url` (scheme + host from config) pointed at
4178/// the listener even when `http_addr` requested an ephemeral `:0` port.
4179fn with_loopback_port(base: &str, port: u16) -> String {
4180    let trimmed = base.trim_end_matches('/');
4181    match trimmed.rsplit_once(':') {
4182        // Only treat the suffix as a port when it is all digits — otherwise the match is
4183        // the `:` inside the scheme (`http://host` with no port) and we append instead.
4184        Some((prefix, suffix))
4185            if !suffix.is_empty() && suffix.bytes().all(|b| b.is_ascii_digit()) =>
4186        {
4187            format!("{prefix}:{port}")
4188        }
4189        _ => format!("{trimmed}:{port}"),
4190    }
4191}
4192
4193/// Serve the inbound `POST /nexus/callback` completion endpoint on `listener` until
4194/// `cancel` fires. Each request is delegated to [`handle_nexus_callback`], which resolves
4195/// the originator workflow's pending operation — the inbound half of the async-completion
4196/// loopback the runtime's firing client drives.
4197fn spawn_nexus_callback_server(
4198    listener: TcpListener,
4199    runtime: Arc<dyn WorkflowRuntimeApi>,
4200    cancel: CancellationToken,
4201) {
4202    tokio::spawn(async move {
4203        loop {
4204            let stream = tokio::select! {
4205                _ = cancel.cancelled() => break,
4206                accept = listener.accept() => match accept {
4207                    Ok((stream, _peer)) => stream,
4208                    Err(error) => {
4209                        tracing::warn!(?error, "nexus callback listener accept failed");
4210                        continue;
4211                    }
4212                },
4213            };
4214            let io = TokioIo::new(stream);
4215            let runtime = runtime.clone();
4216            let conn_cancel = cancel.clone();
4217            tokio::spawn(async move {
4218                let service = service_fn(move |request| {
4219                    let runtime = runtime.clone();
4220                    async move {
4221                        Ok::<_, Infallible>(
4222                            nexus_callback_response(runtime.as_ref(), request).await,
4223                        )
4224                    }
4225                });
4226                let conn = hyper::server::conn::http1::Builder::new().serve_connection(io, service);
4227                tokio::pin!(conn);
4228                tokio::select! {
4229                    result = conn.as_mut() => {
4230                        if let Err(error) = result {
4231                            tracing::debug!(?error, "nexus callback connection closed with error");
4232                        }
4233                    }
4234                    _ = conn_cancel.cancelled() => {
4235                        conn.as_mut().graceful_shutdown();
4236                        let _ = conn.await;
4237                    }
4238                }
4239            });
4240        }
4241    });
4242}
4243
4244/// Outcome of reading a request body under a byte cap (see [`collect_bounded_body`]).
4245#[derive(Debug)]
4246enum BoundedBody {
4247    /// The body fit within the cap and was fully buffered.
4248    Collected(Bytes),
4249    /// The body exceeded the cap; collection stopped early. Maps to `413`.
4250    TooLarge,
4251    /// The underlying stream errored before completing. Maps to `400`.
4252    ReadFailed(Box<dyn std::error::Error + Send + Sync>),
4253}
4254
4255/// Buffer `body` into memory, refusing to accumulate more than `limit` bytes.
4256///
4257/// [`Limited`] stops polling once the running total would exceed `limit` and surfaces a
4258/// downcastable [`LengthLimitError`], letting the caller reject an oversized body with
4259/// `413` *before* it is fully read — the bound that keeps the network-exposed
4260/// `/nexus/callback` listener from being an unauthenticated memory-exhaustion vector.
4261async fn collect_bounded_body<B>(body: B, limit: usize) -> BoundedBody
4262where
4263    B: hyper::body::Body<Data = Bytes>,
4264    B::Error: Into<Box<dyn std::error::Error + Send + Sync>>,
4265{
4266    match Limited::new(body, limit).collect().await {
4267        Ok(collected) => BoundedBody::Collected(collected.to_bytes()),
4268        Err(error) if error.downcast_ref::<LengthLimitError>().is_some() => BoundedBody::TooLarge,
4269        Err(error) => BoundedBody::ReadFailed(error),
4270    }
4271}
4272
4273/// Adapt a hyper request to the transport-agnostic edge handler. Non-`POST` requests and
4274/// any path other than `/nexus/callback` are 404; the handler owns all other status
4275/// mapping (decode failures → 400, accepted → 200, not-found/stale → 404, internal → 503).
4276async fn nexus_callback_response(
4277    runtime: &dyn WorkflowRuntimeApi,
4278    request: Request<Incoming>,
4279) -> Response<Full<Bytes>> {
4280    let (parts, body) = request.into_parts();
4281
4282    if parts.method != Method::POST || parts.uri.path() != NEXUS_CALLBACK_PATH {
4283        return nexus_response(StatusCode::NOT_FOUND, None);
4284    }
4285
4286    let header = |name: &str| {
4287        parts
4288            .headers
4289            .get(name)
4290            .and_then(|value| value.to_str().ok())
4291            .map(str::to_owned)
4292    };
4293    let token = header(TEMPORAL_CALLBACK_TOKEN_HEADER);
4294    let operation_token = header(NEXUS_OPERATION_TOKEN_HEADER);
4295    let state = header(NEXUS_OPERATION_STATE_HEADER);
4296    let content_type = header(hyper::header::CONTENT_TYPE.as_str());
4297
4298    // Bound the body before buffering it. This listener is network-exposed (default
4299    // bind is loopback, but operators may widen it) and reads the whole body *before*
4300    // `handle_nexus_callback` validates the callback token, so an unbounded `collect()`
4301    // is an unauthenticated memory-exhaustion vector — and the process runs
4302    // `panic = "abort"`, so an OOM abort drops every in-flight workflow on the node.
4303    // Mirror the cap the caller-facing Nexus transport already enforces.
4304    let body_bytes = match collect_bounded_body(body, MAX_NEXUS_PAYLOAD_BYTES).await {
4305        BoundedBody::Collected(bytes) => bytes,
4306        BoundedBody::TooLarge => {
4307            tracing::debug!(
4308                limit = MAX_NEXUS_PAYLOAD_BYTES,
4309                "nexus callback body too large"
4310            );
4311            return nexus_response(
4312                StatusCode::PAYLOAD_TOO_LARGE,
4313                Some(b"request body too large".to_vec()),
4314            );
4315        }
4316        BoundedBody::ReadFailed(error) => {
4317            tracing::debug!(?error, "failed to read nexus callback request body");
4318            return nexus_response(
4319                StatusCode::BAD_REQUEST,
4320                Some(b"failed to read request body".to_vec()),
4321            );
4322        }
4323    };
4324
4325    let CallbackResponse { status, body } = handle_nexus_callback(
4326        runtime,
4327        token.as_deref(),
4328        operation_token.as_deref(),
4329        state.as_deref(),
4330        content_type.as_deref(),
4331        &body_bytes,
4332    )
4333    .await;
4334
4335    let status = StatusCode::from_u16(status).unwrap_or(StatusCode::INTERNAL_SERVER_ERROR);
4336    nexus_response(status, body)
4337}
4338
4339fn nexus_response(status: StatusCode, body: Option<Vec<u8>>) -> Response<Full<Bytes>> {
4340    let mut response = Response::new(Full::new(Bytes::from(body.unwrap_or_default())));
4341    *response.status_mut() = status;
4342    response
4343}
4344
4345/// Broadcast buffer for [`LogEvent`]s fanned out to test harnesses.
4346///
4347/// Dimensioned so a slow consumer does not drop observation events under
4348/// typical integration-test cadence (one RPC per worker tick at ~30s).
4349const LOG_BROADCAST_CAPACITY: usize = 256;
4350
4351/// Environment flag selecting the Tier-2 wire-coverage capture layer.
4352///
4353/// Set during a functional-conformance run (see
4354/// `.kiro/specs/temporal-functional-conformance`) so the gRPC server mounts the
4355/// `WireCoverageLayer`. Read as a single env var rather than threaded through
4356/// `TokeiraConfig` because this is a conformance-harness concern, not a server-config
4357/// surface, and must add nothing to the production config schema.
4358const WIRE_COVERAGE_ENV: &str = "TOKEIRA_CONFORMANCE_WIRE_COVERAGE";
4359
4360/// Environment variable naming the file the wire-coverage evidence is written to.
4361///
4362/// Read only when the wire-coverage layer is active (see [`WIRE_COVERAGE_ENV`]). Kept a
4363/// separate var from the enable flag so an operator can leave the output location pinned
4364/// across runs while toggling capture on and off, and so the default path is never
4365/// silently inferred from the enable value. Like [`WIRE_COVERAGE_ENV`] it is read as a
4366/// single env var rather than threaded through `TokeiraConfig`, because the export path is
4367/// a conformance-harness concern, not a production server-config surface.
4368const WIRE_COVERAGE_OUT_ENV: &str = "TOKEIRA_CONFORMANCE_WIRE_COVERAGE_OUT";
4369
4370/// Default wire-coverage evidence path used when capture is on but
4371/// [`WIRE_COVERAGE_OUT_ENV`] is unset.
4372///
4373/// A repo-relative `./wire-coverage.json` is chosen so an operator who enables capture
4374/// without naming a path still gets evidence in the working directory rather than nothing;
4375/// the report can then be pointed at the default. It is never used when capture is off.
4376const WIRE_COVERAGE_DEFAULT_OUT: &str = "./wire-coverage.json";
4377
4378/// Whether the Tier-2 wire-coverage layer should be mounted.
4379///
4380/// True only when [`WIRE_COVERAGE_ENV`] is present and not one of the empty/false-y
4381/// values, so an accidentally-exported empty variable does not silently enable capture in
4382/// production. Any other value (e.g. `1`, `true`, `on`) enables it.
4383fn wire_coverage_enabled() -> bool {
4384    match std::env::var(WIRE_COVERAGE_ENV) {
4385        Ok(value) => {
4386            let trimmed = value.trim();
4387            !trimmed.is_empty()
4388                && !trimmed.eq_ignore_ascii_case("0")
4389                && !trimmed.eq_ignore_ascii_case("false")
4390        }
4391        Err(_) => false,
4392    }
4393}
4394
4395/// Resolve the path the wire-coverage evidence is written to.
4396///
4397/// Returns the value of [`WIRE_COVERAGE_OUT_ENV`] when it is set to a non-empty value,
4398/// otherwise [`WIRE_COVERAGE_DEFAULT_OUT`]. An empty or whitespace-only override is
4399/// treated as unset rather than as a request to write to a blank path, mirroring the
4400/// false-y handling in [`wire_coverage_enabled`] so an accidentally-exported empty
4401/// variable falls back to the documented default instead of erroring.
4402fn wire_coverage_out_path() -> PathBuf {
4403    match std::env::var(WIRE_COVERAGE_OUT_ENV) {
4404        Ok(value) if !value.trim().is_empty() => PathBuf::from(value.trim()),
4405        _ => PathBuf::from(WIRE_COVERAGE_DEFAULT_OUT),
4406    }
4407}
4408
4409/// Build the shared Nexus endpoint store, seeding any bootstrap-configured
4410/// endpoints. The same store backs both runtime dispatch resolution (via
4411/// [`NexusEndpointRegistry`]) and the OperatorService endpoint-admin CRUD, so a
4412/// `CreateNexusEndpoint` at runtime is immediately resolvable for dispatch and a
4413/// seeded endpoint is visible to admin reads. Worker targets resolve the namespace
4414/// name to its id up front (failing if the namespace is unknown), storing both so
4415/// reads echo the name and dispatch routes on the id.
4416async fn build_nexus_endpoint_store(
4417    namespaces: &dyn NamespaceCache,
4418    configs: HashMap<String, BootstrapNexusEndpointConfig>,
4419) -> Result<Arc<InMemoryNexusEndpointStore>> {
4420    let store = Arc::new(InMemoryNexusEndpointStore::new());
4421    let now = std::time::SystemTime::now()
4422        .duration_since(std::time::UNIX_EPOCH)
4423        .map(|d| d.as_nanos() as i64)
4424        .unwrap_or(0);
4425    for (endpoint_name, config) in configs {
4426        let target = match config.target {
4427            BootstrapNexusEndpointTarget::External { address } => {
4428                NexusEndpointSpecTarget::External { url: address }
4429            }
4430            BootstrapNexusEndpointTarget::Worker {
4431                namespace_name,
4432                task_queue,
4433            } => {
4434                let namespace = namespaces.get(&namespace_name).await?.ok_or_else(|| {
4435                    anyhow!(
4436                        "failed to register nexus worker endpoint `{endpoint_name}`: namespace `{namespace_name}` not found"
4437                    )
4438                })?;
4439                NexusEndpointSpecTarget::Worker {
4440                    namespace_name: namespace.name.clone(),
4441                    namespace_id: namespace_id_for(&namespace.name).0.to_string(),
4442                    task_queue,
4443                }
4444            }
4445        };
4446        store
4447            .create(
4448                NexusEndpointSpec {
4449                    name: endpoint_name.clone(),
4450                    description: Vec::new(),
4451                    target,
4452                },
4453                now,
4454            )
4455            .map_err(|e| anyhow!("failed to seed nexus endpoint `{endpoint_name}`: {e}"))?;
4456    }
4457    Ok(store)
4458}
4459
4460async fn self_assign_dsql_shards<R>(
4461    runtime: &TokeiraRuntime<R>,
4462    lease_repository: &R,
4463    shard_count: u32,
4464    node_id: &IncarnationId,
4465    node_endpoint: &NodeEndpoint,
4466    require_all: bool,
4467) -> Result<Vec<SelfAssignedShardLease>>
4468where
4469    R: LeaseRepository + RunRepository + 'static,
4470{
4471    // In single-node compose mode (no controller), any existing leases are stale
4472    // by definition — there is no other node. Relinquish all existing leases first
4473    // so try_acquire_bundle succeeds regardless of prior owner or expiry.
4474    let existing_leases = lease_repository.list_bundle_leases().await?;
4475    for lease in &existing_leases {
4476        if let Some(owner) = &lease.owner_node_id {
4477            let _ = lease_repository
4478                .relinquish_bundle(lease.bundle_id, owner.clone(), lease.epoch)
4479                .await;
4480        }
4481    }
4482
4483    let mut acquired = Vec::new();
4484    let mut recovered = SweepResult::default();
4485    let owner = node_id.to_string();
4486    for shard_index in 0..shard_count {
4487        let shard_id = ShardId(shard_index);
4488        match lease_repository
4489            .try_acquire_bundle(shard_id, owner.clone(), node_endpoint.as_authority())
4490            .await
4491        {
4492            Ok(LeaseOutcome::Acquired { epoch } | LeaseOutcome::Renewed { epoch }) => {
4493                // Being the only owner removes the need for a lease renewer,
4494                // not for recovery: the previous process's offered workflow
4495                // tasks and timeout tracking ended with it, so the shard is
4496                // swept before it admits anything (runtime-sweeper-recovery
4497                // Requirement 11.6).
4498                match runtime.recover_self_assigned_shard(shard_id, epoch).await {
4499                    Ok(swept) => {
4500                        recovered.workflow_tasks_republished += swept.workflow_tasks_republished;
4501                        recovered.activity_tasks_republished += swept.activity_tasks_republished;
4502                        recovered.due_timers_injected += swept.due_timers_injected;
4503                        recovered.activity_tracking_entries_reconstructed +=
4504                            swept.activity_tracking_entries_reconstructed;
4505                        acquired.push(SelfAssignedShardLease {
4506                            shard_id,
4507                            owner: owner.clone(),
4508                            epoch,
4509                        });
4510                    }
4511                    Err(error) => {
4512                        // The runtime has dropped the shard; release its lease.
4513                        let _ = lease_repository
4514                            .relinquish_bundle(shard_id, owner.clone(), epoch)
4515                            .await;
4516                        if require_all {
4517                            rollback_self_assigned_shards(runtime, lease_repository, &acquired)
4518                                .await;
4519                            return Err(error).context(format!(
4520                                "failed to recover embedded DSQL shard {shard_index}"
4521                            ));
4522                        }
4523                        tracing::warn!(
4524                            shard_index,
4525                            ?error,
4526                            "failed to recover self-assigned shard; leaving it unowned"
4527                        );
4528                    }
4529                }
4530            }
4531            Ok(LeaseOutcome::Rejected {
4532                current_owner,
4533                current_epoch,
4534            }) => {
4535                if require_all {
4536                    rollback_self_assigned_shards(runtime, lease_repository, &acquired).await;
4537                    return Err(anyhow!(
4538                        "embedded DSQL shard {shard_index} is held by owner {current_owner} at epoch {}",
4539                        current_epoch.0
4540                    ));
4541                }
4542                tracing::warn!(
4543                    shard_index,
4544                    %current_owner,
4545                    current_epoch = current_epoch.0,
4546                    "failed to self-assign shard: lease is held by another owner"
4547                );
4548            }
4549            Err(error) => {
4550                if require_all {
4551                    rollback_self_assigned_shards(runtime, lease_repository, &acquired).await;
4552                    return Err(error).context(format!(
4553                        "failed to self-assign embedded DSQL shard {shard_index}"
4554                    ));
4555                }
4556                tracing::warn!(shard_index, ?error, "failed to self-assign shard");
4557            }
4558        }
4559    }
4560    info!(
4561        acquired = acquired.len(),
4562        shard_count,
4563        workflow_tasks_republished = recovered.workflow_tasks_republished,
4564        activity_tasks_republished = recovered.activity_tasks_republished,
4565        due_timers_injected = recovered.due_timers_injected,
4566        activity_tracking_entries_reconstructed = recovered.activity_tracking_entries_reconstructed,
4567        "self-assigned and recovered DSQL shards (no controller)"
4568    );
4569    Ok(acquired)
4570}
4571
4572async fn rollback_self_assigned_shards<R>(
4573    runtime: &TokeiraRuntime<R>,
4574    lease_repository: &R,
4575    leases: &[SelfAssignedShardLease],
4576) where
4577    R: LeaseRepository + RunRepository + 'static,
4578{
4579    for lease in leases.iter().rev() {
4580        runtime.relinquish_shard(lease.shard_id).await;
4581        let _ = lease_repository
4582            .relinquish_bundle(lease.shard_id, lease.owner.clone(), lease.epoch)
4583            .await;
4584    }
4585}
4586
4587fn self_assigned_shard_cleanup<R>(
4588    runtime: Arc<TokeiraRuntime<R>>,
4589    lease_repository: Arc<R>,
4590    leases: Vec<SelfAssignedShardLease>,
4591) -> CleanupAction
4592where
4593    R: LeaseRepository + RunRepository + 'static,
4594{
4595    Box::new(move |deadline| {
4596        Box::pin(async move {
4597            let mut first_error = None;
4598            for lease in leases.into_iter().rev() {
4599                runtime.relinquish_shard(lease.shard_id).await;
4600                let now = Instant::now();
4601                if now >= deadline {
4602                    first_error.get_or_insert_with(|| {
4603                        anyhow!("deadline elapsed while relinquishing embedded DSQL shards")
4604                    });
4605                    continue;
4606                }
4607                let release = tokio::time::timeout(
4608                    deadline.saturating_duration_since(now),
4609                    lease_repository.relinquish_bundle(lease.shard_id, lease.owner, lease.epoch),
4610                )
4611                .await;
4612                match release {
4613                    Ok(Ok(LeaseOutcome::Acquired { .. })) => {}
4614                    Ok(Ok(outcome)) => {
4615                        first_error.get_or_insert_with(|| {
4616                            anyhow!("embedded DSQL shard release was fenced: {outcome:?}")
4617                        });
4618                    }
4619                    Ok(Err(error)) => {
4620                        first_error.get_or_insert(error);
4621                    }
4622                    Err(_) => {
4623                        first_error.get_or_insert_with(|| {
4624                            anyhow!("deadline elapsed while relinquishing embedded DSQL shard")
4625                        });
4626                    }
4627                }
4628            }
4629            match first_error {
4630                Some(error) => Err(error),
4631                None => Ok(()),
4632            }
4633        })
4634    })
4635}
4636
4637fn configured_node_endpoint(config: &TokeiraConfig, listen_addr: SocketAddr) -> NodeEndpoint {
4638    NodeEndpoint {
4639        host: config.infrastructure.placement.node_host.clone(),
4640        port: config
4641            .infrastructure
4642            .placement
4643            .node_port
4644            .unwrap_or_else(|| listen_addr.port()),
4645    }
4646}
4647
4648fn placement_config(config: &TokeiraConfig) -> PlacementConfig {
4649    let placement = &config.infrastructure.placement;
4650    PlacementConfig {
4651        shard_count: placement.shard_count,
4652        bundle_count: placement.bundle_count,
4653        partition_count: placement.partition_count,
4654        hash_version: placement.hash_version,
4655    }
4656}
4657
4658fn membership_config(
4659    config: &TokeiraConfig,
4660    node_id: IncarnationId,
4661    node_endpoint: NodeEndpoint,
4662    controller_endpoint: String,
4663) -> MembershipConfig {
4664    let placement = &config.infrastructure.placement;
4665    MembershipConfig {
4666        controller_endpoint,
4667        heartbeat_interval: std::time::Duration::from_millis(placement.heartbeat_interval_ms),
4668        reconnect_base_delay: std::time::Duration::from_millis(placement.reconnect_base_delay_ms),
4669        reconnect_max_delay: std::time::Duration::from_millis(placement.reconnect_max_delay_ms),
4670        node_id,
4671        node_endpoint,
4672        zone: None,
4673        version: tokeira_build_info::TOKEIRA_VERSION.to_string(),
4674        build_id: tokeira_build_info::TOKEIRA_GIT_SHA.to_string(),
4675    }
4676}
4677
4678/// Budget applier for stacks without a connection reservoir (the in-memory
4679/// store): directives are accepted and discarded, and no headroom is reported.
4680#[derive(Debug)]
4681struct NoopConnectionBudgetApplier;
4682
4683impl ConnectionBudgetApplier for NoopConnectionBudgetApplier {
4684    fn apply_budget(
4685        &self,
4686        _rate_per_second: f64,
4687        _capacity: u64,
4688        _max_reservoir_size: u32,
4689    ) -> Result<()> {
4690        Ok(())
4691    }
4692
4693    fn available_connections(&self) -> u32 {
4694        0
4695    }
4696}
4697
4698/// Applies controller connection budgets to the DSQL connection director the
4699/// whole stack shares, and reports its warm headroom in heartbeats.
4700///
4701/// Only the reservoir cap is applied. The directive's `rate_per_second` and
4702/// `capacity` describe this node's share of the cluster creation rate, but on
4703/// the distributed path the creation-rate limiter is the DynamoDB token bucket
4704/// every node shares, which already enforces the cluster rate as one row: a
4705/// node writing its per-node share into that row would replace the cluster
4706/// rate with a fraction of it. A process-local governor in front of the shared
4707/// bucket is the mechanism the share would drive, and does not exist yet.
4708#[derive(Debug)]
4709struct DsqlConnectionBudgetApplier {
4710    director: Arc<DsqlConnectionDirector>,
4711}
4712
4713impl ConnectionBudgetApplier for DsqlConnectionBudgetApplier {
4714    fn apply_budget(
4715        &self,
4716        rate_per_second: f64,
4717        capacity: u64,
4718        max_reservoir_size: u32,
4719    ) -> Result<()> {
4720        let retired = self.director.apply_connection_budget(max_reservoir_size)?;
4721        tracing::debug!(
4722            rate_per_second,
4723            capacity,
4724            max_reservoir_size,
4725            reservoir_target = self.director.reservoir_target(),
4726            retired,
4727            "applied controller connection budget to the DSQL reservoir"
4728        );
4729        Ok(())
4730    }
4731
4732    fn available_connections(&self) -> u32 {
4733        u32::try_from(self.director.ready_connections()).unwrap_or(u32::MAX)
4734    }
4735}
4736
4737/// Resolves executions and builds describe responses straight from the run
4738/// repository. The namespace is taken from each request (via `namespace_id_for`),
4739/// not bound at construction — a single server serves every namespace, so freezing
4740/// it to one (e.g. `default`) would make query/describe miss every other namespace's
4741/// runs while history (which derives the namespace per request) still found them.
4742struct StoreExecutionResolver<R> {
4743    repo: Arc<R>,
4744}
4745
4746impl<R> StoreExecutionResolver<R> {
4747    fn new(repo: Arc<R>) -> Self {
4748        Self { repo }
4749    }
4750}
4751
4752fn apply_reported_problem_search_attribute(
4753    search_attributes: &mut tokeira_types::SearchAttributes,
4754    problem: Option<WorkflowTaskReportedProblem>,
4755) {
4756    let Some(problem) = problem else {
4757        return;
4758    };
4759    // v1.31.0 exposes exactly these two KeywordList elements, derived from the
4760    // last NON-TRANSIENT WFT problem once AttemptsSinceLastSuccess reaches the
4761    // configured threshold — a `Failed`-flavored pair or a `TimedOut` pair,
4762    // per the persisted `LastWorkflowTaskFailure` oneof
4763    // (`mutable_state_impl.go:6478-6491 @ v1.31.0`; the timed-out cause
4764    // renders `TimeoutType.String()`, which is "StartToClose" for the only
4765    // type v1.31.0 ever stores).
4766    let entries = match &problem.problem {
4767        tokeira_kernel::WorkflowTaskProblem::Failed(cause) => vec![
4768            "category=WorkflowTaskFailed".to_string(),
4769            format!("cause=WorkflowTaskFailedCause{}", cause.as_str()),
4770        ],
4771        tokeira_kernel::WorkflowTaskProblem::TimedOutStartToClose => vec![
4772            "category=WorkflowTaskTimedOut".to_string(),
4773            "cause=WorkflowTaskTimedOutCauseStartToClose".to_string(),
4774        ],
4775    };
4776    search_attributes.0.insert(
4777        "TemporalReportedProblems".to_string(),
4778        SearchAttrValue::KeywordList(entries),
4779    );
4780}
4781
4782#[async_trait]
4783impl<R> ExecutionResolver for StoreExecutionResolver<R>
4784where
4785    R: RunRepository + 'static,
4786{
4787    async fn current_run_key(
4788        &self,
4789        namespace: &str,
4790        workflow_id: &str,
4791    ) -> Result<Option<tokeira_types::RunKey>> {
4792        let namespace_id = namespace_id_for(namespace);
4793        let result = self
4794            .repo
4795            .resolve_execution(&ExecutionRef {
4796                namespace_id,
4797                workflow_id: WorkflowId(workflow_id.to_string()),
4798                run_id: None,
4799            })
4800            .await?;
4801        if result.is_some() {
4802            return Ok(result);
4803        }
4804        self.repo
4805            .find_latest_run(namespace_id, &WorkflowId(workflow_id.to_string()))
4806            .await
4807    }
4808
4809    async fn describe_execution(
4810        &self,
4811        namespace: &str,
4812        workflow_id: &str,
4813        run_id: Option<tokeira_types::RunId>,
4814    ) -> Result<Option<WorkflowExecutionDescription>> {
4815        let namespace_id = namespace_id_for(namespace);
4816        let run_key = match self
4817            .repo
4818            .resolve_execution(&ExecutionRef {
4819                namespace_id,
4820                workflow_id: WorkflowId(workflow_id.to_string()),
4821                run_id,
4822            })
4823            .await?
4824        {
4825            Some(rk) => rk,
4826            None if run_id.is_none() => {
4827                match self
4828                    .repo
4829                    .find_latest_run(namespace_id, &WorkflowId(workflow_id.to_string()))
4830                    .await?
4831                {
4832                    Some(rk) => rk,
4833                    None => return Ok(None),
4834                }
4835            }
4836            None => return Ok(None),
4837        };
4838
4839        let (loaded, stats) = self.repo.load_run_with_stats(run_key).await?;
4840        match loaded {
4841            LoadedRun::Existing(state) => {
4842                // Describe surfaces history size and external-payload statistics
4843                // (`describeworkflow/api.go:126,166 @ v1.31.0`). Both are
4844                // state-maintained: the History Size is the statistic storage
4845                // advances with every committed batch, the same number the
4846                // continue-as-new advice and visibility read, and the payload
4847                // statistics accumulate on the kernel state. No history read
4848                // happens here (continue-as-new-advice, Requirements 7.1, 7.2).
4849                let history_size_bytes = stats.history_size_bytes;
4850                let external_payload_count = state.external_payload_count;
4851                let external_payload_size_bytes = state.external_payload_size_bytes;
4852                // Derived straight from committed kernel state: the counter
4853                // and problem identity live on the run, the threshold is read
4854                // live at derive time (Tier 3.22 `DynamicConfigChanges`).
4855                let reported_problem = reported_problem_from_state(&state);
4856                let versioning_info = state.versioning_info.clone();
4857                let most_recent_worker_version_stamp = versioning_info
4858                    .as_ref()
4859                    .and_then(|info| info.most_recent_worker_version_stamp.clone());
4860                let pending_nexus_operations =
4861                    tokeira_edge::translate::describe_pending_nexus_operations(&state);
4862                let mut search_attributes = state.search_attributes;
4863                apply_reported_problem_search_attribute(&mut search_attributes, reported_problem);
4864                if let Some(build_ids) = versioning_info
4865                    .as_ref()
4866                    .map(|info| &info.build_id_search_attributes)
4867                    .filter(|build_ids| !build_ids.is_empty())
4868                {
4869                    search_attributes.0.insert(
4870                        "BuildIds".to_string(),
4871                        tokeira_types::SearchAttrValue::KeywordList(build_ids.clone()),
4872                    );
4873                }
4874                Ok(Some(WorkflowExecutionDescription {
4875                    namespace: namespace.to_string(),
4876                    workflow_id: state.workflow_id.0,
4877                    run_key: state.run_key,
4878                    run_id: state.run_id,
4879                    workflow_type: state.workflow_type.0,
4880                    task_queue: state.task_queue.0.clone(),
4881                    status: state.status,
4882                    start_time: Some(state.started_at),
4883                    close_time: state.closed_at,
4884                    // v1.31.0 ExecutionTime = this run's StartTime + FirstWorkflowTaskBackoff
4885                    // (mutable_state_impl.go:2859) — NOT the chain's first-run start. tokeira
4886                    // carries that backoff (client start delay / workflow-retry backoff) as
4887                    // `workflow_start_delay`; TestWorkflowRetry asserts start + backoff per attempt.
4888                    execution_time: state.started_at
4889                        + state.workflow_start_delay.unwrap_or_default(),
4890                    execution_config: tokeira_edge::translate::ExecutionConfigDescription {
4891                        task_queue: state.task_queue.0.clone(),
4892                        workflow_execution_timeout: state.workflow_execution_timeout,
4893                        workflow_run_timeout: state.workflow_run_timeout,
4894                        default_workflow_task_timeout: state.workflow_task_timeout,
4895                        // Describe returns start-event metadata verbatim, even after later
4896                        // transitions (`describeworkflow/api.go:98-110 @ v1.31.0`). The kernel
4897                        // summary is replayed from that event, so no history scan is needed here.
4898                        user_metadata: state.user_metadata.map(|metadata| {
4899                            tokeira_edge::translate::UserMetadata {
4900                                summary: metadata.summary,
4901                                details: metadata.details,
4902                            }
4903                        }),
4904                    },
4905                    history_length: state.last_event_id,
4906                    history_size_bytes,
4907                    state_transition_count: state.transition_seq.0 as i64,
4908                    parent_namespace_id: state
4909                        .parent_namespace_id
4910                        .map(|namespace_id| namespace_id.0.to_string()),
4911                    parent_workflow_id: state.parent_workflow_id.clone(),
4912                    parent_run_id: state.parent_run_id,
4913                    root_workflow_id: state.root_workflow_id.clone(),
4914                    root_run_id: state.root_run_id,
4915                    first_run_id: state.first_execution_run_id,
4916                    memo: state.memo,
4917                    search_attributes,
4918                    pending_activities: state
4919                        .activities
4920                        .values()
4921                        .map(
4922                            |activity| tokeira_edge::translate::PendingActivityDescription {
4923                                activity_id: activity.activity_id.clone(),
4924                                activity_type: activity.activity_type.clone(),
4925                                is_started: activity.started_at.is_some(),
4926                                cancel_requested: activity.cancel_requested,
4927                                attempt: activity.attempt,
4928                                maximum_attempts: activity
4929                                    .retry_policy
4930                                    .as_ref()
4931                                    .map(|policy| policy.maximum_attempts)
4932                                    .unwrap_or_default(),
4933                                scheduled_at: activity.scheduled_at,
4934                                started_at: activity.started_at,
4935                                last_failure: activity.last_failure.clone(),
4936                                heartbeat_details: activity.heartbeat_details.clone(),
4937                                last_heartbeat_at: activity.last_heartbeat_at,
4938                                last_worker_identity: {
4939                                    // `LastWorkerIdentity = StartedIdentity`, falling
4940                                    // back to `RetryLastWorkerIdentity` when empty and
4941                                    // a retry policy exists (`GetPendingActivityInfo`,
4942                                    // workflow/activity.go:159-166 @ v1.31.0).
4943                                    let started = activity
4944                                        .started_identity
4945                                        .clone()
4946                                        .map(|identity| identity.0)
4947                                        .unwrap_or_default();
4948                                    if started.is_empty() && activity.retry_policy.is_some() {
4949                                        activity
4950                                            .retry_last_worker_identity
4951                                            .clone()
4952                                            .map(|identity| identity.0)
4953                                            .unwrap_or_default()
4954                                    } else {
4955                                        started
4956                                    }
4957                                },
4958                                paused: activity.pause_info.is_some(),
4959                                pause_info: activity.pause_info.as_ref().map(|info| {
4960                                    tokeira_edge::translate::PauseInfoDescription {
4961                                        identity: info.identity.clone(),
4962                                        paused_time: info.pause_time,
4963                                        reason: info.reason.clone(),
4964                                        rule_id: info.rule_id.clone(),
4965                                    }
4966                                }),
4967                                activity_options: tokeira_edge::translate::ActivityOptions {
4968                                    task_queue: Some(activity.task_queue.0.clone()),
4969                                    schedule_to_close_timeout: activity.schedule_to_close_timeout,
4970                                    schedule_to_start_timeout: activity.schedule_to_start_timeout,
4971                                    start_to_close_timeout: activity.start_to_close_timeout,
4972                                    heartbeat_timeout: activity.heartbeat_timeout,
4973                                    retry_policy: activity.retry_policy.clone(),
4974                                    priority: activity.priority.as_ref().map(|priority| {
4975                                        tokeira_edge::translate::Priority {
4976                                            priority_key: priority.priority_key,
4977                                            fairness_key: priority.fairness_key.clone(),
4978                                            fairness_weight: priority.fairness_weight,
4979                                        }
4980                                    }),
4981                                },
4982                            },
4983                        )
4984                        .collect(),
4985                    pending_children: state
4986                        .children
4987                        .values()
4988                        .map(|child| tokeira_edge::translate::PendingChildDescription {
4989                            workflow_id: child.child_workflow_id.0.clone(),
4990                            run_id: child
4991                                .child_run_id
4992                                .as_ref()
4993                                .map(|run_id| run_id.0.to_string()),
4994                            workflow_type: String::new(),
4995                            initiated_event_id: child.initiated_event_id,
4996                            parent_close_policy: child.parent_close_policy,
4997                        })
4998                        .collect(),
4999                    pending_workflow_task: state.pending_workflow_task.as_ref().map(|task| {
5000                        tokeira_edge::translate::PendingWorkflowTaskDescription {
5001                            is_started: task.started_event_id.is_some(),
5002                            scheduled_at: task.scheduled_at,
5003                            started_at: task.started_at,
5004                            attempt: task.attempt,
5005                        }
5006                    }),
5007                    callbacks: state.completion_callbacks.clone(),
5008                    pending_nexus_operations,
5009                    pause_info: state.pause_info.as_ref().map(|info| {
5010                        tokeira_edge::translate::PauseInfoDescription {
5011                            identity: info.identity.clone(),
5012                            paused_time: info.pause_time,
5013                            reason: info.reason.clone(),
5014                            rule_id: None,
5015                        }
5016                    }),
5017                    execution_expiration_time: state.workflow_execution_timeout.map(|timeout| {
5018                        state.first_run_started_at.unwrap_or(state.started_at) + timeout
5019                    }),
5020                    run_expiration_time: state
5021                        .workflow_run_timeout
5022                        .map(|timeout| state.started_at + timeout),
5023                    cancel_requested: state.cancel_requested,
5024                    original_start_time: state.first_run_started_at.unwrap_or(state.started_at),
5025                    versioning_info: versioning_info
5026                        .filter(|info| info.has_execution_versioning_info()),
5027                    worker_deployment_name: state.worker_deployment_name.clone(),
5028                    priority: state.priority.as_ref().map(|priority| {
5029                        tokeira_edge::translate::Priority {
5030                            priority_key: priority.priority_key,
5031                            fairness_key: priority.fairness_key.clone(),
5032                            fairness_weight: priority.fairness_weight,
5033                        }
5034                    }),
5035                    auto_reset_points: state.auto_reset_points.clone(),
5036                    most_recent_worker_version_stamp,
5037                    request_id_infos: state.request_id_infos.clone(),
5038                    external_payload_count,
5039                    external_payload_size_bytes,
5040                }))
5041            }
5042            LoadedRun::Absent => Err(anyhow!("resolved run missing from storage: {:?}", run_key)),
5043        }
5044    }
5045}
5046
5047#[doc(hidden)]
5048pub fn __cli_parse() -> Cli {
5049    Cli::parse()
5050}
5051
5052#[cfg(test)]
5053// Edition-2024 `std::env` mutation in tests — each site carries its SAFETY comment.
5054#[allow(unsafe_code)]
5055mod tests {
5056    use super::*;
5057    use metrics::with_local_recorder;
5058    use metrics_util::debugging::DebuggingRecorder;
5059    use opentelemetry::trace::TracerProvider as _;
5060    use opentelemetry_sdk::{
5061        error::{OTelSdkError, OTelSdkResult},
5062        trace::{SdkTracerProvider, SpanData, SpanExporter},
5063    };
5064    use proptest::prelude::*;
5065    use tracing_subscriber::layer::SubscriberExt as _;
5066    // Endpoint target/config types are referenced only by the Nexus endpoint store
5067    // tests below; importing them here (not at crate scope) keeps the lib build clean.
5068    use tokeira_runtime::{EndpointTarget, NexusEndpointConfig};
5069
5070    #[derive(Clone, Copy, Debug, PartialEq, Eq)]
5071    enum StartupModelPhase {
5072        Cluster,
5073        Pool,
5074        Schema,
5075        Ownership,
5076        Runtime,
5077    }
5078
5079    #[derive(Clone, Copy, Debug, PartialEq, Eq)]
5080    enum StartupModelEvent {
5081        Attempt(StartupModelPhase),
5082        Complete(StartupModelPhase),
5083        ReleaseOwnership,
5084        ClosePool,
5085        ReturnHandle,
5086    }
5087
5088    const STARTUP_MODEL_PHASES: [StartupModelPhase; 5] = [
5089        StartupModelPhase::Cluster,
5090        StartupModelPhase::Pool,
5091        StartupModelPhase::Schema,
5092        StartupModelPhase::Ownership,
5093        StartupModelPhase::Runtime,
5094    ];
5095
5096    fn startup_model(failure_boundary: Option<usize>) -> Vec<StartupModelEvent> {
5097        let mut events = Vec::new();
5098        let mut pool_open = false;
5099        let mut ownership_held = false;
5100        for (index, phase) in STARTUP_MODEL_PHASES.into_iter().enumerate() {
5101            events.push(StartupModelEvent::Attempt(phase));
5102            if failure_boundary == Some(index) {
5103                if ownership_held {
5104                    events.push(StartupModelEvent::ReleaseOwnership);
5105                }
5106                if pool_open {
5107                    events.push(StartupModelEvent::ClosePool);
5108                }
5109                return events;
5110            }
5111            events.push(StartupModelEvent::Complete(phase));
5112            pool_open |= phase == StartupModelPhase::Pool;
5113            ownership_held |= phase == StartupModelPhase::Ownership;
5114        }
5115        events.push(StartupModelEvent::ReturnHandle);
5116        events
5117    }
5118
5119    #[derive(Clone, Copy, Debug, PartialEq, Eq)]
5120    enum ShutdownModelEvent {
5121        CloseAdmission,
5122        CancelTasks,
5123        CallsComplete,
5124        DrainComplete,
5125        TasksComplete,
5126        JoinComplete,
5127        FinishTelemetry,
5128        AttemptShardRelease,
5129        CloseOwnershipAdmission,
5130        AttemptOwnershipRelease,
5131        AttemptPoolClose,
5132        ReturnToHost,
5133    }
5134
5135    fn shutdown_model(calls_pending: bool, tasks_pending: bool) -> Vec<ShutdownModelEvent> {
5136        let mut events = vec![
5137            ShutdownModelEvent::CloseAdmission,
5138            ShutdownModelEvent::CancelTasks,
5139        ];
5140        if calls_pending {
5141            events.push(ShutdownModelEvent::CallsComplete);
5142        }
5143        events.push(ShutdownModelEvent::DrainComplete);
5144        if tasks_pending {
5145            events.push(ShutdownModelEvent::TasksComplete);
5146        }
5147        events.extend([
5148            ShutdownModelEvent::JoinComplete,
5149            ShutdownModelEvent::FinishTelemetry,
5150            ShutdownModelEvent::AttemptShardRelease,
5151            ShutdownModelEvent::CloseOwnershipAdmission,
5152            ShutdownModelEvent::AttemptOwnershipRelease,
5153            ShutdownModelEvent::AttemptPoolClose,
5154            ShutdownModelEvent::ReturnToHost,
5155        ]);
5156        events
5157    }
5158
5159    #[derive(Clone, Copy, Debug)]
5160    enum HostInstrumentationModel {
5161        None,
5162        LocalRecorder,
5163        LocalSubscriber,
5164        RecorderAndSubscriber,
5165    }
5166
5167    fn host_instrumentation_strategy() -> impl Strategy<Value = HostInstrumentationModel> {
5168        prop_oneof![
5169            Just(HostInstrumentationModel::None),
5170            Just(HostInstrumentationModel::LocalRecorder),
5171            Just(HostInstrumentationModel::LocalSubscriber),
5172            Just(HostInstrumentationModel::RecorderAndSubscriber),
5173        ]
5174    }
5175
5176    #[derive(Clone, Copy, Debug)]
5177    enum EmbeddedStorageModeModel {
5178        InMemory,
5179        ManagedDsql,
5180        ExistingDsql,
5181    }
5182
5183    fn storage_mode_strategy() -> impl Strategy<Value = EmbeddedStorageModeModel> {
5184        prop_oneof![
5185            Just(EmbeddedStorageModeModel::InMemory),
5186            Just(EmbeddedStorageModeModel::ManagedDsql),
5187            Just(EmbeddedStorageModeModel::ExistingDsql),
5188        ]
5189    }
5190
5191    #[derive(Clone, Copy, Debug, PartialEq, Eq)]
5192    struct EmbeddedConstructionEffects {
5193        listener_attempts: usize,
5194        global_install_attempts: usize,
5195        signal_handler_attempts: usize,
5196        local_emission_attempts: usize,
5197    }
5198
5199    fn embedded_construction_effects(
5200        _host: HostInstrumentationModel,
5201        _storage: EmbeddedStorageModeModel,
5202    ) -> EmbeddedConstructionEffects {
5203        // Every storage choice crosses the same library boundary. Durable
5204        // modes add AWS/SQL I/O but never acquire host process facilities.
5205        EmbeddedConstructionEffects {
5206            listener_attempts: 0,
5207            global_install_attempts: 0,
5208            signal_handler_attempts: 0,
5209            local_emission_attempts: 1,
5210        }
5211    }
5212
5213    #[derive(Clone, Copy, Debug)]
5214    enum TelemetryFixture {
5215        NoSubscriber,
5216        LocalRecorder,
5217        FailingExporter,
5218    }
5219
5220    fn telemetry_fixture_strategy() -> impl Strategy<Value = TelemetryFixture> {
5221        prop_oneof![
5222            Just(TelemetryFixture::NoSubscriber),
5223            Just(TelemetryFixture::LocalRecorder),
5224            Just(TelemetryFixture::FailingExporter),
5225        ]
5226    }
5227
5228    #[derive(Clone, Debug)]
5229    struct FailingSpanExporter;
5230
5231    impl SpanExporter for FailingSpanExporter {
5232        async fn export(&self, _batch: Vec<SpanData>) -> OTelSdkResult {
5233            Err(OTelSdkError::InternalFailure(
5234                "fixture exporter rejected the batch".to_owned(),
5235            ))
5236        }
5237    }
5238
5239    fn apply_observed_sequence(ops: &[i16]) -> Vec<u8> {
5240        let mut state = 0_i64;
5241        let mut committed = Vec::with_capacity(ops.len() * std::mem::size_of::<i64>());
5242        let span = tracing::info_span!(
5243            "embedded.observational_fixture",
5244            tokeira.operation = "transition_sequence"
5245        );
5246        let _entered = span.enter();
5247        for op in ops {
5248            record_embedded_lifecycle(
5249                EmbeddedStorageModeLabel::InMemory,
5250                ClusterStatusLabel::NotApplicable,
5251                SchemaOutcomeLabel::NotApplicable,
5252                OwnershipOutcomeLabel::NotApplicable,
5253                DbClassLabel::Commit,
5254                EmbeddedOperationLabel::Startup,
5255                ErrorClassLabel::None,
5256            );
5257            state = state.wrapping_add(i64::from(*op));
5258            committed.extend_from_slice(&state.to_le_bytes());
5259        }
5260        committed
5261    }
5262
5263    fn observed_sequence_bytes(ops: &[i16], fixture: TelemetryFixture) -> Vec<u8> {
5264        match fixture {
5265            TelemetryFixture::NoSubscriber => apply_observed_sequence(ops),
5266            TelemetryFixture::LocalRecorder => {
5267                let recorder = DebuggingRecorder::new();
5268                with_local_recorder(&recorder, || apply_observed_sequence(ops))
5269            }
5270            TelemetryFixture::FailingExporter => {
5271                let provider = SdkTracerProvider::builder()
5272                    .with_simple_exporter(FailingSpanExporter)
5273                    .build();
5274                let tracer = provider.tracer("observational-property");
5275                let subscriber = tracing_subscriber::registry()
5276                    .with(tracing_opentelemetry::layer().with_tracer(tracer));
5277                let dispatch = tracing::Dispatch::new(subscriber);
5278                let bytes =
5279                    tracing::dispatcher::with_default(&dispatch, || apply_observed_sequence(ops));
5280                let _ = provider.force_flush();
5281                bytes
5282            }
5283        }
5284    }
5285
5286    proptest! {
5287        #![proptest_config(ProptestConfig::with_cases(128))]
5288
5289        // Feature: managed-embedded-dsql, Property 12: startup is prefix-safe and failure-atomic
5290        #[test]
5291        fn startup_is_prefix_safe_and_failure_atomic(
5292            failure_boundary in prop::option::of(0usize..STARTUP_MODEL_PHASES.len()),
5293        ) {
5294            let events = startup_model(failure_boundary);
5295            let attempted = events.iter().filter_map(|event| match event {
5296                StartupModelEvent::Attempt(phase) => Some(*phase),
5297                _ => None,
5298            }).collect::<Vec<_>>();
5299            let completed = events.iter().filter_map(|event| match event {
5300                StartupModelEvent::Complete(phase) => Some(*phase),
5301                _ => None,
5302            }).collect::<Vec<_>>();
5303
5304            prop_assert_eq!(attempted.as_slice(), &STARTUP_MODEL_PHASES[..attempted.len()]);
5305            prop_assert_eq!(completed.as_slice(), &STARTUP_MODEL_PHASES[..completed.len()]);
5306            if let Some(boundary) = failure_boundary {
5307                prop_assert!(!events.contains(&StartupModelEvent::ReturnHandle));
5308                let rollback = events.iter().filter(|event| matches!(
5309                    event,
5310                    StartupModelEvent::ReleaseOwnership | StartupModelEvent::ClosePool
5311                )).copied().collect::<Vec<_>>();
5312                let expected = match boundary {
5313                    0 | 1 => Vec::new(),
5314                    2 | 3 => vec![StartupModelEvent::ClosePool],
5315                    _ => vec![
5316                        StartupModelEvent::ReleaseOwnership,
5317                        StartupModelEvent::ClosePool,
5318                    ],
5319                };
5320                prop_assert_eq!(rollback, expected);
5321            } else {
5322                prop_assert_eq!(completed.len(), STARTUP_MODEL_PHASES.len());
5323                prop_assert_eq!(events.last(), Some(&StartupModelEvent::ReturnHandle));
5324            }
5325        }
5326
5327        // Feature: managed-embedded-dsql, Property 19: shutdown establishes the host flush boundary
5328        #[test]
5329        fn shutdown_establishes_the_host_flush_boundary(
5330            calls_pending in any::<bool>(),
5331            tasks_pending in any::<bool>(),
5332            independent_failure_mask in 0u8..8,
5333        ) {
5334            let events = shutdown_model(calls_pending, tasks_pending);
5335            prop_assert_eq!(events.first(), Some(&ShutdownModelEvent::CloseAdmission));
5336            prop_assert_eq!(events.get(1), Some(&ShutdownModelEvent::CancelTasks));
5337            prop_assert_eq!(events.last(), Some(&ShutdownModelEvent::ReturnToHost));
5338            for cleanup in [
5339                ShutdownModelEvent::AttemptShardRelease,
5340                ShutdownModelEvent::AttemptOwnershipRelease,
5341                ShutdownModelEvent::AttemptPoolClose,
5342            ] {
5343                prop_assert!(events.contains(&cleanup));
5344            }
5345            prop_assert!(
5346                events.iter().position(|event| *event == ShutdownModelEvent::AttemptShardRelease)
5347                    < events.iter().position(|event| *event == ShutdownModelEvent::CloseOwnershipAdmission)
5348            );
5349            prop_assert!(
5350                events.iter().position(|event| *event == ShutdownModelEvent::CloseOwnershipAdmission)
5351                    < events.iter().position(|event| *event == ShutdownModelEvent::AttemptOwnershipRelease)
5352            );
5353            let failures = (0..3)
5354                .filter(|bit| independent_failure_mask & (1 << bit) != 0)
5355                .count();
5356            prop_assert_eq!(failures, independent_failure_mask.count_ones() as usize);
5357            prop_assert!(
5358                events.iter().position(|event| *event == ShutdownModelEvent::FinishTelemetry)
5359                    < events.iter().position(|event| *event == ShutdownModelEvent::ReturnToHost)
5360            );
5361        }
5362
5363        // Feature: managed-embedded-dsql, Property 14: embedded construction is transport- and global-state-neutral
5364        #[test]
5365        fn embedded_construction_is_transport_and_global_state_neutral(
5366            host in host_instrumentation_strategy(),
5367            storage in storage_mode_strategy(),
5368        ) {
5369            let effects = embedded_construction_effects(host, storage);
5370            prop_assert_eq!(effects.listener_attempts, 0);
5371            prop_assert_eq!(effects.global_install_attempts, 0);
5372            prop_assert_eq!(effects.signal_handler_attempts, 0);
5373            prop_assert_eq!(effects.local_emission_attempts, 1);
5374        }
5375
5376        // Feature: embedded-engine-listener, Property 6: startup stays zero-listener
5377        #[test]
5378        fn startup_stays_zero_listener(
5379            host in host_instrumentation_strategy(),
5380            storage in storage_mode_strategy(),
5381            explicit_listens in 0usize..4,
5382        ) {
5383            // Every start path contributes no bind; only explicit `listen`
5384            // calls do, one socket each.
5385            let startup = embedded_construction_effects(host, storage);
5386            prop_assert_eq!(startup.listener_attempts, 0);
5387            prop_assert_eq!(
5388                startup.listener_attempts + explicit_listens,
5389                explicit_listens
5390            );
5391        }
5392
5393        // Feature: managed-embedded-dsql, Property 20: telemetry is observational only
5394        #[test]
5395        fn telemetry_is_observational_only(
5396            operations in prop::collection::vec(any::<i16>(), 0..64),
5397            first in telemetry_fixture_strategy(),
5398            second in telemetry_fixture_strategy(),
5399        ) {
5400            let expected = observed_sequence_bytes(&operations, TelemetryFixture::NoSubscriber);
5401            prop_assert_eq!(observed_sequence_bytes(&operations, first), expected.clone());
5402            prop_assert_eq!(observed_sequence_bytes(&operations, second), expected);
5403        }
5404    }
5405
5406    #[tokio::test]
5407    async fn listen_after_shutdown_began_is_refused_without_binding() {
5408        let engine = Engine::start().await.expect("engine starts");
5409        // The only way to observe a shutting-down engine from `listen` is the
5410        // engine's own cancellation token: `shutdown` consumes the engine, so a
5411        // host cannot race it, but internal cancellation must still be refused.
5412        engine.background_cancel.cancel();
5413        let error = engine
5414            .listen("127.0.0.1:0".parse().expect("static loopback address"))
5415            .await
5416            .expect_err("a cancelled engine must refuse to bind");
5417        assert!(matches!(error, EngineListenError::ShutDown));
5418        assert_eq!(engine.listeners.attached(), 0);
5419    }
5420
5421    #[test]
5422    fn every_startup_failure_boundary_rolls_back_without_returning_a_handle() {
5423        for boundary in 0..STARTUP_MODEL_PHASES.len() {
5424            let events = startup_model(Some(boundary));
5425            assert!(!events.contains(&StartupModelEvent::ReturnHandle));
5426        }
5427    }
5428
5429    #[test]
5430    fn durable_startup_report_exposes_only_approved_evidence() {
5431        let report = EngineStartupReport {
5432            storage_mode: EmbeddedStorageMode::ManagedDsql,
5433            cluster: Some(ClusterStartupReport {
5434                region: "eu-west-1".to_owned(),
5435                cluster_id: "cluster-1".to_owned(),
5436                cluster_arn: "arn:aws:dsql:eu-west-1:123456789012:cluster/cluster-1".to_owned(),
5437                endpoint: "cluster-1.dsql.eu-west-1.on.aws".to_owned(),
5438                action: ClusterAction::Recovered,
5439            }),
5440            schema: Some(SchemaStartupReport {
5441                observed_version: 1,
5442                minimum_supported_version: 1,
5443                target_version: 1,
5444                maximum_readable_version: 1,
5445                migration_set_digest: "release-digest".to_owned(),
5446                outcome: SchemaStartupOutcome::Compatible,
5447            }),
5448            ownership: Some(OwnershipStartupReport {
5449                outcome: ControlLeaseAcquireOutcome::Clean,
5450                fence_token: 7,
5451            }),
5452        };
5453        let diagnostic = format!("{report:?}");
5454        assert!(!diagnostic.contains("descriptor-path-secret"));
5455        assert!(!diagnostic.contains("create-token-secret"));
5456        assert!(!diagnostic.contains("credential-secret"));
5457    }
5458
5459    #[tokio::test]
5460    async fn explicit_in_memory_start_preserves_legacy_mode_and_shuts_down_cleanly() {
5461        let engine = Engine::start_with_embedded_config(EmbeddedEngineConfig::default())
5462            .await
5463            .expect("explicit in-memory startup succeeds");
5464        assert_eq!(engine.startup_report(), &EngineStartupReport::in_memory());
5465        engine.shutdown().await.expect("tracked shutdown succeeds");
5466    }
5467
5468    #[test]
5469    fn embedded_lifecycle_composes_with_a_host_local_recorder() {
5470        let recorder = DebuggingRecorder::new();
5471        let runtime = tokio::runtime::Builder::new_current_thread()
5472            .enable_all()
5473            .build()
5474            .expect("embedded fixture runtime");
5475
5476        with_local_recorder(&recorder, || {
5477            runtime.block_on(async {
5478                // Starting twice in one process also proves the library did not
5479                // consume the one-shot process observability reservation.
5480                for _ in 0..2 {
5481                    let engine =
5482                        Engine::start_with_embedded_config(EmbeddedEngineConfig::default())
5483                            .await
5484                            .expect("embedded startup");
5485                    engine.shutdown().await.expect("embedded shutdown");
5486                }
5487            });
5488        });
5489
5490        let lifecycle_observations = recorder
5491            .snapshotter()
5492            .snapshot()
5493            .into_vec()
5494            .into_iter()
5495            .filter(|(key, _, _, _)| {
5496                key.key().name() == tokeira_observability::EMBEDDED_LIFECYCLE_OPERATIONS_TOTAL
5497            })
5498            .count();
5499        assert_eq!(lifecycle_observations, 2);
5500    }
5501
5502    #[tokio::test]
5503    async fn invalid_explicit_config_fails_before_startup() {
5504        let config = EmbeddedEngineConfig {
5505            startup_timeout_ms: 0,
5506            ..EmbeddedEngineConfig::default()
5507        };
5508        assert!(matches!(
5509            Engine::start_with_embedded_config(config).await,
5510            Err(EmbeddedEngineStartError::InvalidConfiguration(_))
5511        ));
5512    }
5513
5514    #[tokio::test]
5515    async fn unequal_home_counts_fail_before_descriptor_or_listener_work() {
5516        let descriptor_path = std::env::temp_dir().join(format!(
5517            "tokeira-home-count-preflight-{}.json",
5518            IncarnationId::new()
5519        ));
5520        let mut config = EmbeddedEngineConfig::default();
5521        config.server.infrastructure.placement.shard_count = 8;
5522        config.server.infrastructure.placement.bundle_count = 4;
5523        config.storage =
5524            EmbeddedStorageConfig::ManagedDsql(tokeira_config::ManagedEmbeddedDsqlConfig {
5525                intent: tokeira_config::ManagedClusterIntent::CreateOrRecover,
5526                descriptor_path: descriptor_path.clone(),
5527                region: "test-region".to_owned(),
5528                migration_policy: None,
5529                limits: tokeira_config::EmbeddedDsqlLimits::default(),
5530                tags: std::collections::BTreeMap::new(),
5531            });
5532        let error = Engine::start_with_embedded_config(config.clone())
5533            .await
5534            .expect_err("unequal counts must fail before any AWS work");
5535        assert!(matches!(
5536            error,
5537            EmbeddedEngineStartError::InvalidConfiguration(_)
5538        ));
5539        assert!(
5540            error
5541                .to_string()
5542                .contains("shard_count (8) must equal bundle_count (4)")
5543        );
5544        assert!(!descriptor_path.exists());
5545
5546        let address = "127.0.0.1:0".parse().expect("test address");
5547        let error = TokeiradHandle::start_in_memory_with_config(address, config.server)
5548            .await
5549            .expect_err("unequal counts must fail before binding a listener");
5550        assert!(format!("{error:#}").contains("shard_count (8) must equal bundle_count (4)"));
5551    }
5552
5553    #[tokio::test]
5554    async fn durable_snapshot_conflict_fails_before_descriptor_or_aws_work() {
5555        let descriptor_path = std::env::temp_dir().join(format!(
5556            "tokeira-managed-dsql-preflight-{}.json",
5557            IncarnationId::new()
5558        ));
5559        let mut config = EmbeddedEngineConfig::default();
5560        config.server.policy.snapshot = Some(SnapshotPolicyConfig {
5561            location: descriptor_path.with_extension("snapshot"),
5562            interval_ms: 1_000,
5563        });
5564        config.storage =
5565            EmbeddedStorageConfig::ManagedDsql(tokeira_config::ManagedEmbeddedDsqlConfig {
5566                intent: tokeira_config::ManagedClusterIntent::CreateOrRecover,
5567                descriptor_path: descriptor_path.clone(),
5568                region: "eu-west-2".to_owned(),
5569                migration_policy: None,
5570                limits: tokeira_config::EmbeddedDsqlLimits::default(),
5571                tags: std::collections::BTreeMap::new(),
5572            });
5573
5574        assert!(matches!(
5575            Engine::start_with_embedded_config(config).await,
5576            Err(EmbeddedEngineStartError::Phase {
5577                phase: EmbeddedStartupPhase::Configuration
5578            })
5579        ));
5580        assert!(!descriptor_path.exists());
5581    }
5582
5583    #[tokio::test]
5584    async fn drop_synchronously_closes_endpoint_admission() {
5585        let engine = Engine::start().await.expect("engine starts");
5586        let endpoint = engine.endpoint();
5587        drop(engine);
5588
5589        let error = endpoint
5590            .call(InProcessGrpcRequest {
5591                service: "temporal.api.workflowservice.v1.WorkflowService".to_owned(),
5592                rpc: "GetSystemInfo".to_owned(),
5593                headers: http::HeaderMap::new(),
5594                proto: Bytes::new(),
5595            })
5596            .await
5597            .expect_err("dropped engine rejects endpoint clones");
5598        assert_eq!(error.code(), tonic::Code::Unavailable);
5599    }
5600
5601    #[tokio::test]
5602    async fn snapshot_policy_over_dsql_storage_is_rejected() {
5603        let mut config = TokeiraConfig::default();
5604        config.infrastructure.storage = ConfigStorageKind::Dsql;
5605        config.policy.snapshot = Some(SnapshotPolicyConfig {
5606            location: std::path::PathBuf::from("unused.snapshot"),
5607            interval_ms: 60_000,
5608        });
5609        let error = build_and_serve(
5610            "127.0.0.1:0".parse().expect("loopback addr"),
5611            Arc::new(config),
5612        )
5613        .await
5614        .expect_err("snapshot policy over DSQL must be rejected before any I/O");
5615        let message = format!("{error:#}");
5616        assert!(
5617            message.contains("in-memory store's persistence mechanism"),
5618            "the rejection must name the mechanism: {message}"
5619        );
5620    }
5621
5622    // Feature: distributed-schema-startup-check, Property 3: The policy
5623    // setting selects the check
5624    #[tokio::test]
5625    async fn property_the_policy_setting_selects_the_check() {
5626        for policy in [
5627            None,
5628            Some(DsqlMigrationPolicy::ValidateOnly),
5629            Some(DsqlMigrationPolicy::Automatic),
5630        ] {
5631            let mut config = TokeiraConfig::default();
5632            config.infrastructure.storage = ConfigStorageKind::Dsql;
5633            config.infrastructure.dsql.migration_policy = policy;
5634            let error = build_and_serve(
5635                "127.0.0.1:0".parse().expect("loopback addr"),
5636                Arc::new(config),
5637            )
5638            .await
5639            .expect_err("no endpoint is configured");
5640            let message = format!("{error:#}");
5641            // A start the policy lets through goes on to configure its DSQL
5642            // connection, which needs the endpoint this configuration lacks.
5643            let refused_by_policy = message.contains("migration_policy");
5644            assert_eq!(
5645                refused_by_policy,
5646                policy == Some(DsqlMigrationPolicy::Automatic),
5647                "{policy:?}: {message}"
5648            );
5649            assert!(
5650                refused_by_policy || message.contains("infrastructure.dsql.endpoint must be set"),
5651                "{policy:?}: {message}"
5652            );
5653        }
5654    }
5655
5656    #[test]
5657    fn each_schema_refusal_says_what_was_found_and_what_to_do() {
5658        let refusal = |decision| {
5659            format!(
5660                "{:#}",
5661                distributed_schema_admission(decision).expect_err("the node refuses")
5662            )
5663        };
5664        let reject = |category| {
5665            SchemaDecision::Reject(tokeira_storage::dsql::SchemaIncompatibility {
5666                observed_version: Some(83),
5667                minimum_supported_version: 1,
5668                target_version: 82,
5669                maximum_readable_version: 82,
5670                category,
5671            })
5672        };
5673        assert_eq!(
5674            distributed_schema_admission(SchemaDecision::Compatible {
5675                current: 82,
5676                legacy_backfill: false,
5677            })
5678            .expect("a compatible schema starts"),
5679            82
5680        );
5681        let message = refusal(SchemaDecision::MigrationRequired {
5682            current: 0,
5683            target: 82,
5684        });
5685        assert!(
5686            message.contains("no Tokeira schema")
5687                && message.contains("V82")
5688                && message.contains("tkr schema setup"),
5689            "{message}"
5690        );
5691        let message = refusal(SchemaDecision::MigrationRequired {
5692            current: 81,
5693            target: 82,
5694        });
5695        assert!(
5696            message.contains("V81")
5697                && message.contains("V82")
5698                && message.contains("tkr schema setup"),
5699            "{message}"
5700        );
5701        let message = refusal(reject(SchemaIncompatibilityCategory::FutureSchema));
5702        assert!(
5703            message.contains("a later release migrated it") && message.contains("FutureSchema"),
5704            "{message}"
5705        );
5706        let message = refusal(reject(SchemaIncompatibilityCategory::BelowMinimum));
5707        assert!(
5708            message.contains("older than this release migrates from"),
5709            "{message}"
5710        );
5711        let message = refusal(reject(SchemaIncompatibilityCategory::ChecksumMismatch {
5712            version: 7,
5713            expected: "expected".to_owned(),
5714            observed: "observed".to_owned(),
5715        }));
5716        assert!(
5717            message.contains("migration ledger doesn't match")
5718                && message.contains("ChecksumMismatch"),
5719            "{message}"
5720        );
5721        for decision in [
5722            SchemaDecision::Initialize { target: 82 },
5723            SchemaDecision::Migrate { from: 81, to: 82 },
5724        ] {
5725            let message = refusal(decision);
5726            assert!(
5727                message.contains("validate-only schema assessment decided"),
5728                "{message}"
5729            );
5730        }
5731    }
5732
5733    proptest! {
5734        #![proptest_config(ProptestConfig::with_cases(256))]
5735
5736        // Feature: distributed-schema-startup-check, Property 1: A node starts
5737        // only on a schema its release can run on
5738        #[test]
5739        fn property_a_node_starts_only_on_a_schema_its_release_runs_on(
5740            known in 2_u32..30,
5741            maximum_back in 0_u32..3,
5742            target_back in 0_u32..3,
5743            minimum_seed in 1_u32..30,
5744            current in 0_u32..34,
5745            tampered in proptest::option::of(0_u32..30),
5746        ) {
5747            use tokeira_storage::{
5748                dsql::{
5749                    AppliedMigration, SchemaCompatibilityContract, SchemaObservation,
5750                    assess_schema_compatibility,
5751                },
5752                schema_contract::{MigrationIdentity, migration_set_digest, sha256_hex},
5753            };
5754
5755            let maximum = known.saturating_sub(maximum_back).max(1);
5756            let target = maximum.saturating_sub(target_back).max(1);
5757            let minimum = minimum_seed.clamp(1, target);
5758            let recognized: Vec<MigrationIdentity> = (1..=known)
5759                .map(|version| MigrationIdentity {
5760                    version,
5761                    name: format!("migration_{version}"),
5762                    checksum: sha256_hex(format!("sql-{version}").as_bytes()),
5763                })
5764                .collect();
5765            let contract = SchemaCompatibilityContract {
5766                tokeira_release: "this".to_owned(),
5767                minimum_supported_version: minimum,
5768                target_version: target,
5769                maximum_readable_version: maximum,
5770                migration_set_digest: migration_set_digest(&recognized, maximum)
5771                    .expect("the readable prefix exists"),
5772            };
5773            // The ledger holds versions 1 to `current`; those past `known` are
5774            // a later release's, and `tampered` alters one of the release's.
5775            let tampered = tampered.filter(|version| *version >= 1 && *version <= current.min(known));
5776            let applied_migrations = (1..=current)
5777                .map(|version| match recognized.get(version as usize - 1) {
5778                    Some(identity) => AppliedMigration {
5779                        version,
5780                        name: identity.name.clone(),
5781                        checksum: if tampered == Some(version) {
5782                            sha256_hex(b"changed")
5783                        } else {
5784                            identity.checksum.clone()
5785                        },
5786                    },
5787                    None => AppliedMigration {
5788                        version,
5789                        name: "later_release".to_owned(),
5790                        checksum: sha256_hex(format!("later-{version}").as_bytes()),
5791                    },
5792                })
5793                .collect();
5794            let decision = assess_schema_compatibility(
5795                &contract,
5796                &recognized,
5797                &SchemaObservation { applied_migrations, compatibility: None },
5798                SchemaMigrationPolicy::ValidateOnly,
5799            );
5800
5801            let runs = tampered.is_none() && target <= current && current <= maximum;
5802            match distributed_schema_admission(decision) {
5803                Ok(version) => {
5804                    prop_assert!(runs, "admitted V{current}");
5805                    prop_assert_eq!(version, current);
5806                }
5807                Err(error) => {
5808                    prop_assert!(!runs, "refused V{current}");
5809                    let message = format!("{error:#}");
5810                    if tampered.is_none() && current < target && (current == 0 || current >= minimum) {
5811                        prop_assert!(
5812                            message.contains(&format!("V{target}")) && message.contains("tkr schema setup"),
5813                            "{message}"
5814                        );
5815                    } else {
5816                        prop_assert!(message.contains("schema is incompatible"), "{message}");
5817                    }
5818                }
5819            }
5820        }
5821    }
5822
5823    /// A distributed node never migrates the schema it shares, so `automatic`
5824    /// is refused before any I/O; nothing here names an endpoint, so a startup
5825    /// that went on to connect would fail on that instead.
5826    #[tokio::test]
5827    async fn automatic_migration_policy_over_distributed_dsql_is_refused() {
5828        let mut config = TokeiraConfig::default();
5829        config.infrastructure.storage = ConfigStorageKind::Dsql;
5830        config.infrastructure.dsql.migration_policy = Some(DsqlMigrationPolicy::Automatic);
5831        let error = build_and_serve(
5832            "127.0.0.1:0".parse().expect("loopback addr"),
5833            Arc::new(config),
5834        )
5835        .await
5836        .expect_err("automatic must be refused before any I/O");
5837        let message = format!("{error:#}");
5838        assert!(
5839            message.contains("\"automatic\"") && message.contains("tkr schema setup"),
5840            "the refusal must name the policy and the way to migrate: {message}"
5841        );
5842    }
5843
5844    /// The daemon drain regression: a real SIGTERM to the serving process
5845    /// finishes in-flight work, persists the final snapshot, and the next
5846    /// boot restores from it. Runs in-process (nextest isolates one process
5847    /// per test, so signalling ourselves is safe); the restart half uses the
5848    /// listener facade, which shares the same build path minus the
5849    /// signal/observability install that must stay daemon-only.
5850    #[tokio::test(flavor = "multi_thread")]
5851    async fn sigterm_drains_persists_final_snapshot_and_next_boot_restores() {
5852        use tokeira_proto::workflowservice::{
5853            DescribeWorkflowExecutionRequest, StartWorkflowExecutionRequest,
5854            workflow_service_client::WorkflowServiceClient,
5855        };
5856
5857        let scratch =
5858            std::env::temp_dir().join(format!("tokeira-engine-sigterm-{}", std::process::id()));
5859        std::fs::create_dir_all(&scratch).expect("create scratch dir");
5860        let snapshot_location = scratch.join("tokeirad.snapshot");
5861
5862        // Reserve a loopback port for the daemon config. The tiny window
5863        // between drop and rebind is acceptable in a single-purpose test
5864        // process.
5865        let reserved = std::net::TcpListener::bind("127.0.0.1:0").expect("reserve port");
5866        let addr = reserved.local_addr().expect("reserved addr");
5867        drop(reserved);
5868
5869        let mut config = TokeiraConfig::default();
5870        config.infrastructure.network.grpc_addr = addr.to_string();
5871        // Loopback-ephemeral Nexus completion listener, exactly as the
5872        // in-memory facade configures it, so parallel tests never collide.
5873        config.policy.nexus_completion.http_addr = "127.0.0.1:0".to_string();
5874        config.policy.nexus_completion.system_callback_url = "http://127.0.0.1:0".to_string();
5875        config.policy.snapshot = Some(SnapshotPolicyConfig {
5876            location: snapshot_location.clone(),
5877            // Effectively periodic-off: this regression is about the FINAL
5878            // cut on the signal path.
5879            interval_ms: 3_600_000,
5880        });
5881        let config_path = scratch.join("tokeirad.toml");
5882        std::fs::write(&config_path, config.to_toml().expect("render config"))
5883            .expect("write config");
5884
5885        let mut daemon = tokio::spawn(run_from_cli(Cli {
5886            config: Some(config_path.display().to_string()),
5887            dump_config: false,
5888            version: false,
5889            verbose: false,
5890            json: false,
5891        }));
5892
5893        // Connect-poll until the daemon serves. The retry interval is a
5894        // polling cadence against an external TCP bind (the one boundary a
5895        // channel cannot observe), bounded by the deadline; a daemon that
5896        // exits early fails the wait immediately with its own error.
5897        let target = format!("http://{addr}");
5898        let deadline = tokio::time::Instant::now() + std::time::Duration::from_secs(20);
5899        let mut client = loop {
5900            assert!(
5901                tokio::time::Instant::now() < deadline,
5902                "daemon must start serving"
5903            );
5904            tokio::select! {
5905                result = &mut daemon => {
5906                    panic!("daemon exited before serving: {result:?}");
5907                }
5908                connected = WorkflowServiceClient::connect(target.clone()) => {
5909                    match connected {
5910                        Ok(client) => break client,
5911                        Err(_) => {
5912                            tokio::time::sleep(std::time::Duration::from_millis(50)).await;
5913                        }
5914                    }
5915                }
5916            }
5917        };
5918        client
5919            .start_workflow_execution(StartWorkflowExecutionRequest {
5920                namespace: "default".to_owned(),
5921                workflow_id: "sigterm-canary".to_owned(),
5922                workflow_type: Some(tokeira_proto::common::WorkflowType {
5923                    name: "sigterm-workflow".to_owned(),
5924                }),
5925                task_queue: Some(tokeira_proto::taskqueue::TaskQueue {
5926                    name: "sigterm-queue".to_owned(),
5927                    ..Default::default()
5928                }),
5929                request_id: "sigterm-start".to_owned(),
5930                ..Default::default()
5931            })
5932            .await
5933            .expect("start canary workflow");
5934
5935        // The real signal, sent to this very process; the drain must treat it
5936        // exactly as an operator's `kill`.
5937        let delivered = std::process::Command::new("kill")
5938            .args(["-TERM", &std::process::id().to_string()])
5939            .status()
5940            .expect("send SIGTERM");
5941        assert!(delivered.success(), "kill -TERM must be deliverable");
5942
5943        daemon
5944            .await
5945            .expect("daemon task must not panic")
5946            .expect("SIGTERM drain must exit cleanly");
5947        assert!(
5948            snapshot_location.is_file(),
5949            "the drain must persist the final snapshot"
5950        );
5951
5952        // Next boot restores the canary from the snapshot.
5953        let restored =
5954            TokeiradHandle::start_in_memory_with_config("127.0.0.1:0".parse().expect("addr"), {
5955                let mut config = TokeiraConfig::default();
5956                config.policy.snapshot = Some(SnapshotPolicyConfig {
5957                    location: snapshot_location.clone(),
5958                    interval_ms: 3_600_000,
5959                });
5960                config
5961            })
5962            .await
5963            .expect("restart over the final snapshot");
5964        let mut client =
5965            WorkflowServiceClient::connect(format!("http://{}", restored.bound_addr()))
5966                .await
5967                .expect("connect to restarted server");
5968        let described = client
5969            .describe_workflow_execution(DescribeWorkflowExecutionRequest {
5970                namespace: "default".to_owned(),
5971                execution: Some(tokeira_proto::common::WorkflowExecution {
5972                    workflow_id: "sigterm-canary".to_owned(),
5973                    run_id: String::new(),
5974                }),
5975            })
5976            .await
5977            .expect("restored workflow must be describable")
5978            .into_inner();
5979        assert!(
5980            described.workflow_execution_info.is_some(),
5981            "state must survive the SIGTERM drain via snapshot restore"
5982        );
5983        restored.shutdown().await.expect("clean restart shutdown");
5984        let _ = std::fs::remove_dir_all(&scratch);
5985    }
5986
5987    #[test]
5988    fn embedded_config_removes_daemon_storage_and_placement() {
5989        let mut config = TokeiraConfig::default();
5990        config.infrastructure.storage = ConfigStorageKind::Dsql;
5991        config.infrastructure.placement.controller_endpoint =
5992            Some("http://controller.internal:8080".to_owned());
5993
5994        let config = embedded_config(config).expect("embedded defaults are valid");
5995
5996        assert_eq!(config.infrastructure.storage, ConfigStorageKind::InMemory);
5997        assert!(
5998            config
5999                .infrastructure
6000                .placement
6001                .controller_endpoint
6002                .is_none()
6003        );
6004    }
6005
6006    #[tokio::test]
6007    async fn embedded_system_nexus_callback_never_falls_through_to_http() {
6008        let client = InProcessNexusCompletionClient::new();
6009        let target = system_callback_post_url(EMBEDDED_NEXUS_CALLBACK_BASE);
6010        let outcome = client
6011            .complete_operation(
6012                &target,
6013                "not-yet-attached",
6014                "operation-token",
6015                NexusCompletion::Succeeded(tokeira_types::Payloads::default()),
6016                &[],
6017            )
6018            .await
6019            .expect("the in-process preflight outcome is classifiable");
6020
6021        assert!(matches!(
6022            outcome,
6023            CompletionDeliveryOutcome::RetryableError { detail }
6024                if detail == "embedded Nexus completion runtime is unavailable"
6025        ));
6026    }
6027
6028    // Security F1 (apps scope): the network-exposed `/nexus/callback` listener must cap
6029    // its request body before buffering, so an unauthenticated caller cannot exhaust
6030    // memory and abort the `panic = "abort"` process.
6031    #[tokio::test]
6032    async fn bounded_body_caps_oversized_payload() {
6033        // Within the cap: fully collected.
6034        match collect_bounded_body(Full::new(Bytes::from(vec![0u8; 8])), 16).await {
6035            BoundedBody::Collected(bytes) => assert_eq!(bytes.len(), 8),
6036            other => panic!("expected Collected, got {other:?}"),
6037        }
6038        // Over the cap: rejected before the whole body is buffered.
6039        assert!(matches!(
6040            collect_bounded_body(Full::new(Bytes::from(vec![0u8; 17])), 16).await,
6041            BoundedBody::TooLarge
6042        ));
6043    }
6044
6045    #[test]
6046    fn with_loopback_port_rewrites_or_appends_port() {
6047        // A base URL with an explicit port has it replaced with the bound port.
6048        assert_eq!(
6049            with_loopback_port("http://127.0.0.1:7253", 51_000),
6050            "http://127.0.0.1:51000"
6051        );
6052        // A trailing slash is trimmed before rewriting.
6053        assert_eq!(
6054            with_loopback_port("http://127.0.0.1:7253/", 51_000),
6055            "http://127.0.0.1:51000"
6056        );
6057        // A port-less base URL gets the port appended (the scheme `:` is not mistaken for
6058        // a port because the suffix is not all-digits).
6059        assert_eq!(
6060            with_loopback_port("http://localhost", 51_000),
6061            "http://localhost:51000"
6062        );
6063    }
6064
6065    #[test]
6066    fn version_renderer_is_deterministic() {
6067        let short = render_build_info(false, false);
6068        let verbose = render_build_info(true, false);
6069        let json = render_build_info(false, true);
6070
6071        assert_eq!(short, render_build_info(false, false));
6072        assert_eq!(verbose, render_build_info(true, false));
6073        assert_eq!(json, render_build_info(false, true));
6074        assert!(short.contains(tokeira_build_info::SERVER_VERSION));
6075        assert!(short.contains(tokeira_build_info::TEMPORAL_PROTO_VERSION));
6076        assert!(short.contains(tokeira_build_info::TEMPORAL_SERVER_COMPAT));
6077        assert!(json.contains("server_version"));
6078        assert!(json.contains("temporal_proto_version"));
6079    }
6080
6081    #[test]
6082    fn reported_problem_search_attribute_has_exact_v131_keyword_list() {
6083        let mut search_attributes = tokeira_types::SearchAttributes::default();
6084        apply_reported_problem_search_attribute(
6085            &mut search_attributes,
6086            Some(WorkflowTaskReportedProblem {
6087                attempts_since_last_success: 5,
6088                problem: tokeira_kernel::WorkflowTaskProblem::Failed(
6089                    tokeira_kernel::WorkflowTaskFailedCause::WorkflowWorkerUnhandledFailure,
6090                ),
6091            }),
6092        );
6093
6094        assert_eq!(
6095            search_attributes.0.get("TemporalReportedProblems"),
6096            Some(&SearchAttrValue::KeywordList(vec![
6097                "category=WorkflowTaskFailed".to_string(),
6098                "cause=WorkflowTaskFailedCauseWorkflowWorkerUnhandledFailure".to_string(),
6099            ]))
6100        );
6101    }
6102
6103    #[test]
6104    fn reported_problem_search_attribute_renders_timeout_category() {
6105        // A timeout-first sequence reports the TimedOut pair — v1.31.0 renders
6106        // `TimeoutType.String()` for the only stored type
6107        // (`mutable_state_impl.go:6486-6491 @ v1.31.0`).
6108        let mut search_attributes = tokeira_types::SearchAttributes::default();
6109        apply_reported_problem_search_attribute(
6110            &mut search_attributes,
6111            Some(WorkflowTaskReportedProblem {
6112                attempts_since_last_success: 5,
6113                problem: tokeira_kernel::WorkflowTaskProblem::TimedOutStartToClose,
6114            }),
6115        );
6116
6117        assert_eq!(
6118            search_attributes.0.get("TemporalReportedProblems"),
6119            Some(&SearchAttrValue::KeywordList(vec![
6120                "category=WorkflowTaskTimedOut".to_string(),
6121                "cause=WorkflowTaskTimedOutCauseStartToClose".to_string(),
6122            ]))
6123        );
6124    }
6125
6126    fn self_assigning_runtime(
6127        store: Arc<InMemoryStore>,
6128        shard_count: u32,
6129    ) -> Arc<TokeiraRuntime<InMemoryStore>> {
6130        Arc::new(TokeiraRuntime::new_with_nexus_and_shards_and_endpoint(
6131            store,
6132            2,
6133            tokeira_runtime::LaneConfig::default(),
6134            tokeira_runtime::TimerScannerConfig::default(),
6135            tokeira_runtime::WorkflowTimeoutScannerConfig::default(),
6136            tokeira_runtime::BacklogConfig::default(),
6137            tokeira_runtime::ActivityTimeoutScannerConfig::default(),
6138            tokeira_runtime::NexusTimeoutScannerConfig::default(),
6139            NexusEndpointRegistry::default(),
6140            Arc::new(tokeira_runtime::NoopNexusHttpClient),
6141            NexusCompletionDeps::default(),
6142            shard_count,
6143            IncarnationId::new().to_string(),
6144            "127.0.0.1:7233".to_owned(),
6145            false,
6146            None,
6147        ))
6148    }
6149
6150    fn boot_start_request(
6151        namespace_id: tokeira_types::NamespaceId,
6152        index: usize,
6153    ) -> tokeira_kernel::StartRequest {
6154        let run_id = tokeira_types::RunId::new();
6155        let workflow_id = tokeira_types::WorkflowId(format!("boot-{index}"));
6156        let now = time::OffsetDateTime::now_utc();
6157        tokeira_kernel::StartRequest {
6158            advice_policy: tokeira_kernel::ContinueAsNewAdvicePolicy::V1_31_0,
6159            initiator: None,
6160            run_key: tokeira_types::RunKey::derive(namespace_id, &workflow_id, run_id),
6161            namespace_id,
6162            workflow_id,
6163            run_id,
6164            workflow_type: tokeira_types::WorkflowType("boot".to_owned()),
6165            task_queue: tokeira_types::TaskQueueName("boot-queue".to_owned()),
6166            input: tokeira_types::Payloads::default(),
6167            header: None,
6168            memo: tokeira_types::Memo::default(),
6169            search_attributes: tokeira_types::SearchAttributes::default(),
6170            workflow_execution_timeout: None,
6171            workflow_run_timeout: None,
6172            workflow_task_timeout: time::Duration::seconds(10),
6173            retry_policy: None,
6174            conflict_policy: tokeira_kernel::WorkflowIdConflictPolicy::Fail,
6175            reuse_policy: tokeira_kernel::WorkflowIdReusePolicy::AllowDuplicate,
6176            deployment: None,
6177            build_id: None,
6178            versioning_override: None,
6179            workflow_start_delay: None,
6180            completion_callbacks: Vec::new(),
6181            user_metadata: None,
6182            links: Vec::new(),
6183            on_conflict_options: None,
6184            priority: None,
6185            attempt: 1,
6186            continued_execution_run_id: None,
6187            first_execution_run_id: None,
6188            parent_run_key: None,
6189            parent_workflow_id: None,
6190            parent_run_id: None,
6191            parent_namespace_id: None,
6192            parent_namespace_name: None,
6193            parent_initiated_event_id: 0,
6194            root_workflow_id: None,
6195            root_run_id: None,
6196            original_execution_run_id: Some(run_id),
6197            continued_failure: None,
6198            last_completion_result: None,
6199            first_run_started_at: None,
6200            request: tokeira_types::RequestContext {
6201                request_id: tokeira_types::RequestId(format!("boot-start-{index}")),
6202                caller_identity: None,
6203                principal: None,
6204                received_at: now,
6205            },
6206            now,
6207            client_cron_schedule: None,
6208            cron_schedule: None,
6209            eager_execution_accepted: false,
6210            reserved_poller_identity: None,
6211            inherited_versioning_info: None,
6212        }
6213    }
6214
6215    #[tokio::test]
6216    async fn self_assigned_dsql_boot_recovers_every_shard_before_admission() -> Result<()> {
6217        // A single node without a controller takes every shard's lease at
6218        // boot. It must also sweep each shard: the previous process's offered
6219        // workflow tasks ended with it (runtime-sweeper-recovery
6220        // Requirement 11.6).
6221        let shard_count = 2;
6222        let store = Arc::new(InMemoryStore::with_shard_count(shard_count));
6223        let endpoint = NodeEndpoint {
6224            host: "127.0.0.1".to_owned(),
6225            port: 7233,
6226        };
6227        let namespace_id = tokeira_types::NamespaceId::new();
6228
6229        let previous = self_assigning_runtime(Arc::clone(&store), shard_count);
6230        self_assign_dsql_shards(
6231            previous.as_ref(),
6232            store.as_ref(),
6233            shard_count,
6234            &IncarnationId::new(),
6235            &endpoint,
6236            true,
6237        )
6238        .await?;
6239        let mut started = Vec::new();
6240        let mut shards = std::collections::BTreeSet::new();
6241        for index in 0..8 {
6242            let request = boot_start_request(namespace_id, index);
6243            shards.insert(tokeira_types::execution_home_bundle(
6244                namespace_id.0.as_bytes(),
6245                request.workflow_id.0.as_bytes(),
6246                shard_count,
6247            ));
6248            started.push(request.run_key);
6249            let result = previous.start_workflow(request).await?;
6250            assert!(matches!(
6251                result,
6252                tokeira_storage::CommitResult::Applied { .. }
6253            ));
6254        }
6255        assert_eq!(shards.len(), 2, "the seeded runs span both shards");
6256
6257        let restarted = self_assigning_runtime(Arc::clone(&store), shard_count);
6258        let leases = self_assign_dsql_shards(
6259            restarted.as_ref(),
6260            store.as_ref(),
6261            shard_count,
6262            &IncarnationId::new(),
6263            &endpoint,
6264            true,
6265        )
6266        .await?;
6267        assert_eq!(leases.len(), 2);
6268        let mut active = restarted.active_shards();
6269        active.sort();
6270        assert_eq!(active, vec![ShardId(0), ShardId(1)]);
6271
6272        let queue = tokeira_types::QueueKey {
6273            namespace_id,
6274            task_queue: tokeira_types::TaskQueueName("boot-queue".to_owned()),
6275            task_kind: tokeira_types::TaskKind::Workflow,
6276            deployment: None,
6277            build_id: None,
6278        };
6279        let mut offered = Vec::new();
6280        while offered.len() < started.len() {
6281            let task = restarted
6282                .poll_workflow_task(
6283                    queue.clone(),
6284                    tokeira_types::WorkerIdentity("worker".to_owned()),
6285                    tokio::time::Duration::from_secs(1),
6286                )
6287                .await?
6288                .expect("every pending first workflow task is offered again");
6289            offered.push(task.run_key);
6290        }
6291        offered.sort();
6292        started.sort();
6293        assert_eq!(offered, started);
6294        Ok(())
6295    }
6296
6297    #[tokio::test]
6298    async fn build_nexus_endpoint_store_resolves_worker_namespace_names() {
6299        let cache = InMemoryNamespaceCache::new();
6300        let namespace = ResolvedNamespace::active("payments");
6301        let namespace_id = namespace_id_for("payments");
6302        cache.insert(namespace).await.expect("namespace insert");
6303
6304        let store = build_nexus_endpoint_store(
6305            &cache,
6306            HashMap::from([(
6307                "payments-endpoint".to_string(),
6308                BootstrapNexusEndpointConfig {
6309                    target: BootstrapNexusEndpointTarget::Worker {
6310                        namespace_name: "payments".to_string(),
6311                        task_queue: "nexus-q".to_string(),
6312                    },
6313                },
6314            )]),
6315        )
6316        .await
6317        .expect("store should build");
6318
6319        let registry = NexusEndpointRegistry::new(store);
6320        let config = registry.resolve("payments-endpoint").expect("endpoint");
6321        assert_eq!(
6322            config,
6323            NexusEndpointConfig {
6324                target: EndpointTarget::Worker {
6325                    namespace_id,
6326                    task_queue: tokeira_types::TaskQueueName("nexus-q".to_string()),
6327                },
6328            }
6329        );
6330    }
6331
6332    #[tokio::test]
6333    async fn build_nexus_endpoint_store_rejects_unknown_worker_namespace() {
6334        let cache = InMemoryNamespaceCache::new();
6335
6336        let result = build_nexus_endpoint_store(
6337            &cache,
6338            HashMap::from([(
6339                "payments-endpoint".to_string(),
6340                BootstrapNexusEndpointConfig {
6341                    target: BootstrapNexusEndpointTarget::Worker {
6342                        namespace_name: "missing".to_string(),
6343                        task_queue: "nexus-q".to_string(),
6344                    },
6345                },
6346            )]),
6347        )
6348        .await;
6349
6350        assert!(result.is_err(), "missing namespace should fail");
6351        let error = result.expect_err("error");
6352
6353        assert!(error.to_string().contains("namespace `missing` not found"));
6354    }
6355
6356    #[test]
6357    fn placement_helpers_build_runtime_and_edge_config_from_server_config() {
6358        let mut config = TokeiraConfig::default();
6359        config.infrastructure.placement.controller_endpoint =
6360            Some("http://127.0.0.1:7240".to_string());
6361        config.infrastructure.placement.node_host = "10.0.0.9".to_string();
6362        config.infrastructure.placement.node_port = Some(8123);
6363        config.infrastructure.placement.shard_count = 8;
6364        config.infrastructure.placement.bundle_count = 4;
6365        config.infrastructure.placement.partition_count = 64;
6366        config.infrastructure.placement.hash_version = 2;
6367        let listen_addr = "127.0.0.1:7233".parse().unwrap();
6368        let node_id = IncarnationId::new();
6369        let endpoint = configured_node_endpoint(&config, listen_addr);
6370        let membership = membership_config(
6371            &config,
6372            node_id,
6373            endpoint.clone(),
6374            "http://127.0.0.1:7240".into(),
6375        );
6376
6377        assert_eq!(endpoint.as_authority(), "10.0.0.9:8123");
6378        assert_eq!(membership.node_id, node_id);
6379        assert_eq!(membership.node_endpoint, endpoint);
6380        assert_eq!(
6381            placement_config(&config),
6382            PlacementConfig {
6383                shard_count: 8,
6384                bundle_count: 4,
6385                partition_count: 64,
6386                hash_version: 2,
6387            }
6388        );
6389    }
6390
6391    #[test]
6392    fn dsql_pool_config_uses_effective_placement_values() {
6393        let mut config = TokeiraConfig::default();
6394        config.infrastructure.placement.shard_count = 32;
6395        config.infrastructure.placement.partition_count = 4;
6396
6397        let (pool_config, _ddb_client) =
6398            dsql_pool_config_with_client(&config, tokeira_storage::dsql::offline_ddb_client());
6399
6400        assert_eq!(pool_config.shard_count, 32);
6401        assert_eq!(pool_config.projection_partition_count, 4);
6402    }
6403
6404    #[test]
6405    fn wire_coverage_out_path_prefers_env_override_then_falls_back_to_default() {
6406        // Exercised in one test because all cases mutate the same process-global env var;
6407        // splitting them risks cross-test interference under the parallel test runner.
6408
6409        // Unset: the documented default path is used.
6410        // SAFETY: all mutation of this env var is confined to this single test.
6411        unsafe { std::env::remove_var(WIRE_COVERAGE_OUT_ENV) };
6412        assert_eq!(
6413            wire_coverage_out_path(),
6414            PathBuf::from(WIRE_COVERAGE_DEFAULT_OUT)
6415        );
6416
6417        // A real path is honoured verbatim (after trimming).
6418        // SAFETY: all mutation of this env var is confined to this single test.
6419        unsafe { std::env::set_var(WIRE_COVERAGE_OUT_ENV, "  /tmp/cov.json  ") };
6420        assert_eq!(wire_coverage_out_path(), PathBuf::from("/tmp/cov.json"));
6421
6422        // An empty/whitespace override is treated as unset, not as a blank path.
6423        // SAFETY: all mutation of this env var is confined to this single test.
6424        unsafe { std::env::set_var(WIRE_COVERAGE_OUT_ENV, "   ") };
6425        assert_eq!(
6426            wire_coverage_out_path(),
6427            PathBuf::from(WIRE_COVERAGE_DEFAULT_OUT)
6428        );
6429
6430        // SAFETY: all mutation of this env var is confined to this single test.
6431        unsafe { std::env::remove_var(WIRE_COVERAGE_OUT_ENV) };
6432    }
6433
6434    #[test]
6435    fn export_wire_coverage_writes_pretty_json_snapshot() {
6436        let recorder = WireCoverageRecorder::new();
6437        recorder.record(
6438            "/temporal.api.workflowservice.v1.WorkflowService/StartWorkflowExecution",
6439            0,
6440        );
6441
6442        let dir = std::env::temp_dir().join(format!("tokeirad-cov-{}", std::process::id()));
6443        std::fs::create_dir_all(&dir).expect("temp dir");
6444        let path = dir.join("wire-coverage.json");
6445
6446        export_wire_coverage(&recorder, &path).expect("export succeeds");
6447
6448        let written = std::fs::read_to_string(&path).expect("evidence file readable");
6449        assert!(written.contains("StartWorkflowExecution"));
6450        // Pretty JSON is multi-line; a compact encoding would be a single line.
6451        assert!(written.contains('\n'));
6452
6453        std::fs::remove_dir_all(&dir).ok();
6454    }
6455}