Skip to main content

tokeira_engine/
lib.rs

1//! Embeddable Temporal-compatible execution engine and `tokeirad` service bootstrap.
2//!
3//! [`Engine`] is the zero-listener entry point: it owns an in-memory authoritative
4//! store, runtime workers, and the Temporal edge router. Its [`TemporalEndpoint`]
5//! accepts raw protobuf calls and, with the default `temporalio-client` feature,
6//! produces a callback service for `ConnectionOptions::service_override`.
7//!
8//! `tokeirad` also consumes this crate. [`TokeiradHandle::start_in_memory`] and
9//! [`run_from_cli`] attach the optional gRPC, HTTP/JSON, gRPC-Web, and Nexus HTTP
10//! transports after the shared service graph has been constructed. Embedded and
11//! listener-backed operation therefore execute the same edge handlers and runtime
12//! semantics; only their transport and shutdown handles differ.
13
14#![deny(rust_2018_idioms)]
15// The bootstrap facade wires many subsystems and accepts them as parameters.
16#![allow(clippy::too_many_arguments)]
17
18use tokeira_kernel::deadline_after;
19
20use std::{
21    collections::HashMap,
22    convert::Infallible,
23    future::Future,
24    net::SocketAddr,
25    path::{Path, PathBuf},
26    pin::Pin,
27    sync::{
28        Arc, RwLock, Weak,
29        atomic::{AtomicU64, Ordering},
30    },
31    time::{Duration as StdDuration, Instant},
32};
33
34use anyhow::{Context, Result, anyhow};
35use async_trait::async_trait;
36use clap::Parser;
37use http_body_util::{BodyExt, Full, LengthLimitError, Limited};
38use hyper::{
39    Method, Request, Response, StatusCode,
40    body::{Bytes, Incoming},
41    service::service_fn,
42};
43use hyper_util::rt::TokioIo;
44use tokio::{
45    fs::{self, OpenOptions},
46    io::AsyncWriteExt as _,
47    net::TcpListener,
48    sync::{broadcast, mpsc, oneshot},
49    task::JoinHandle,
50};
51use tokio_stream::wrappers::TcpListenerStream;
52use tokio_util::sync::CancellationToken;
53use tonic::transport::Server;
54use tonic_web::GrpcWebLayer;
55use tower_http::cors::CorsLayer;
56use tracing::{error, info};
57
58#[cfg(feature = "chasm-extensions")]
59pub mod chasm;
60mod chasm_extensions;
61#[cfg(feature = "chasm-extensions")]
62pub use chasm_extensions::EngineBuilder;
63use chasm_extensions::{
64    ChasmExtensions, check_rebuilt_outboxes, check_registered_archetypes,
65    declared_search_attributes, embedded_stack_error, registry_start_error,
66    seed_declared_search_attributes, seed_start_error,
67};
68
69pub mod correlation_format;
70#[cfg(all(test, feature = "dsql-integration"))]
71mod distributed_schema_live_tests;
72#[cfg(test)]
73mod embedded_dsql_tests;
74#[doc(hidden)]
75pub mod harness;
76mod http_api_transport;
77mod listener;
78mod nexus_http_transport;
79mod node_claim;
80pub mod observability;
81mod peer;
82#[cfg(all(test, feature = "dsql-integration"))]
83mod single_node_claim_live_tests;
84mod storage_maintenance;
85
86pub use listener::{EngineListenError, EngineListener, EngineListenerShutdownError};
87
88use http_api_transport::HttpApiLayer;
89use nexus_http_transport::NexusHttpLayer;
90use tokeira_auth::{
91    Authorizer, ClaimMapper, DefaultAuthorizer, GrantRule, GrantRules, JwksKeyProvider,
92    JwtAuthenticator, JwtIssuerProfile, MultiSourceClaimMapper, StsAuthenticator, WorkerScope,
93    WorkerScopeRule, WorkerScopeRules,
94};
95use tokeira_chasm::Library as _;
96use tokeira_config::{Cli, CliCommand, ConfigStorageKind, PeerCredentialCommand};
97pub use tokeira_config::{
98    DsqlMigrationPolicy, EmbeddedEngineConfig, EmbeddedStorageConfig, SnapshotPolicyConfig,
99    TokeiraConfig,
100};
101use tokeira_edge::{
102    Authenticator, CacheBackedRouter, CallbackResponse, EdgeInterceptors, EdgeRoutingConfig,
103    HistoryNotifyingRepository, HistoryWaitRegistry, HttpApiCatalog, HttpApiPolicy,
104    InMemoryNamespaceCache, InMemoryOperatorApi, InProcessGrpcService, LocalOnlyRouter,
105    LongPollConfig, LongPollGate, NamespaceCache, OperatorService, PendingQueryStore,
106    PollerRegistry, ResolvedNamespace, RoutingCache, ScopedWorkerSessionRegistry,
107    WorkerComputeNamespaceCatalogAdapter, WorkflowExecutionDescription, WorkflowService,
108    conformance::{WireCoverageLayer, WireCoverageRecorder},
109    grpc::{
110        admin_service::AdminServiceGrpc, operator_service::OperatorServiceGrpc,
111        runtime_adapter::RuntimeAdapter, workflow_service::WorkflowServiceGrpc,
112    },
113    handle_nexus_callback,
114    nexus_http::MAX_NEXUS_PAYLOAD_BYTES,
115    operator_service::{ClusterInfo, OperatorApi, SearchAttributeDefinition},
116    run_routing_subscription,
117    translate::to_internal::namespace_id_for,
118    workflow_service::{ExecutionResolver, WorkflowRuntimeApi},
119};
120pub use tokeira_edge::{InProcessGrpcRequest, InProcessGrpcResponse};
121
122struct AuthorizationStack {
123    grpc: Arc<dyn Authenticator>,
124    nexus: Arc<dyn tokeira_edge::nexus_http::NexusHttpAuthorizer>,
125    principal_attribution: bool,
126}
127
128async fn build_authorization_stack(config: &TokeiraConfig) -> Result<AuthorizationStack> {
129    let Some(authorization) = config
130        .policy
131        .authorization
132        .as_ref()
133        .filter(|authorization| authorization.has_identity_source())
134    else {
135        let grpc = harness::installed()
136            .and_then(|hooks| hooks.fallback_grpc_authenticator.clone())
137            .unwrap_or_else(|| Arc::new(tokeira_edge::AllowAllAuthenticator));
138        let stack = AuthorizationStack {
139            grpc,
140            nexus: Arc::new(tokeira_edge::nexus_http::PermissiveNexusHttpAuthorizer),
141            principal_attribution: false,
142        };
143        return Ok(stack);
144    };
145
146    let mut profiles = Vec::with_capacity(authorization.jwt.issuers.len());
147    for issuer in &authorization.jwt.issuers {
148        let grants = GrantRules::new(
149            issuer
150                .grants
151                .iter()
152                .map(|rule| {
153                    GrantRule::new(
154                        rule.match_sub.clone(),
155                        rule.grant.iter().map(String::as_str),
156                    )
157                    .with_context(|| format!("invalid JWT grants for issuer {}", issuer.name))
158                })
159                .collect::<Result<Vec<_>>>()?,
160        );
161        let worker_scopes = WorkerScopeRules::new(
162            issuer
163                .worker_scopes
164                .iter()
165                .map(|rule| {
166                    let scope = WorkerScope::try_new(
167                        rule.namespace.clone(),
168                        rule.task_queues.clone(),
169                        rule.deployment_name.clone(),
170                        rule.build_id.clone(),
171                    )
172                    .with_context(|| {
173                        format!("invalid JWT Worker scope for issuer {}", issuer.name)
174                    })?;
175                    WorkerScopeRule::new(rule.match_sub.clone(), scope).with_context(|| {
176                        format!(
177                            "invalid JWT Worker-scope subject for issuer {}",
178                            issuer.name
179                        )
180                    })
181                })
182                .collect::<Result<Vec<_>>>()?,
183        );
184        let keys =
185            JwksKeyProvider::start(issuer.jwks_uri.clone(), issuer.refresh_interval_duration())
186                .await;
187        profiles.push(
188            JwtIssuerProfile::new(
189                issuer.name.clone(),
190                issuer.issuer.clone(),
191                issuer.audience.clone(),
192                issuer.permissions_claim.clone(),
193                grants,
194                keys,
195            )
196            .with_worker_scopes(worker_scopes),
197        );
198    }
199    let jwt = (!profiles.is_empty()).then(|| JwtAuthenticator::new(profiles));
200    let sts = authorization
201        .aws_iam
202        .as_ref()
203        .map(|aws_iam| {
204            let grants = aws_iam
205                .grants
206                .iter()
207                .map(|rule| {
208                    GrantRule::new(
209                        rule.match_arn.clone(),
210                        rule.grant.iter().map(String::as_str),
211                    )
212                    .context("invalid AWS IAM grants")
213                })
214                .collect::<Result<Vec<_>>>()
215                .map(GrantRules::new)?;
216            let worker_scopes = WorkerScopeRules::new(
217                aws_iam
218                    .worker_scopes
219                    .iter()
220                    .map(|rule| {
221                        let scope = WorkerScope::try_new(
222                            rule.namespace.clone(),
223                            rule.task_queues.clone(),
224                            rule.deployment_name.clone(),
225                            rule.build_id.clone(),
226                        )
227                        .context("invalid AWS IAM Worker scope")?;
228                        WorkerScopeRule::new(rule.match_arn.clone(), scope)
229                            .context("invalid AWS IAM Worker-scope ARN")
230                    })
231                    .collect::<Result<Vec<_>>>()?,
232            );
233            Ok::<_, anyhow::Error>(StsAuthenticator::new(grants).with_worker_scopes(worker_scopes))
234        })
235        .transpose()?;
236    let mapper: Arc<dyn ClaimMapper> = Arc::new(MultiSourceClaimMapper::new(jwt, sts));
237    let authorizer: Arc<dyn Authorizer> = Arc::new(DefaultAuthorizer);
238    Ok(AuthorizationStack {
239        grpc: Arc::new(tokeira_edge::PolicyAuthenticator::new(
240            mapper.clone(),
241            authorizer.clone(),
242            authorization.expose_authorizer_errors,
243        )),
244        nexus: Arc::new(tokeira_edge::nexus_http::PolicyNexusHttpAuthorizer::new(
245            mapper,
246            authorizer,
247            authorization.expose_authorizer_errors,
248        )),
249        principal_attribution: authorization.principal_attribution,
250    })
251}
252use tokeira_kernel::{Link, LoadedRun};
253use tokeira_managed_dsql::{
254    AwsDsqlControlPlane, CanonicalClusterIdentity, ClusterAction, CreateOrRecoverRequest,
255    DsqlControlPlane, LifecycleEnvironment, LocalClusterDescriptorStore, ManagedDsqlLifecycle,
256    Readiness, ResolvedCluster, StartupDeadline, SystemLifecycleEnvironment,
257};
258use tokeira_observability::{
259    ClusterStatusLabel, DbClassLabel, EmbeddedOperationLabel, EmbeddedStorageModeLabel,
260    ErrorClassLabel, OwnershipOutcomeLabel, SchemaOutcomeLabel, record_embedded_lifecycle,
261};
262use tokeira_projection::{
263    DsqlVisibilityStore, InMemoryVisibilityStore, ProjectionSink, ProjectorHome,
264    ProjectorSupervisor, SearchAttrType, VisibilityQueryService, VisibilitySink, VisibilityStore,
265};
266use tokeira_runtime::{
267    ChannelDemandObservationSink, ChannelWorkerComputeReconcileSink,
268    CompletionCallbackScannerConfig, CompletionDeliveryOutcome, ConnectionBudgetApplier,
269    HttpNexusClient, HttpNexusCompletionClient, InMemoryNexusEndpointStore, MembershipConfig,
270    NEXUS_CALLBACK_PATH, NEXUS_OPERATION_STATE_HEADER, NEXUS_OPERATION_TOKEN_HEADER,
271    NexusCompletion, NexusCompletionClient, NexusCompletionDeps, NexusCompletionRuntimeConfig,
272    NexusEndpointRegistry, NexusEndpointSpec, NexusEndpointSpecTarget, NexusEndpointStore,
273    NexusNamespaceResolver, NexusWorkerComputeProvider, OBSERVATION_CHANNEL_CAPACITY,
274    RECONCILE_CHANNEL_CAPACITY, RepositoryBackedTaskQueueConfigStore, RuntimeConfig,
275    RuntimeShutdownHandle, ScheduleEngineConfig, ScheduleStore, SweepResult,
276    SystemWorkerComputeClock, TEMPORAL_CALLBACK_TOKEN_HEADER, TokeiraRuntime,
277    WorkerComputeControllerService, WorkerComputeOutbox, WorkerComputeReconciler,
278    WorkflowTaskReportedProblem, nexus_payload_to_body, reported_problem_from_state,
279    run_schedule_engine, system_callback_post_url,
280};
281use tokeira_storage::{
282    ConnectionDirector, DbClass, InMemoryStore, InMemoryWorkerComputeRepository, LeaseOutcome,
283    LeaseRepository, ProjectionLog, RunRepository, TaskQueueConfigRepository,
284    WorkerComputeRepository, WorkerDeploymentRepository, WorkerTaskProvenanceStore,
285    dsql::{
286        ConnectionControlLeaseRepository, ControlLeaseAcquireOutcome, ControlLeaseAcquireRequest,
287        ControlLeaseClusterIdentity, ControlLeaseError, ControlLeaseGuard, DsqlAuthConfig,
288        DsqlConnectionDirector, DsqlCoordinationConfig, DsqlPoolConfig, DsqlStore,
289        EmbeddedDsqlPoolConfig, MigrationRunner, OWNER_ADMISSION_MARGIN, OWNER_LEASE_DURATION,
290        OWNER_RENEW_INTERVAL, OwnershipAdmissionGate, OwnershipAdmissionState,
291        SCHEMA_MIGRATION_ADMISSION_MARGIN, SCHEMA_MIGRATION_LEASE_DURATION,
292        SchemaCompatibilityError, SchemaDecision, SchemaIncompatibilityCategory,
293        SchemaMigrationPolicy, WarmupDeadline,
294    },
295};
296use tokeira_types::{
297    ExecutionRef, IncarnationId, NamespaceId, NodeEndpoint, PlacementConfig, SearchAttrValue,
298    ShardEpoch, ShardId, WorkflowId,
299};
300
301/// Nexus-endpoint bootstrap target used by the dev startup path. Kept in the
302/// library so both the CLI wrapper and the `start_in_memory` facade build the
303/// same initial registry shape.
304#[allow(dead_code)]
305#[derive(Clone, Debug)]
306pub enum BootstrapNexusEndpointTarget {
307    External {
308        address: String,
309    },
310    Worker {
311        namespace_name: String,
312        task_queue: String,
313    },
314}
315
316#[derive(Debug)]
317struct VisibilityRegistryOperatorApi<V> {
318    inner: InMemoryOperatorApi,
319    visibility_store: V,
320    declared_search_attributes: Vec<(String, SearchAttrType)>,
321}
322
323impl<V> VisibilityRegistryOperatorApi<V> {
324    fn new(inner: InMemoryOperatorApi, visibility_store: V) -> Self {
325        Self {
326            inner,
327            visibility_store,
328            declared_search_attributes: Vec::new(),
329        }
330    }
331
332    fn with_declared_search_attributes(mut self, declared: Vec<(String, SearchAttrType)>) -> Self {
333        self.declared_search_attributes = declared;
334        self
335    }
336}
337
338#[async_trait]
339impl<V> OperatorApi for VisibilityRegistryOperatorApi<V>
340where
341    V: VisibilityStore + Clone + 'static,
342{
343    async fn cluster_info(&self) -> Result<ClusterInfo> {
344        self.inner.cluster_info().await
345    }
346
347    async fn list_search_attributes(
348        &self,
349        namespace: Option<&str>,
350    ) -> Result<Vec<SearchAttributeDefinition>> {
351        self.inner.list_search_attributes(namespace).await
352    }
353
354    async fn upsert_search_attribute(
355        &self,
356        namespace: &str,
357        attr: SearchAttributeDefinition,
358    ) -> Result<()> {
359        let attr_type = visibility_search_attr_type(&attr.attr_type)?;
360        // OperatorService mutates the user-visible catalog, but projection is
361        // the authority that makes a field queryable. Register first so a
362        // successful AddSearchAttributes response cannot expose an attribute
363        // that the visibility compiler still rejects.
364        self.visibility_store
365            .register_attr(namespace_id_for(namespace), attr.name.clone(), attr_type)
366            .await?;
367        self.inner.upsert_search_attribute(namespace, attr).await
368    }
369
370    async fn remove_search_attribute(&self, namespace: &str, attr_name: &str) -> Result<()> {
371        self.inner
372            .remove_search_attribute(namespace, attr_name)
373            .await
374    }
375
376    /// Register the predefined search-attribute set into the visibility store for
377    /// this namespace (idempotent upserts), so visibility queries in a runtime-
378    /// created namespace resolve predefined fields exactly as the bootstrapped
379    /// `default` namespace does. Routed to the store, not `self.inner`, so the
380    /// predefined set stays out of the user-visible catalog.
381    async fn seed_predefined_search_attributes(&self, namespace: &str) -> Result<()> {
382        tokeira_projection::seed_predefined_search_attributes(
383            &self.visibility_store,
384            namespace_id_for(namespace),
385        )
386        .await?;
387        // Resolve before register: the projection's idempotent registration does
388        // not reject an existing key of another type. New namespaces get the same
389        // declarations as namespaces present at startup.
390        seed_declared_search_attributes(
391            &self.visibility_store,
392            namespace_id_for(namespace),
393            &self.declared_search_attributes,
394        )
395        .await?;
396        Ok(())
397    }
398}
399
400/// [`NexusNamespaceResolver`] backed by the edge namespace cache.
401///
402/// The runtime publisher needs a namespace's *name* where it holds only its
403/// [`NamespaceId`]: to tag the External-endpoint outbound Nexus metric with the
404/// originator's namespace, and to name the target namespace in the event that resolves an
405/// external signal or cancel request whose command named none. tokeira namespace ids are a
406/// non-invertible function of the name, so resolution scans the (small) registered set and
407/// matches `namespace_id_for(name)` — the inverse the runtime needs. Returns `None` when no
408/// registered namespace hashes to the id, leaving the metric unrecorded rather than
409/// mistagged, and the resolving event's name empty.
410struct CacheNexusNamespaceResolver {
411    namespaces: Arc<dyn NamespaceCache>,
412}
413
414#[async_trait]
415impl NexusNamespaceResolver for CacheNexusNamespaceResolver {
416    async fn name_for_id(&self, namespace_id: NamespaceId) -> Result<Option<String>> {
417        Ok(self
418            .namespaces
419            .list_all()
420            .await?
421            .into_iter()
422            .map(|namespace| namespace.name)
423            .find(|name| namespace_id_for(name) == namespace_id))
424    }
425}
426
427fn visibility_search_attr_type(value: &str) -> Result<SearchAttrType> {
428    match value {
429        "keyword" => Ok(SearchAttrType::Keyword),
430        "keyword_list" => Ok(SearchAttrType::KeywordList),
431        "int" => Ok(SearchAttrType::Int),
432        "bool" => Ok(SearchAttrType::Bool),
433        "double" => Ok(SearchAttrType::Double),
434        "datetime" => Ok(SearchAttrType::Datetime),
435        "text" => Ok(SearchAttrType::Text),
436        other => Err(anyhow!("unsupported search attribute type `{other}`")),
437    }
438}
439
440/// Companion to [`BootstrapNexusEndpointTarget`] carrying per-endpoint config
441/// during bootstrap.
442#[allow(dead_code)]
443#[derive(Clone, Debug)]
444pub struct BootstrapNexusEndpointConfig {
445    pub target: BootstrapNexusEndpointTarget,
446}
447
448/// A running Tokeira engine that binds no listener at startup.
449///
450/// The engine owns the authoritative store, runtime workers, and edge
451/// services. [`Self::endpoint`] is cloneable and can be adapted directly to the
452/// Temporal Rust SDK's callback transport; [`Self::listen`] optionally serves the
453/// same services on a host-chosen TCP address. Dropping the engine cancels
454/// background work, stops every attached listener, and makes every endpoint
455/// clone reject new calls. When snapshot policy is configured, call
456/// [`Self::shutdown`] for the final graceful-shutdown snapshot; `Drop` cannot
457/// perform asynchronous file I/O.
458#[derive(Debug)]
459pub struct Engine {
460    // Shared bootstrap state exists in every deployment; typed access is opt-in.
461    #[cfg_attr(not(feature = "chasm-extensions"), allow(dead_code))]
462    component_visibility_store: chasm_extensions::VisibilityQueryStore,
463    // Owned in every deployment; typed access is an opt-in public surface.
464    #[cfg_attr(not(feature = "chasm-extensions"), allow(dead_code))]
465    chasm_engine: Arc<tokeira_runtime::chasm::ChasmEngine>,
466    #[cfg_attr(not(feature = "chasm-extensions"), allow(dead_code))]
467    registry: Arc<tokeira_chasm::Registry>,
468    endpoint: TemporalEndpoint,
469    background_cancel: CancellationToken,
470    log_broadcast: broadcast::Sender<LogEvent>,
471    snapshot_policy: Option<EngineSnapshotPolicy>,
472    recovery_task: Option<JoinHandle<Result<()>>>,
473    startup_report: EngineStartupReport,
474    shutdown_coordinator: Option<EmbeddedShutdownCoordinator>,
475    listeners: listener::ListenerRegistry,
476}
477
478/// Storage mode selected by the explicit embedded startup boundary.
479#[derive(Clone, Copy, Debug, PartialEq, Eq)]
480pub enum EmbeddedStorageMode {
481    /// Ephemeral process-local authoritative storage.
482    InMemory,
483    /// Dedicated cluster created or recovered by the managed lifecycle.
484    ManagedDsql,
485    /// Canonical operator-supplied DSQL identity.
486    ExistingDsql,
487}
488
489/// Complete redacted report returned by a successful embedded startup.
490#[derive(Clone, Debug, PartialEq, Eq)]
491pub struct EngineStartupReport {
492    /// Explicit storage mode that reached admission-open.
493    pub storage_mode: EmbeddedStorageMode,
494    /// Canonical cluster observation for a durable mode.
495    pub cluster: Option<ClusterStartupReport>,
496    /// Release-pinned schema outcome for a durable mode.
497    pub schema: Option<SchemaStartupReport>,
498    /// Exclusive embedded ownership outcome for a durable mode.
499    pub ownership: Option<OwnershipStartupReport>,
500}
501
502impl EngineStartupReport {
503    fn in_memory() -> Self {
504        Self {
505            storage_mode: EmbeddedStorageMode::InMemory,
506            cluster: None,
507            schema: None,
508            ownership: None,
509        }
510    }
511}
512
513/// Canonical cluster fields safe for host diagnostics.
514#[derive(Clone, Debug, PartialEq, Eq)]
515pub struct ClusterStartupReport {
516    /// AWS Region used by lifecycle and IAM signing.
517    pub region: String,
518    /// Canonical AWS DSQL cluster ID.
519    pub cluster_id: String,
520    /// Canonical AWS DSQL cluster ARN.
521    pub cluster_arn: String,
522    /// Locator supplied to this generation's database connection factory.
523    ///
524    /// Existing mode uses the configured endpoint; managed mode uses the AWS
525    /// endpoint observed before pool warmup. Later wake observations do not
526    /// retarget the pool or change this report field. Never used as identity.
527    pub endpoint: String,
528    /// Whether startup created, recovered, or validated the cluster.
529    pub action: ClusterAction,
530}
531
532/// Schema migration work completed during startup.
533#[derive(Clone, Copy, Debug, PartialEq, Eq)]
534pub enum SchemaStartupOutcome {
535    /// The database already satisfied the release contract.
536    Compatible,
537    /// A previously empty database received the complete Tokeira schema.
538    Initialized,
539    /// A verified older schema advanced to the target version.
540    Migrated,
541    /// Legacy compatibility metadata was backfilled without schema DDL.
542    MetadataBackfilled,
543}
544
545/// Redacted release/schema compatibility evidence.
546#[derive(Clone, Debug, PartialEq, Eq)]
547pub struct SchemaStartupReport {
548    /// Highest observed migration version after policy application.
549    pub observed_version: u32,
550    /// Oldest schema version this release may read.
551    pub minimum_supported_version: u32,
552    /// Automatic migration target for this release.
553    pub target_version: u32,
554    /// Newest schema version this release may read.
555    pub maximum_readable_version: u32,
556    /// Release-pinned migration-set digest; never migration SQL.
557    pub migration_set_digest: String,
558    /// Startup migration outcome.
559    pub outcome: SchemaStartupOutcome,
560}
561
562/// Exclusive ownership acquisition evidence.
563#[derive(Clone, Copy, Debug, PartialEq, Eq)]
564pub struct OwnershipStartupReport {
565    /// Whether the claim was clean or replaced an expired owner.
566    pub outcome: ControlLeaseAcquireOutcome,
567    /// Database-issued monotonic fence token.
568    pub fence_token: i64,
569}
570
571/// Ordered startup phase used by typed, redacted failures.
572#[derive(Clone, Copy, Debug, PartialEq, Eq)]
573pub enum EmbeddedStartupPhase {
574    /// Validate explicit storage intent and ordinary server policy.
575    Configuration,
576    /// Create, recover, or validate the canonical cluster identity.
577    ClusterResolution,
578    /// Establish and warm the bounded process-local DSQL pool.
579    ConnectionWarmup,
580    /// Assess and apply the release-pinned schema policy.
581    Schema,
582    /// Acquire the exclusive embedded owner claim.
583    Ownership,
584    /// Reconstruct runtime state and acquire self-assigned shard leases.
585    RuntimeRestore,
586}
587
588impl std::fmt::Display for EmbeddedStartupPhase {
589    fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
590        formatter.write_str(match self {
591            Self::Configuration => "configuration validation",
592            Self::ClusterResolution => "cluster resolution",
593            Self::ConnectionWarmup => "DSQL connection warmup",
594            Self::Schema => "schema compatibility",
595            Self::Ownership => "embedded ownership",
596            Self::RuntimeRestore => "runtime restoration",
597        })
598    }
599}
600
601/// Redacted failure from explicit embedded startup.
602#[derive(Debug)]
603pub enum EmbeddedEngineStartError {
604    /// A library or executor conflicts with the immutable registry.
605    Registry(tokeira_chasm::ChasmError),
606    /// Persisted executions require a library absent from this deployment.
607    UnregisteredArchetype {
608        /// Persisted root component type.
609        archetype_id: u32,
610        /// Stored executions requiring it.
611        executions: u64,
612    },
613    /// A declared search attribute conflicts with the namespace's registered type.
614    SearchAttributeType {
615        /// Namespace whose registry conflicts.
616        namespace: NamespaceId,
617        /// Declared attribute name.
618        key: String,
619        /// Type required by the library.
620        declared: SearchAttrType,
621        /// Type already registered in projection storage.
622        existing: SearchAttrType,
623    },
624    /// Startup rebuild isolated executions this deployment cannot serve.
625    UnserviceableOutbox {
626        /// Archetype of the first failing execution in scan order.
627        archetype_id: u32,
628        /// Missing data, handler or undecodable payload encountered first.
629        cause: tokeira_runtime::chasm::RebuildFailure,
630        /// Number of executions isolated during the pass.
631        executions: usize,
632    },
633    /// Configuration failed before any external resource was touched.
634    InvalidConfiguration(tokeira_config::EmbeddedConfigError),
635    /// One startup phase failed; the nested cause is deliberately discarded.
636    Phase {
637        /// Phase that did not complete.
638        phase: EmbeddedStartupPhase,
639    },
640    /// The one host-configured startup budget elapsed in the named phase.
641    DeadlineExceeded {
642        /// Phase active when the shared deadline elapsed.
643        phase: EmbeddedStartupPhase,
644    },
645}
646
647impl std::fmt::Display for EmbeddedEngineStartError {
648    fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
649        match self {
650            Self::Registry(error) => write!(
651                formatter,
652                "CHASM registration failed: {error}; correct the conflicting library or executor before restarting"
653            ),
654            Self::UnregisteredArchetype {
655                archetype_id,
656                executions,
657            } => write!(
658                formatter,
659                "CHASM archetype {archetype_id} is not registered but has {executions} stored executions; load its library before restarting"
660            ),
661            Self::SearchAttributeType {
662                namespace,
663                key,
664                declared,
665                existing,
666            } => write!(
667                formatter,
668                "namespace {} search attribute `{key}` declares {declared:?} but storage has {existing:?}; restore the matching declaration or correct the registered type before restarting",
669                namespace.0
670            ),
671            Self::UnserviceableOutbox {
672                archetype_id,
673                cause,
674                executions,
675            } => write!(
676                formatter,
677                "CHASM rebuild cannot serve {executions} executions; first failure in archetype {archetype_id}: {cause}; restore the matching library or repair its persisted data before restarting"
678            ),
679            Self::InvalidConfiguration(error) => {
680                write!(formatter, "invalid embedded engine configuration: {error}")
681            }
682            Self::Phase { phase } => {
683                write!(formatter, "embedded engine startup failed during {phase}")
684            }
685            Self::DeadlineExceeded { phase } => {
686                write!(
687                    formatter,
688                    "embedded engine startup deadline exceeded during {phase}"
689                )
690            }
691        }
692    }
693}
694
695impl std::error::Error for EmbeddedEngineStartError {}
696
697impl From<tokeira_config::EmbeddedConfigError> for EmbeddedEngineStartError {
698    fn from(error: tokeira_config::EmbeddedConfigError) -> Self {
699        Self::InvalidConfiguration(error)
700    }
701}
702
703/// Independent cleanup stage that failed during explicit shutdown.
704#[derive(Clone, Copy, Debug, PartialEq, Eq)]
705pub enum EmbeddedShutdownFailure {
706    /// In-process handlers did not drain by the deadline.
707    RpcDrain,
708    /// Runtime or engine-owned tasks did not join by the deadline.
709    RuntimeTasks,
710    /// A self-assigned shard lease could not be relinquished.
711    ShardRelease,
712    /// The conditional embedded-owner release failed.
713    OwnershipRelease,
714    /// Embedded DSQL pool drain/closure failed.
715    Storage,
716    /// Final in-memory snapshot persistence failed.
717    Snapshot,
718    /// An attached listener did not stop and join by the deadline.
719    ListenerDrain,
720}
721
722/// Aggregated explicit-shutdown failure.
723#[derive(Debug)]
724pub struct EmbeddedEngineShutdownError {
725    failures: Vec<EmbeddedShutdownFailure>,
726}
727
728impl std::fmt::Display for EmbeddedEngineShutdownError {
729    fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
730        write!(
731            formatter,
732            "embedded engine shutdown completed with {} cleanup failure(s)",
733            self.failures.len()
734        )
735    }
736}
737
738impl std::error::Error for EmbeddedEngineShutdownError {}
739
740impl EmbeddedEngineShutdownError {
741    /// Ordered cleanup stages that failed; later cleanup was still attempted.
742    pub fn failures(&self) -> &[EmbeddedShutdownFailure] {
743        &self.failures
744    }
745}
746
747type CleanupFuture = Pin<Box<dyn Future<Output = Result<()>> + Send>>;
748type CleanupAction = Box<dyn FnOnce(Instant) -> CleanupFuture + Send>;
749
750#[derive(Clone)]
751struct EmbeddedOwnership {
752    repository: ConnectionControlLeaseRepository,
753    guard: Arc<tokio::sync::Mutex<Option<ControlLeaseGuard>>>,
754    gate: OwnershipAdmissionGate,
755}
756
757impl std::fmt::Debug for EmbeddedOwnership {
758    fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
759        formatter
760            .debug_struct("EmbeddedOwnership")
761            .field("gate", &self.gate)
762            .finish_non_exhaustive()
763    }
764}
765
766struct EmbeddedShutdownCoordinator {
767    service: InProcessGrpcService,
768    runtime_tasks: RuntimeShutdownHandle,
769    engine_tasks: RuntimeShutdownHandle,
770    shard_cleanup: Option<CleanupAction>,
771    director: Option<Arc<DsqlConnectionDirector>>,
772    ownership: Option<EmbeddedOwnership>,
773}
774
775impl std::fmt::Debug for EmbeddedShutdownCoordinator {
776    fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
777        formatter
778            .debug_struct("EmbeddedShutdownCoordinator")
779            .field("service", &self.service)
780            .field("runtime_tasks", &self.runtime_tasks)
781            .field("engine_tasks", &self.engine_tasks)
782            .field("has_shard_cleanup", &self.shard_cleanup.is_some())
783            .field("has_director", &self.director.is_some())
784            .field("ownership", &self.ownership)
785            .finish()
786    }
787}
788
789#[derive(Debug)]
790struct EngineSnapshotPolicy {
791    store: InMemoryStore,
792    location: PathBuf,
793    periodic_task: Option<JoinHandle<()>>,
794}
795
796static SNAPSHOT_TEMP_SEQUENCE: AtomicU64 = AtomicU64::new(0);
797
798impl EngineSnapshotPolicy {
799    fn start(
800        store: InMemoryStore,
801        config: SnapshotPolicyConfig,
802        cancel: CancellationToken,
803    ) -> Self {
804        let interval_duration = std::time::Duration::from_millis(config.interval_ms);
805        let location = config.location;
806        let periodic_store = store.clone();
807        let periodic_location = location.clone();
808        // Anchor the cadence before spawning. A busy executor may not poll the
809        // task immediately, but elapsed startup time still counts toward the
810        // operator-configured interval.
811        let first_tick = tokio::time::Instant::now() + interval_duration;
812        let periodic_task = tokio::spawn(async move {
813            // The first capture belongs one complete interval after startup. An
814            // immediate tick would turn "every N milliseconds" into an
815            // undocumented snapshot-at-boot policy and add I/O before any state
816            // could have changed.
817            let mut interval = tokio::time::interval_at(first_tick, interval_duration);
818            interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
819            loop {
820                tokio::select! {
821                    biased;
822                    _ = cancel.cancelled() => break,
823                    _ = interval.tick() => {
824                        if let Err(error) = persist_snapshot(&periodic_store, &periodic_location).await {
825                            error!(
826                                snapshot_path = %periodic_location.display(),
827                                error = %error,
828                                "failed to persist periodic engine snapshot"
829                            );
830                        }
831                    }
832                }
833            }
834        });
835        Self {
836            store,
837            location,
838            periodic_task: Some(periodic_task),
839        }
840    }
841}
842
843/// Cloneable raw-protobuf endpoint for an embedded [`Engine`].
844///
845/// This is intentionally SDK-neutral: its request and response are the same wire
846/// fields carried by `temporalio-client::callback_based::GrpcRequest` and
847/// `GrpcSuccessResponse`, but the core endpoint does not expose the SDK's tonic
848/// generation in its public types.
849#[derive(Clone, Debug)]
850pub struct TemporalEndpoint {
851    service: InProcessGrpcService,
852    shutdown: CancellationToken,
853}
854
855impl Engine {
856    /// Discover projected executions for a registered root in one namespace.
857    ///
858    /// The embedder must authorize namespace access before obtaining this handle;
859    /// it bypasses the Temporal transport and its interceptors. Results are
860    /// eventually consistent. Use [`Self::chasm`] for authoritative state/commands.
861    #[cfg(feature = "chasm-extensions")]
862    pub fn chasm_visibility<C>(
863        &self,
864        namespace_id: tokeira_types::NamespaceId,
865    ) -> Result<tokeira_projection::ComponentVisibility, tokeira_chasm::ChasmError>
866    where
867        C: tokeira_chasm::EngineComponent
868            + tokeira_chasm::RootComponent
869            + tokeira_chasm::SearchAttributeProvider
870            + tokeira_chasm::VisibilityContributor,
871    {
872        let archetype_id = self.registry.archetype_id(C::FQN).ok_or_else(|| {
873            tokeira_chasm::ChasmError::Internal(format!("archetype `{}` is not registered", C::FQN))
874        })?;
875        Ok(tokeira_projection::ComponentVisibility::new(
876            self.component_visibility_store.0.clone(),
877            namespace_id,
878            tokeira_types::ArchetypeId(archetype_id),
879        ))
880    }
881    /// Configure unstable CHASM libraries, executors and time before startup.
882    #[cfg(feature = "chasm-extensions")]
883    pub fn builder(config: EmbeddedEngineConfig) -> EngineBuilder {
884        EngineBuilder {
885            config,
886            extensions: ChasmExtensions::default(),
887        }
888    }
889
890    /// Return a typed handle only for a registered component. The built-in
891    /// activity archetype is always reachable, including with no extension libraries.
892    #[cfg(feature = "chasm-extensions")]
893    pub fn chasm<C>(
894        &self,
895    ) -> Result<tokeira_runtime::chasm::TypedEngine<C>, tokeira_chasm::ChasmError>
896    where
897        C: tokeira_chasm::EngineComponent
898            + tokeira_chasm::RootComponent
899            + tokeira_chasm::SearchAttributeProvider
900            + tokeira_chasm::VisibilityContributor,
901    {
902        if self.registry.archetype_id(C::FQN).is_none() {
903            return Err(tokeira_chasm::ChasmError::Internal(format!(
904                "archetype `{}` is not registered",
905                C::FQN
906            )));
907        }
908        Ok(tokeira_runtime::chasm::TypedEngine::new(
909            self.chasm_engine.clone(),
910        ))
911    }
912
913    /// Start a zero-listener engine with the default in-memory configuration.
914    pub async fn start() -> Result<Self> {
915        Self::start_with_config(TokeiraConfig::default()).await
916    }
917
918    /// Alias for [`Self::start`] that makes the operating mode explicit at call sites.
919    pub async fn embedded() -> Result<Self> {
920        Self::start().await
921    }
922
923    /// Start a zero-listener engine with an explicit validated configuration.
924    ///
925    /// Storage is always forced to in-memory: durable external storage and cluster
926    /// placement are daemon concerns, while this facade guarantees process-local
927    /// ownership and no bound sockets.
928    pub async fn start_with_config(config: TokeiraConfig) -> Result<Self> {
929        Self::start_in_memory_with_extensions(config, ChasmExtensions::default()).await
930    }
931
932    async fn start_in_memory_with_extensions(
933        config: TokeiraConfig,
934        extensions: ChasmExtensions,
935    ) -> Result<Self> {
936        let config = embedded_config(config)?;
937        let snapshot_config = config.policy.snapshot.clone();
938        let (store, restored) = restore_snapshot_store(&config).await?;
939        let stack = build_embedded(Arc::new(config), store.clone(), restored, extensions).await?;
940        let snapshot_policy = snapshot_config.map(|config| {
941            EngineSnapshotPolicy::start(store, config, stack.background_cancel.clone())
942        });
943        stack.engine_tasks.close_registration();
944        let shutdown_coordinator = EmbeddedShutdownCoordinator {
945            service: stack.service.clone(),
946            runtime_tasks: stack.runtime_tasks,
947            engine_tasks: stack.engine_tasks,
948            shard_cleanup: stack.shard_cleanup,
949            director: None,
950            ownership: None,
951        };
952        let engine = Self {
953            component_visibility_store: stack.component_visibility_store,
954            chasm_engine: stack.chasm_engine,
955            registry: stack.registry,
956            endpoint: TemporalEndpoint {
957                service: stack.service,
958                shutdown: stack.background_cancel.clone(),
959            },
960            background_cancel: stack.background_cancel,
961            log_broadcast: stack.log_broadcast,
962            snapshot_policy,
963            recovery_task: stack.recovery_task,
964            startup_report: EngineStartupReport::in_memory(),
965            shutdown_coordinator: Some(shutdown_coordinator),
966            listeners: listener::ListenerRegistry::default(),
967        };
968        record_embedded_startup(&engine.startup_report);
969        Ok(engine)
970    }
971
972    /// Start a zero-listener engine from an explicit embedded storage decision.
973    ///
974    /// The legacy [`Self::start`] and [`Self::start_with_config`] methods remain
975    /// in-memory only. This is the sole engine boundary that may create or
976    /// recover managed Aurora DSQL infrastructure.
977    pub async fn start_with_embedded_config(
978        config: EmbeddedEngineConfig,
979    ) -> Result<Self, EmbeddedEngineStartError> {
980        Self::start_with_extensions(config, ChasmExtensions::default()).await
981    }
982
983    async fn start_with_extensions(
984        config: EmbeddedEngineConfig,
985        extensions: ChasmExtensions,
986    ) -> Result<Self, EmbeddedEngineStartError> {
987        if !matches!(config.storage, EmbeddedStorageConfig::InMemory)
988            && config.server.policy.snapshot.is_some()
989        {
990            return Err(EmbeddedEngineStartError::Phase {
991                phase: EmbeddedStartupPhase::Configuration,
992            });
993        }
994        config.validate()?;
995        if matches!(config.storage, EmbeddedStorageConfig::InMemory) {
996            return Self::start_in_memory_with_extensions(config.server, extensions)
997                .await
998                .map_err(embedded_stack_error);
999        }
1000        start_embedded_dsql(config, extensions).await
1001    }
1002
1003    /// Return the complete redacted report for the startup that produced this engine.
1004    pub fn startup_report(&self) -> &EngineStartupReport {
1005        &self.startup_report
1006    }
1007
1008    /// Return a cloneable raw-protobuf endpoint for this engine.
1009    pub fn endpoint(&self) -> TemporalEndpoint {
1010        self.endpoint.clone()
1011    }
1012
1013    /// Subscribe to the same narrow RPC-event stream exposed by [`TokeiradHandle`].
1014    pub fn log_sink(&self) -> broadcast::Receiver<LogEvent> {
1015        self.log_broadcast.subscribe()
1016    }
1017
1018    /// Adapt this engine to `ConnectionOptions::service_override`.
1019    ///
1020    /// DNS load balancing must be disabled on the SDK connection because the
1021    /// override performs no name resolution or network I/O.
1022    #[cfg(feature = "temporalio-client")]
1023    pub fn service_override(&self) -> temporalio_client::callback_based::CallbackBasedGrpcService {
1024        self.endpoint.service_override()
1025    }
1026
1027    /// Stop accepting calls, cancel background work, and persist the final snapshot.
1028    ///
1029    /// Attached listeners are stopped and joined first, after in-process
1030    /// admission has closed and the runtime has been signalled but before the
1031    /// in-process drain: from then on no network handler can be admitted, so
1032    /// the coordinator's drain, task join, lease release, ownership release,
1033    /// and storage close run in the same world they did before listeners
1034    /// existed. A listener that does not join by the deadline is reported as
1035    /// [`EmbeddedShutdownFailure::ListenerDrain`] and shutdown continues.
1036    ///
1037    /// When snapshot policy is disabled this retains the previous cancellation-only
1038    /// behaviour. A configured final snapshot failure is returned to the caller: a
1039    /// graceful shutdown must not silently claim durability it did not achieve.
1040    pub async fn shutdown(mut self) -> Result<(), EmbeddedEngineShutdownError> {
1041        let deadline = Instant::now() + StdDuration::from_secs(30);
1042        let mut failures = Vec::new();
1043        if let Some(mut coordinator) = self.shutdown_coordinator.take() {
1044            coordinator.begin_shutdown();
1045            self.listeners.stop_all(deadline, &mut failures).await;
1046            coordinator.shutdown(deadline, &mut failures).await;
1047        } else {
1048            self.listeners.stop_all(deadline, &mut failures).await;
1049            self.background_cancel.cancel();
1050        }
1051        if let Some(task) = self.recovery_task.take()
1052            && task.await.is_err()
1053        {
1054            failures.push(EmbeddedShutdownFailure::ShardRelease);
1055        }
1056        if let Some(policy) = self.snapshot_policy.take()
1057            && finish_snapshot_policy(policy).await.is_err()
1058        {
1059            failures.push(EmbeddedShutdownFailure::Snapshot);
1060        }
1061        record_embedded_shutdown(&self.startup_report, &failures);
1062        if failures.is_empty() {
1063            Ok(())
1064        } else {
1065            Err(EmbeddedEngineShutdownError { failures })
1066        }
1067    }
1068}
1069
1070impl EmbeddedShutdownCoordinator {
1071    fn begin_shutdown(&self) {
1072        self.service.begin_shutdown();
1073        self.engine_tasks.begin_shutdown();
1074        self.runtime_tasks.begin_shutdown();
1075    }
1076
1077    async fn shutdown(&mut self, deadline: Instant, failures: &mut Vec<EmbeddedShutdownFailure>) {
1078        if self.service.drain(deadline).await.is_err() {
1079            failures.push(EmbeddedShutdownFailure::RpcDrain);
1080        }
1081        if self.engine_tasks.wait(deadline).await.is_err() {
1082            failures.push(EmbeddedShutdownFailure::RuntimeTasks);
1083        }
1084        if self.runtime_tasks.wait(deadline).await.is_err() {
1085            failures.push(EmbeddedShutdownFailure::RuntimeTasks);
1086        }
1087        if let Some(cleanup) = self.shard_cleanup.take()
1088            && cleanup(deadline).await.is_err()
1089        {
1090            failures.push(EmbeddedShutdownFailure::ShardRelease);
1091        }
1092        // RPC admission is already closed and both task groups have joined, so
1093        // shard release is the last operation allowed through ordinary owner
1094        // admission. Closing earlier makes that conditional DSQL release unable
1095        // to acquire its bounded Control connection and also prevents already
1096        // admitted RPCs from draining gracefully.
1097        if let Some(ownership) = &self.ownership {
1098            ownership.gate.begin_closing();
1099        }
1100        if let (Some(ownership), Some(director)) = (&self.ownership, &self.director) {
1101            let now = Instant::now();
1102            let ownership_drained = now < deadline
1103                && tokio::time::timeout(
1104                    deadline.saturating_duration_since(now),
1105                    ownership.gate.wait_for_drain(),
1106                )
1107                .await
1108                .is_ok();
1109            if ownership_drained {
1110                let now = Instant::now();
1111                let release_result = if now < deadline {
1112                    tokio::time::timeout(
1113                        deadline.saturating_duration_since(now),
1114                        release_embedded_ownership(ownership, director),
1115                    )
1116                    .await
1117                    .map_err(|_| anyhow!("embedded ownership release deadline elapsed"))
1118                    .and_then(|result| result)
1119                } else {
1120                    Err(anyhow!("embedded ownership release deadline elapsed"))
1121                };
1122                if release_result.is_err() {
1123                    failures.push(EmbeddedShutdownFailure::OwnershipRelease);
1124                }
1125            } else {
1126                // Releasing while a checkout still uses the claim would let a
1127                // clean takeover overlap prior database work. Leave the claim
1128                // to expire instead and report the incomplete cleanup.
1129                failures.push(EmbeddedShutdownFailure::OwnershipRelease);
1130            }
1131        }
1132        if let Some(director) = &self.director
1133            && director.shutdown_with_deadline(deadline).await.is_err()
1134        {
1135            failures.push(EmbeddedShutdownFailure::Storage);
1136        }
1137    }
1138}
1139
1140async fn start_embedded_dsql(
1141    config: EmbeddedEngineConfig,
1142    extensions: ChasmExtensions,
1143) -> Result<Engine, EmbeddedEngineStartError> {
1144    let deadline = Instant::now() + StdDuration::from_millis(config.startup_timeout_ms);
1145    let storage_mode = match &config.storage {
1146        EmbeddedStorageConfig::ManagedDsql(_) => EmbeddedStorageMode::ManagedDsql,
1147        EmbeddedStorageConfig::ExistingDsql(_) => EmbeddedStorageMode::ExistingDsql,
1148        EmbeddedStorageConfig::InMemory => unreachable!("in-memory handled by caller"),
1149    };
1150    let region = match &config.storage {
1151        EmbeddedStorageConfig::ManagedDsql(managed) => managed.region.clone(),
1152        EmbeddedStorageConfig::ExistingDsql(existing) => existing.region.clone(),
1153        EmbeddedStorageConfig::InMemory => unreachable!("in-memory handled by caller"),
1154    };
1155    let control = startup_infallible_phase(
1156        deadline,
1157        EmbeddedStartupPhase::ClusterResolution,
1158        AwsDsqlControlPlane::from_region(region.clone()),
1159    )
1160    .await?;
1161    let migration_policy = config
1162        .effective_migration_policy()
1163        .expect("durable embedded mode always has a migration policy");
1164    let storage = config.storage.clone();
1165    let ConnectedEmbeddedDsql {
1166        server,
1167        cluster,
1168        connection_endpoint,
1169        store: dsql_store,
1170        wake_required,
1171    } = connect_embedded_dsql(
1172        config,
1173        control.clone(),
1174        SystemLifecycleEnvironment,
1175        deadline,
1176        DsqlStore::connect_embedded,
1177    )
1178    .await?;
1179    let lifecycle_deadline = StartupDeadline::at(deadline);
1180    let director = dsql_store.connection_director_arc();
1181
1182    let cluster = if wake_required {
1183        match refresh_cluster_after_wake(
1184            &storage,
1185            control,
1186            SystemLifecycleEnvironment,
1187            cluster,
1188            lifecycle_deadline,
1189            deadline,
1190        )
1191        .await
1192        {
1193            Ok(cluster) => cluster,
1194            Err(error) => {
1195                let _ = director.shutdown_with_deadline(deadline).await;
1196                return Err(error);
1197            }
1198        }
1199    } else {
1200        cluster
1201    };
1202
1203    let schema = match startup_phase(
1204        deadline,
1205        EmbeddedStartupPhase::Schema,
1206        apply_embedded_schema(
1207            director.clone(),
1208            &cluster,
1209            schema_migration_policy(migration_policy),
1210            deadline,
1211        ),
1212    )
1213    .await
1214    {
1215        Ok(report) => report,
1216        Err(error) => {
1217            let _ = director.shutdown_with_deadline(deadline).await;
1218            return Err(error);
1219        }
1220    };
1221
1222    let (ownership, ownership_report) = match startup_phase(
1223        deadline,
1224        EmbeddedStartupPhase::Ownership,
1225        acquire_embedded_ownership(director.clone(), &cluster, deadline),
1226    )
1227    .await
1228    {
1229        Ok(ownership) => ownership,
1230        Err(error) => {
1231            let _ = director.shutdown_with_deadline(deadline).await;
1232            return Err(error);
1233        }
1234    };
1235    if director
1236        .install_ownership_gate(ownership.gate.clone())
1237        .is_err()
1238    {
1239        rollback_embedded_dsql(&ownership, &director, deadline).await;
1240        return Err(EmbeddedEngineStartError::Phase {
1241            phase: EmbeddedStartupPhase::Ownership,
1242        });
1243    }
1244
1245    let stack =
1246        match startup_infallible_phase(deadline, EmbeddedStartupPhase::RuntimeRestore, async {
1247            build_dsql_stack(
1248                StackTransport::Embedded,
1249                Arc::new(server),
1250                dsql_store,
1251                connection_endpoint.clone(),
1252                extensions,
1253                None,
1254            )
1255            .await
1256            .map_err(embedded_stack_error)
1257        })
1258        .await
1259        .and_then(|result| result)
1260        {
1261            Ok(ConstructedStack::Embedded(stack)) => stack,
1262            Ok(ConstructedStack::Network(_)) => {
1263                rollback_embedded_dsql(&ownership, &director, deadline).await;
1264                return Err(EmbeddedEngineStartError::Phase {
1265                    phase: EmbeddedStartupPhase::RuntimeRestore,
1266                });
1267            }
1268            Err(error) => {
1269                rollback_embedded_dsql(&ownership, &director, deadline).await;
1270                return Err(error);
1271            }
1272        };
1273
1274    spawn_ownership_renewal(
1275        &stack.engine_tasks,
1276        director.clone(),
1277        ownership.clone(),
1278        stack.service.clone(),
1279    );
1280    stack.engine_tasks.close_registration();
1281    let report = EngineStartupReport {
1282        storage_mode,
1283        cluster: Some(cluster_startup_report(&cluster, &connection_endpoint)),
1284        schema: Some(schema),
1285        ownership: Some(ownership_report),
1286    };
1287    let coordinator = EmbeddedShutdownCoordinator {
1288        service: stack.service.clone(),
1289        runtime_tasks: stack.runtime_tasks,
1290        engine_tasks: stack.engine_tasks,
1291        shard_cleanup: stack.shard_cleanup,
1292        director: Some(director),
1293        ownership: Some(ownership),
1294    };
1295    let engine = Engine {
1296        component_visibility_store: stack.component_visibility_store,
1297        chasm_engine: stack.chasm_engine,
1298        registry: stack.registry,
1299        endpoint: TemporalEndpoint {
1300            service: stack.service,
1301            shutdown: stack.background_cancel.clone(),
1302        },
1303        background_cancel: stack.background_cancel,
1304        log_broadcast: stack.log_broadcast,
1305        snapshot_policy: None,
1306        recovery_task: stack.recovery_task,
1307        startup_report: report,
1308        shutdown_coordinator: Some(coordinator),
1309        listeners: listener::ListenerRegistry::default(),
1310    };
1311    record_embedded_startup(&engine.startup_report);
1312    Ok(engine)
1313}
1314
1315// Keep the AWS observation separate from the locator captured by the pool's
1316// connection factory. Readiness and wake polling may refresh that observation;
1317// they cannot retarget connections already established by this generation.
1318#[derive(Debug)]
1319struct ConnectedEmbeddedDsql<S> {
1320    server: TokeiraConfig,
1321    cluster: ResolvedCluster,
1322    connection_endpoint: String,
1323    store: S,
1324    wake_required: bool,
1325}
1326
1327// The injected control plane, clock, and connector exercise the actual startup
1328// wiring offline without replacing identity validation or readiness decisions.
1329async fn connect_embedded_dsql<C, T, F, Fut, S>(
1330    config: EmbeddedEngineConfig,
1331    control: C,
1332    time: T,
1333    deadline: Instant,
1334    connect: F,
1335) -> Result<ConnectedEmbeddedDsql<S>, EmbeddedEngineStartError>
1336where
1337    C: DsqlControlPlane + Clone,
1338    T: LifecycleEnvironment + Clone,
1339    F: FnOnce(DsqlAuthConfig, EmbeddedDsqlPoolConfig, WarmupDeadline) -> Fut,
1340    Fut: Future<Output = Result<S>>,
1341{
1342    let lifecycle_deadline = StartupDeadline::at(deadline);
1343    let resolved = match &config.storage {
1344        EmbeddedStorageConfig::ManagedDsql(managed) => {
1345            let lifecycle = ManagedDsqlLifecycle::new(
1346                control.clone(),
1347                LocalClusterDescriptorStore::new(&managed.descriptor_path),
1348                time.clone(),
1349            );
1350            startup_phase(
1351                deadline,
1352                EmbeddedStartupPhase::ClusterResolution,
1353                lifecycle.create_or_recover(
1354                    CreateOrRecoverRequest {
1355                        region: managed.region.clone(),
1356                        tags: managed.tags.clone(),
1357                    },
1358                    lifecycle_deadline,
1359                ),
1360            )
1361            .await?
1362        }
1363        EmbeddedStorageConfig::ExistingDsql(existing) => {
1364            // `resolve_existing` cannot touch the descriptor seam. The inert
1365            // path is never opened and exists only to satisfy the lifecycle's
1366            // generic state-store type without adding another public adapter.
1367            let unused_descriptor = LocalClusterDescriptorStore::new(PathBuf::new());
1368            let lifecycle =
1369                ManagedDsqlLifecycle::new(control.clone(), unused_descriptor, time.clone());
1370            let identity = CanonicalClusterIdentity::new(
1371                &existing.region,
1372                &existing.cluster_id,
1373                &existing.cluster_arn,
1374            )
1375            .map_err(|_| EmbeddedEngineStartError::Phase {
1376                phase: EmbeddedStartupPhase::ClusterResolution,
1377            })?;
1378            startup_phase(
1379                deadline,
1380                EmbeddedStartupPhase::ClusterResolution,
1381                lifecycle.resolve_existing(identity, lifecycle_deadline),
1382            )
1383            .await?
1384        }
1385        EmbeddedStorageConfig::InMemory => unreachable!("in-memory handled by caller"),
1386    };
1387    let readiness = refresh_cluster_until_storage_handoff(
1388        &config.storage,
1389        control.clone(),
1390        time,
1391        resolved,
1392        lifecycle_deadline,
1393        deadline,
1394    )
1395    .await?;
1396    let cluster = match &readiness {
1397        Readiness::Active(usable) => usable.resolved().clone(),
1398        Readiness::WakeRequired(cluster) => cluster.clone(),
1399    };
1400
1401    // GetCluster validates the canonical resource and status, but its public
1402    // endpoint cannot select the caller's network path. In particular, AWS's
1403    // PrivateLink database hostname is distinct from the management observation
1404    // (Aurora DSQL User Guide, "Managing and connecting ... using AWS PrivateLink").
1405    // Capture the locator once: the connector uses it for IAM signing, TLS, and
1406    // every replenished connection, including the initial scale-to-zero wake.
1407    let connection_endpoint = match &config.storage {
1408        EmbeddedStorageConfig::ExistingDsql(existing) => existing.endpoint.clone(),
1409        EmbeddedStorageConfig::ManagedDsql(_) => cluster.endpoint.clone(),
1410        EmbeddedStorageConfig::InMemory => unreachable!("in-memory handled by caller"),
1411    };
1412    let mut server = config.server;
1413    server.infrastructure.storage = ConfigStorageKind::Dsql;
1414    server.infrastructure.placement.controller_endpoint = None;
1415    server.infrastructure.dsql.endpoint = Some(connection_endpoint.clone());
1416    server.infrastructure.dsql.region = Some(cluster.identity.region.clone());
1417    server
1418        .validate()
1419        .map_err(|_| EmbeddedEngineStartError::Phase {
1420            phase: EmbeddedStartupPhase::Configuration,
1421        })?;
1422
1423    let limits = match &config.storage {
1424        EmbeddedStorageConfig::ManagedDsql(managed) => &managed.limits,
1425        EmbeddedStorageConfig::ExistingDsql(existing) => &existing.limits,
1426        EmbeddedStorageConfig::InMemory => unreachable!("in-memory handled by caller"),
1427    };
1428    let mut pool_config = EmbeddedDsqlPoolConfig::with_limits(
1429        limits.max_connections,
1430        limits.concurrent_connection_creations,
1431        limits.connection_rate_per_second,
1432        limits.connection_burst,
1433    );
1434    pool_config.shard_count = server.infrastructure.placement.shard_count;
1435    pool_config.projection_partition_count = server.infrastructure.placement.partition_count;
1436    let auth = DsqlAuthConfig {
1437        endpoint: connection_endpoint.clone(),
1438        region: Some(cluster.identity.region.clone()),
1439        admin_role_arn: server.infrastructure.dsql.admin_role_arn.clone(),
1440        runtime_role_arn: server.infrastructure.dsql.runtime_role_arn.clone(),
1441        readonly_role_arn: server.infrastructure.dsql.readonly_role_arn.clone(),
1442    };
1443    let store = startup_phase(
1444        deadline,
1445        EmbeddedStartupPhase::ConnectionWarmup,
1446        connect(auth, pool_config, WarmupDeadline::new(deadline)),
1447    )
1448    .await?;
1449    Ok(ConnectedEmbeddedDsql {
1450        server,
1451        cluster,
1452        connection_endpoint,
1453        store,
1454        wake_required: matches!(readiness, Readiness::WakeRequired(_)),
1455    })
1456}
1457
1458fn cluster_startup_report(
1459    cluster: &ResolvedCluster,
1460    connection_endpoint: &str,
1461) -> ClusterStartupReport {
1462    ClusterStartupReport {
1463        region: cluster.identity.region.clone(),
1464        cluster_id: cluster.identity.cluster_id.clone(),
1465        cluster_arn: cluster.identity.cluster_arn.clone(),
1466        endpoint: connection_endpoint.to_owned(),
1467        action: cluster.action,
1468    }
1469}
1470
1471fn storage_mode_label(mode: EmbeddedStorageMode) -> EmbeddedStorageModeLabel {
1472    match mode {
1473        EmbeddedStorageMode::InMemory => EmbeddedStorageModeLabel::InMemory,
1474        EmbeddedStorageMode::ManagedDsql => EmbeddedStorageModeLabel::ManagedDsql,
1475        EmbeddedStorageMode::ExistingDsql => EmbeddedStorageModeLabel::ExistingDsql,
1476    }
1477}
1478
1479fn schema_outcome_label(report: &EngineStartupReport) -> SchemaOutcomeLabel {
1480    match report.schema.as_ref().map(|schema| schema.outcome) {
1481        None => SchemaOutcomeLabel::NotApplicable,
1482        Some(SchemaStartupOutcome::Compatible) => SchemaOutcomeLabel::Compatible,
1483        Some(SchemaStartupOutcome::Initialized) => SchemaOutcomeLabel::Initialized,
1484        Some(SchemaStartupOutcome::Migrated) => SchemaOutcomeLabel::Migrated,
1485        Some(SchemaStartupOutcome::MetadataBackfilled) => SchemaOutcomeLabel::MetadataBackfilled,
1486    }
1487}
1488
1489fn ownership_outcome_label(report: &EngineStartupReport) -> OwnershipOutcomeLabel {
1490    match report.ownership.map(|ownership| ownership.outcome) {
1491        None => OwnershipOutcomeLabel::NotApplicable,
1492        Some(ControlLeaseAcquireOutcome::Clean) => OwnershipOutcomeLabel::AcquiredClean,
1493        Some(ControlLeaseAcquireOutcome::ExpiredTakeover) => OwnershipOutcomeLabel::AcquiredExpired,
1494    }
1495}
1496
1497/// Emit only bounded lifecycle dimensions through the host's current recorder.
1498///
1499/// This is deliberately downstream of successful construction. The metrics
1500/// facade is observational: it does not install a recorder, hold an exporter,
1501/// bind a listener, or participate in the startup decision.
1502fn record_embedded_startup(report: &EngineStartupReport) {
1503    record_embedded_lifecycle(
1504        storage_mode_label(report.storage_mode),
1505        if report.cluster.is_some() {
1506            ClusterStatusLabel::Active
1507        } else {
1508            ClusterStatusLabel::NotApplicable
1509        },
1510        schema_outcome_label(report),
1511        ownership_outcome_label(report),
1512        DbClassLabel::Control,
1513        EmbeddedOperationLabel::Startup,
1514        ErrorClassLabel::None,
1515    );
1516}
1517
1518fn record_embedded_shutdown(report: &EngineStartupReport, failures: &[EmbeddedShutdownFailure]) {
1519    let ownership = if report.ownership.is_some() && failures.is_empty() {
1520        OwnershipOutcomeLabel::Released
1521    } else {
1522        ownership_outcome_label(report)
1523    };
1524    record_embedded_lifecycle(
1525        storage_mode_label(report.storage_mode),
1526        if report.cluster.is_some() {
1527            ClusterStatusLabel::Active
1528        } else {
1529            ClusterStatusLabel::NotApplicable
1530        },
1531        schema_outcome_label(report),
1532        ownership,
1533        DbClassLabel::Control,
1534        EmbeddedOperationLabel::Shutdown,
1535        if failures.is_empty() {
1536            ErrorClassLabel::None
1537        } else {
1538            ErrorClassLabel::Internal
1539        },
1540    );
1541}
1542
1543async fn refresh_cluster_until_storage_handoff<C, T>(
1544    storage: &EmbeddedStorageConfig,
1545    control: C,
1546    time: T,
1547    cluster: ResolvedCluster,
1548    lifecycle_deadline: StartupDeadline,
1549    deadline: Instant,
1550) -> Result<Readiness, EmbeddedEngineStartError>
1551where
1552    C: DsqlControlPlane,
1553    T: LifecycleEnvironment,
1554{
1555    match storage {
1556        EmbeddedStorageConfig::ManagedDsql(managed) => {
1557            let lifecycle = ManagedDsqlLifecycle::new(
1558                control,
1559                LocalClusterDescriptorStore::new(&managed.descriptor_path),
1560                time,
1561            );
1562            startup_phase(
1563                deadline,
1564                EmbeddedStartupPhase::ClusterResolution,
1565                lifecycle.refresh_until_usable(cluster, lifecycle_deadline),
1566            )
1567            .await
1568        }
1569        EmbeddedStorageConfig::ExistingDsql(_) => {
1570            let lifecycle = ManagedDsqlLifecycle::new(
1571                control,
1572                LocalClusterDescriptorStore::new(PathBuf::new()),
1573                time,
1574            );
1575            startup_phase(
1576                deadline,
1577                EmbeddedStartupPhase::ClusterResolution,
1578                lifecycle.refresh_until_usable(cluster, lifecycle_deadline),
1579            )
1580            .await
1581        }
1582        EmbeddedStorageConfig::InMemory => unreachable!("in-memory handled by caller"),
1583    }
1584}
1585
1586async fn refresh_cluster_after_wake<C, T>(
1587    storage: &EmbeddedStorageConfig,
1588    control: C,
1589    time: T,
1590    cluster: ResolvedCluster,
1591    lifecycle_deadline: StartupDeadline,
1592    deadline: Instant,
1593) -> Result<ResolvedCluster, EmbeddedEngineStartError>
1594where
1595    C: DsqlControlPlane,
1596    T: LifecycleEnvironment,
1597{
1598    let usable = match storage {
1599        EmbeddedStorageConfig::ManagedDsql(managed) => {
1600            let lifecycle = ManagedDsqlLifecycle::new(
1601                control,
1602                LocalClusterDescriptorStore::new(&managed.descriptor_path),
1603                time,
1604            );
1605            startup_phase(
1606                deadline,
1607                EmbeddedStartupPhase::ConnectionWarmup,
1608                lifecycle.refresh_after_wake(cluster, lifecycle_deadline),
1609            )
1610            .await?
1611        }
1612        EmbeddedStorageConfig::ExistingDsql(_) => {
1613            let lifecycle = ManagedDsqlLifecycle::new(
1614                control,
1615                LocalClusterDescriptorStore::new(PathBuf::new()),
1616                time,
1617            );
1618            startup_phase(
1619                deadline,
1620                EmbeddedStartupPhase::ConnectionWarmup,
1621                lifecycle.refresh_after_wake(cluster, lifecycle_deadline),
1622            )
1623            .await?
1624        }
1625        EmbeddedStorageConfig::InMemory => unreachable!("in-memory handled by caller"),
1626    };
1627    Ok(usable.into_resolved())
1628}
1629
1630fn schema_migration_policy(policy: DsqlMigrationPolicy) -> SchemaMigrationPolicy {
1631    match policy {
1632        DsqlMigrationPolicy::Automatic => SchemaMigrationPolicy::Automatic,
1633        DsqlMigrationPolicy::ValidateOnly => SchemaMigrationPolicy::ValidateOnly,
1634    }
1635}
1636
1637async fn apply_embedded_schema(
1638    director: Arc<DsqlConnectionDirector>,
1639    cluster: &ResolvedCluster,
1640    policy: SchemaMigrationPolicy,
1641    deadline: Instant,
1642) -> Result<SchemaStartupReport> {
1643    let runner = MigrationRunner::embedded();
1644    let contract = MigrationRunner::compatibility_contract();
1645    let leases = ConnectionControlLeaseRepository::new();
1646    let mut permit = director.acquire(DbClass::Control).await?;
1647    let decision = runner
1648        .assess_connection(permit.connection()?, &contract, policy)
1649        .await?;
1650    let outcome = match &decision {
1651        SchemaDecision::Compatible {
1652            legacy_backfill: false,
1653            ..
1654        } => SchemaStartupOutcome::Compatible,
1655        SchemaDecision::Compatible {
1656            legacy_backfill: true,
1657            ..
1658        } => SchemaStartupOutcome::MetadataBackfilled,
1659        SchemaDecision::Initialize { .. } => SchemaStartupOutcome::Initialized,
1660        SchemaDecision::Migrate { .. } => SchemaStartupOutcome::Migrated,
1661        SchemaDecision::MigrationRequired { current, target } => {
1662            return Err(anyhow!(
1663                "schema migration required from V{current} to V{target}"
1664            ));
1665        }
1666        SchemaDecision::Reject(_) => return Err(anyhow!("schema compatibility rejected")),
1667    };
1668    if !matches!(outcome, SchemaStartupOutcome::Compatible) {
1669        runner
1670            .bootstrap_migration_coordination(permit.connection()?, &decision)
1671            .await?;
1672        leases.bootstrap(permit.connection()?).await?;
1673        let mut migration_guard = leases
1674            .acquire(
1675                permit.connection()?,
1676                &ControlLeaseAcquireRequest {
1677                    claim_name: "schema-migration".to_owned(),
1678                    cluster: control_lease_identity(cluster),
1679                    owner_id: format!("schema-{}", IncarnationId::new()),
1680                    lease_duration: SCHEMA_MIGRATION_LEASE_DURATION,
1681                    admission_margin: SCHEMA_MIGRATION_ADMISSION_MARGIN,
1682                    acquire_deadline: deadline,
1683                },
1684            )
1685            .await?;
1686        let migration_gate = OwnershipAdmissionGate::for_guard(&migration_guard);
1687        let apply_result = runner
1688            .apply_decision(
1689                permit.connection()?,
1690                &decision,
1691                &leases,
1692                &mut migration_guard,
1693                &migration_gate,
1694            )
1695            .await;
1696        let release_result = leases
1697            .release(permit.connection()?, &migration_guard, &migration_gate)
1698            .await;
1699        apply_result?;
1700        release_result?;
1701    }
1702    let final_decision = runner
1703        .assess_connection(
1704            permit.connection()?,
1705            &contract,
1706            SchemaMigrationPolicy::ValidateOnly,
1707        )
1708        .await?;
1709    let observed_version = match final_decision {
1710        SchemaDecision::Compatible { current, .. } => current,
1711        _ => return Err(anyhow!("schema did not converge to a readable version")),
1712    };
1713    Ok(SchemaStartupReport {
1714        observed_version,
1715        minimum_supported_version: contract.minimum_supported_version,
1716        target_version: contract.target_version,
1717        maximum_readable_version: contract.maximum_readable_version,
1718        migration_set_digest: contract.migration_set_digest,
1719        outcome,
1720    })
1721}
1722
1723async fn acquire_embedded_ownership(
1724    director: Arc<DsqlConnectionDirector>,
1725    cluster: &ResolvedCluster,
1726    deadline: Instant,
1727) -> Result<(EmbeddedOwnership, OwnershipStartupReport)> {
1728    let repository = ConnectionControlLeaseRepository::new();
1729    let mut permit = director.acquire(DbClass::Control).await?;
1730    repository.bootstrap(permit.connection()?).await?;
1731    let guard = repository
1732        .acquire(
1733            permit.connection()?,
1734            &ControlLeaseAcquireRequest {
1735                claim_name: "embedded-owner".to_owned(),
1736                cluster: control_lease_identity(cluster),
1737                owner_id: format!("engine-{}", IncarnationId::new()),
1738                lease_duration: OWNER_LEASE_DURATION,
1739                admission_margin: OWNER_ADMISSION_MARGIN,
1740                acquire_deadline: deadline,
1741            },
1742        )
1743        .await?;
1744    let gate = OwnershipAdmissionGate::for_guard(&guard);
1745    if let Some(quiescence_deadline) = guard.quiescence_deadline() {
1746        let now = Instant::now();
1747        if now >= deadline || quiescence_deadline > deadline {
1748            return Err(anyhow!("expired owner quiescence exceeds startup deadline"));
1749        }
1750        tokio::time::sleep_until(tokio::time::Instant::from_std(quiescence_deadline)).await;
1751        gate.finish_quiescence(&guard, Instant::now())?;
1752    }
1753    let report = OwnershipStartupReport {
1754        outcome: guard.outcome(),
1755        fence_token: guard.fence_token(),
1756    };
1757    Ok((
1758        EmbeddedOwnership {
1759            repository,
1760            guard: Arc::new(tokio::sync::Mutex::new(Some(guard))),
1761            gate,
1762        },
1763        report,
1764    ))
1765}
1766
1767fn spawn_ownership_renewal(
1768    tasks: &RuntimeShutdownHandle,
1769    director: Arc<DsqlConnectionDirector>,
1770    ownership: EmbeddedOwnership,
1771    service: InProcessGrpcService,
1772) {
1773    let cancel = tasks.cancellation_token();
1774    let _renewal = tasks.spawn(async move {
1775        let interval_duration =
1776            StdDuration::try_from(OWNER_RENEW_INTERVAL).expect("positive owner renewal interval");
1777        let mut interval = tokio::time::interval(interval_duration);
1778        interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
1779        interval.tick().await;
1780        loop {
1781            tokio::select! {
1782                _ = cancel.cancelled() => break,
1783                _ = interval.tick() => {
1784                    let mut guard_slot = ownership.guard.lock().await;
1785                    let Some(guard) = guard_slot.as_mut() else {
1786                        break;
1787                    };
1788                    let result = match director.acquire(DbClass::Control).await {
1789                        Ok(mut permit) => match permit.connection() {
1790                            Ok(connection) => ownership.repository.renew(
1791                                    connection,
1792                                    guard,
1793                                    OWNER_LEASE_DURATION,
1794                                    OWNER_ADMISSION_MARGIN,
1795                                    &ownership.gate,
1796                                )
1797                                .await
1798                                .map_err(anyhow::Error::from),
1799                            Err(error) => Err(error),
1800                        },
1801                        Err(error) => Err(error),
1802                    };
1803                    if let Err(error) = result {
1804                        if matches!(
1805                            error.downcast_ref::<ControlLeaseError>(),
1806                            Some(ControlLeaseError::Fenced)
1807                        ) {
1808                            ownership.gate.fence();
1809                        } else {
1810                            guard.enforce_admission_deadline(Instant::now(), &ownership.gate);
1811                        }
1812                    }
1813                    if ownership.gate.state() != OwnershipAdmissionState::Open {
1814                        service.begin_shutdown();
1815                        break;
1816                    }
1817                }
1818            }
1819        }
1820    });
1821}
1822
1823fn control_lease_identity(cluster: &ResolvedCluster) -> ControlLeaseClusterIdentity {
1824    ControlLeaseClusterIdentity {
1825        cluster_id: cluster.identity.cluster_id.clone(),
1826        cluster_arn: cluster.identity.cluster_arn.clone(),
1827    }
1828}
1829
1830async fn rollback_embedded_dsql(
1831    ownership: &EmbeddedOwnership,
1832    director: &Arc<DsqlConnectionDirector>,
1833    deadline: Instant,
1834) {
1835    ownership.gate.begin_closing();
1836    let now = Instant::now();
1837    if now < deadline {
1838        let _ = tokio::time::timeout(
1839            deadline.saturating_duration_since(now),
1840            release_embedded_ownership(ownership, director),
1841        )
1842        .await;
1843    }
1844    let _ = director.shutdown_with_deadline(deadline).await;
1845}
1846
1847async fn release_embedded_ownership(
1848    ownership: &EmbeddedOwnership,
1849    director: &Arc<DsqlConnectionDirector>,
1850) -> Result<()> {
1851    // Ordinary owner admission is closed here. This shutdown-only director
1852    // seam still consumes the Control budget and one bounded physical slot.
1853    let mut permit = director.acquire_shutdown_control().await?;
1854    let mut guard = ownership.guard.lock().await;
1855    if let Some(guard) = guard.take() {
1856        ownership
1857            .repository
1858            .release(permit.connection()?, &guard, &ownership.gate)
1859            .await?;
1860    }
1861    Ok(())
1862}
1863
1864async fn startup_phase<T, E, F>(
1865    deadline: Instant,
1866    phase: EmbeddedStartupPhase,
1867    future: F,
1868) -> Result<T, EmbeddedEngineStartError>
1869where
1870    F: Future<Output = Result<T, E>>,
1871{
1872    let now = Instant::now();
1873    if now >= deadline {
1874        return Err(EmbeddedEngineStartError::DeadlineExceeded { phase });
1875    }
1876    match tokio::time::timeout(deadline.saturating_duration_since(now), future).await {
1877        Ok(Ok(value)) => Ok(value),
1878        Ok(Err(_)) => Err(EmbeddedEngineStartError::Phase { phase }),
1879        Err(_) => Err(EmbeddedEngineStartError::DeadlineExceeded { phase }),
1880    }
1881}
1882
1883async fn startup_infallible_phase<T, F>(
1884    deadline: Instant,
1885    phase: EmbeddedStartupPhase,
1886    future: F,
1887) -> Result<T, EmbeddedEngineStartError>
1888where
1889    F: Future<Output = T>,
1890{
1891    let now = Instant::now();
1892    if now >= deadline {
1893        return Err(EmbeddedEngineStartError::DeadlineExceeded { phase });
1894    }
1895    tokio::time::timeout(deadline.saturating_duration_since(now), future)
1896        .await
1897        .map_err(|_| EmbeddedEngineStartError::DeadlineExceeded { phase })
1898}
1899
1900/// Drain a cancelled snapshot policy: await its periodic task, then persist
1901/// the final consistent cut.
1902///
1903/// Shared by the embedded facade and the listener-backed daemon so both
1904/// serving modes end with the identical durability sequence. The caller must
1905/// already have cancelled the policy's cancellation token.
1906async fn finish_snapshot_policy(mut policy: EngineSnapshotPolicy) -> Result<()> {
1907    if let Some(task) = policy.periodic_task.take() {
1908        task.await
1909            .context("engine snapshot task panicked during shutdown")?;
1910    }
1911    // Background workers observe the same token. Yield once so
1912    // cancellation-aware workers that are already runnable can leave their
1913    // select loops before the final consistent cut and before the caller
1914    // tears down its Tokio runtime.
1915    tokio::task::yield_now().await;
1916    persist_snapshot(&policy.store, &policy.location)
1917        .await
1918        .with_context(|| {
1919            format!(
1920                "failed to persist final engine snapshot to {}",
1921                policy.location.display()
1922            )
1923        })
1924}
1925
1926async fn restore_snapshot_store(config: &TokeiraConfig) -> Result<(InMemoryStore, bool)> {
1927    let Some(snapshot) = config.policy.snapshot.as_ref() else {
1928        return Ok((InMemoryStore::default(), false));
1929    };
1930    match fs::read(&snapshot.location).await {
1931        Ok(bytes) => {
1932            let store = InMemoryStore::from_snapshot(&bytes).with_context(|| {
1933                format!(
1934                    "failed to restore engine snapshot from {}",
1935                    snapshot.location.display()
1936                )
1937            })?;
1938            retire_snapshot_leases(&store).await.with_context(|| {
1939                format!(
1940                    "failed to retire prior embedded-engine leases from {}",
1941                    snapshot.location.display()
1942                )
1943            })?;
1944            info!(
1945                snapshot_path = %snapshot.location.display(),
1946                "restored engine snapshot"
1947            );
1948            Ok((store, true))
1949        }
1950        Err(error) if error.kind() == std::io::ErrorKind::NotFound => {
1951            Ok((InMemoryStore::default(), false))
1952        }
1953        Err(error) => Err(error).with_context(|| {
1954            format!(
1955                "failed to read engine snapshot from {}",
1956                snapshot.location.display()
1957            )
1958        }),
1959    }
1960}
1961
1962async fn retire_snapshot_leases(store: &InMemoryStore) -> Result<()> {
1963    for lease in store
1964        .list_bundle_leases()
1965        .await
1966        .context("failed to list engine snapshot leases")?
1967    {
1968        let Some(owner) = lease.owner_node_id else {
1969            continue;
1970        };
1971        match store
1972            .relinquish_bundle(lease.bundle_id, owner, lease.epoch)
1973            .await
1974            .context("failed to relinquish an engine snapshot lease")?
1975        {
1976            LeaseOutcome::Acquired { .. } => {}
1977            LeaseOutcome::Rejected {
1978                current_owner,
1979                current_epoch,
1980            } => {
1981                return Err(anyhow!(
1982                    "engine snapshot lease retirement was fenced by owner {current_owner} at epoch {}",
1983                    current_epoch.0
1984                ));
1985            }
1986            LeaseOutcome::Renewed { epoch } => {
1987                return Err(anyhow!(
1988                    "engine snapshot lease retirement unexpectedly renewed epoch {}",
1989                    epoch.0
1990                ));
1991            }
1992        }
1993    }
1994    Ok(())
1995}
1996
1997async fn persist_snapshot(store: &InMemoryStore, location: &Path) -> Result<()> {
1998    let bytes = store
1999        .snapshot()
2000        .await
2001        .context("failed to capture engine snapshot")?;
2002    write_snapshot_atomically(location, &bytes).await
2003}
2004
2005async fn write_snapshot_atomically(location: &Path, bytes: &[u8]) -> Result<()> {
2006    if let Some(parent) = location
2007        .parent()
2008        .filter(|parent| !parent.as_os_str().is_empty())
2009    {
2010        fs::create_dir_all(parent).await.with_context(|| {
2011            format!(
2012                "failed to create engine snapshot directory {}",
2013                parent.display()
2014            )
2015        })?;
2016    }
2017
2018    // A same-directory temporary keeps rename atomic on the target filesystem.
2019    // The process/sequence suffix prevents two embedded engines in one test
2020    // process from sharing a staging file even if they target the same snapshot.
2021    let sequence = SNAPSHOT_TEMP_SEQUENCE.fetch_add(1, Ordering::Relaxed);
2022    let mut temporary_name = location.as_os_str().to_os_string();
2023    temporary_name.push(format!(".tmp-{}-{sequence}", std::process::id()));
2024    let temporary = PathBuf::from(temporary_name);
2025
2026    let write_result: Result<()> = async {
2027        let mut file = OpenOptions::new()
2028            .create(true)
2029            .truncate(true)
2030            .write(true)
2031            .open(&temporary)
2032            .await
2033            .with_context(|| {
2034                format!(
2035                    "failed to create temporary engine snapshot {}",
2036                    temporary.display()
2037                )
2038            })?;
2039        file.write_all(bytes).await.with_context(|| {
2040            format!(
2041                "failed to write temporary engine snapshot {}",
2042                temporary.display()
2043            )
2044        })?;
2045        file.sync_all().await.with_context(|| {
2046            format!(
2047                "failed to sync temporary engine snapshot {}",
2048                temporary.display()
2049            )
2050        })?;
2051        drop(file);
2052        fs::rename(&temporary, location)
2053            .await
2054            .with_context(|| format!("failed to replace engine snapshot {}", location.display()))?;
2055        Ok(())
2056    }
2057    .await;
2058
2059    if write_result.is_err() {
2060        // Cleanup is best-effort and intentionally targets only the unique
2061        // staging file created above; the prior complete snapshot stays intact.
2062        let _ = fs::remove_file(&temporary).await;
2063    }
2064    write_result
2065}
2066
2067fn embedded_config(mut config: TokeiraConfig) -> Result<TokeiraConfig> {
2068    config.infrastructure.storage = ConfigStorageKind::InMemory;
2069    // An embedded engine owns its only runtime node. Retaining a controller
2070    // endpoint would spawn membership and routing clients even though there is
2071    // no advertised listener for another node to reach.
2072    config.infrastructure.placement.controller_endpoint = None;
2073    config
2074        .validate()
2075        .context("invalid embedded engine config")?;
2076    Ok(config)
2077}
2078
2079impl Drop for Engine {
2080    fn drop(&mut self) {
2081        if let Some(coordinator) = &self.shutdown_coordinator {
2082            coordinator.begin_shutdown();
2083        }
2084        self.background_cancel.cancel();
2085    }
2086}
2087
2088impl TemporalEndpoint {
2089    /// Dispatch one raw protobuf request through the Temporal service router.
2090    pub async fn call(
2091        &self,
2092        request: InProcessGrpcRequest,
2093    ) -> Result<InProcessGrpcResponse, tonic::Status> {
2094        tokio::select! {
2095            biased;
2096            _ = self.shutdown.cancelled() => {
2097                Err(tonic::Status::unavailable("embedded Tokeira engine is shut down"))
2098            }
2099            response = self.service.call(request) => response,
2100        }
2101    }
2102
2103    /// Whether the engine that owns this endpoint has begun shutdown.
2104    pub fn is_shutdown(&self) -> bool {
2105        self.shutdown.is_cancelled()
2106    }
2107
2108    /// Build the Temporal Rust SDK callback service for this endpoint.
2109    #[cfg(feature = "temporalio-client")]
2110    pub fn service_override(&self) -> temporalio_client::callback_based::CallbackBasedGrpcService {
2111        let endpoint = self.clone();
2112        temporalio_client::callback_based::CallbackBasedGrpcService {
2113            callback: Arc::new(move |request| {
2114                let endpoint = endpoint.clone();
2115                Box::pin(async move {
2116                    endpoint
2117                        .call(InProcessGrpcRequest {
2118                            service: request.service,
2119                            rpc: request.rpc,
2120                            headers: request.headers,
2121                            proto: request.proto,
2122                        })
2123                        .await
2124                        // The edge and the SDK share one tonic, so the status
2125                        // crosses the seam untouched: code, message, details,
2126                        // and metadata exactly as the edge produced them.
2127                        .map(
2128                            |response| temporalio_client::callback_based::GrpcSuccessResponse {
2129                                headers: response.headers,
2130                                proto: response.proto,
2131                            },
2132                        )
2133                })
2134            }),
2135        }
2136    }
2137}
2138
2139const EMBEDDED_NEXUS_CALLBACK_BASE: &str = "http://tokeira-engine.invalid";
2140
2141/// Completion client that keeps `temporal://system` callbacks inside the engine
2142/// while retaining the normal HTTP client for explicitly external callback URLs.
2143struct InProcessNexusCompletionClient {
2144    runtime: RwLock<Option<Weak<dyn WorkflowRuntimeApi>>>,
2145    http: HttpNexusCompletionClient,
2146}
2147
2148impl std::fmt::Debug for InProcessNexusCompletionClient {
2149    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
2150        f.debug_struct("InProcessNexusCompletionClient")
2151            .finish_non_exhaustive()
2152    }
2153}
2154
2155impl InProcessNexusCompletionClient {
2156    fn new() -> Self {
2157        Self {
2158            runtime: RwLock::new(None),
2159            http: HttpNexusCompletionClient::new(),
2160        }
2161    }
2162
2163    fn attach(&self, runtime: &Arc<dyn WorkflowRuntimeApi>) -> Result<()> {
2164        let mut slot = self
2165            .runtime
2166            .write()
2167            .map_err(|_| anyhow!("embedded Nexus completion runtime lock poisoned"))?;
2168        *slot = Some(Arc::downgrade(runtime));
2169        Ok(())
2170    }
2171}
2172
2173#[async_trait]
2174impl NexusCompletionClient for InProcessNexusCompletionClient {
2175    async fn complete_operation(
2176        &self,
2177        url: &str,
2178        token: &str,
2179        operation_token: &str,
2180        completion: NexusCompletion,
2181        links: &[Link],
2182    ) -> Result<CompletionDeliveryOutcome> {
2183        if url != system_callback_post_url(EMBEDDED_NEXUS_CALLBACK_BASE) {
2184            return self
2185                .http
2186                .complete_operation(url, token, operation_token, completion, links)
2187                .await;
2188        }
2189
2190        let runtime = self
2191            .runtime
2192            .read()
2193            .ok()
2194            .and_then(|slot| slot.as_ref().and_then(Weak::upgrade));
2195        let Some(runtime) = runtime else {
2196            return Ok(CompletionDeliveryOutcome::RetryableError {
2197                detail: "embedded Nexus completion runtime is unavailable".to_owned(),
2198            });
2199        };
2200
2201        let state = completion.operation_state();
2202        let (body, content_type) = match completion {
2203            NexusCompletion::Succeeded(payloads) => nexus_payload_to_body(&payloads),
2204            NexusCompletion::Failed(body) | NexusCompletion::Canceled(body) => {
2205                (body, Some("application/json".to_owned()))
2206            }
2207        };
2208        let response = handle_nexus_callback(
2209            runtime.as_ref(),
2210            Some(token),
2211            Some(operation_token),
2212            Some(state),
2213            content_type.as_deref(),
2214            &body,
2215        )
2216        .await;
2217        let detail = response
2218            .body
2219            .map(|body| String::from_utf8_lossy(&body).into_owned())
2220            .unwrap_or_else(|| format!("Nexus callback returned status {}", response.status));
2221        Ok(match response.status {
2222            200..=299 => CompletionDeliveryOutcome::Delivered,
2223            400 | 401 | 403 | 404 | 409 | 501 => {
2224                CompletionDeliveryOutcome::NonRetryableError { detail }
2225            }
2226            _ => CompletionDeliveryOutcome::RetryableError { detail },
2227        })
2228    }
2229}
2230
2231/// Handle returned by [`TokeiradHandle::start_in_memory`].
2232///
2233/// Drop-safe: the inner server task receives a shutdown signal when the handle
2234/// is dropped, after which the task completes on its own. For deterministic
2235/// teardown (tests that want to assert a clean exit) call [`Self::shutdown`]
2236/// explicitly.
2237///
2238/// The handle also exposes a [`log_sink`](Self::log_sink) that integration
2239/// tests use to observe SDK-facing RPC activity (specifically the per-call
2240/// `tracing::debug!` line that `record_worker_heartbeat` emits). The stream is
2241/// a broadcast receiver over structured tracing events captured inside the
2242/// facade; consumers not subscribed to it see no behaviour change.
2243#[derive(Debug)]
2244pub struct TokeiradHandle {
2245    bound_addr: SocketAddr,
2246    shutdown_tx: Option<oneshot::Sender<()>>,
2247    server_task: Option<JoinHandle<Result<()>>>,
2248    /// Cancellation token for background workers spawned by the facade.
2249    ///
2250    /// Kept alongside the server task so that dropping the handle cancels the
2251    /// membership client, routing subscription, projectors, and
2252    /// schedule engine in lockstep with the gRPC server shutdown.
2253    background_cancel: CancellationToken,
2254    log_broadcast: broadcast::Sender<LogEvent>,
2255    /// In-memory snapshot policy, when configured. The final consistent cut
2256    /// runs in [`Self::shutdown`]; a plain drop cancels only (an async write
2257    /// cannot run in `Drop`), so callers needing durability call `shutdown`.
2258    snapshot_policy: Option<EngineSnapshotPolicy>,
2259}
2260
2261/// One observable RPC event streamed through [`TokeiradHandle::log_sink`].
2262///
2263/// This is deliberately narrow: it captures only the fields integration tests
2264/// need to distinguish one RPC from another. The full tracing-subscriber
2265/// configuration still applies — this channel is a convenience fan-out, not a
2266/// replacement for the log pipeline.
2267#[derive(Clone, Debug)]
2268pub struct LogEvent {
2269    pub rpc: String,
2270    pub namespace: String,
2271    pub heartbeat_count: usize,
2272}
2273
2274impl TokeiradHandle {
2275    /// Start an in-memory `tokeirad` bound to the caller-provided socket.
2276    ///
2277    /// `addr` may be `127.0.0.1:0` to request an ephemeral port; the bound
2278    /// port is then available via [`Self::bound_addr`]. All other server
2279    /// configuration uses `TokeiraConfig::default()`, which selects the
2280    /// in-memory storage path.
2281    pub async fn start_in_memory(addr: SocketAddr) -> Result<Self> {
2282        Self::start_in_memory_with_config(addr, TokeiraConfig::default()).await
2283    }
2284
2285    /// Start an in-memory `tokeirad` with an explicit validated configuration.
2286    ///
2287    /// This is the production bootstrap path with only storage selection and
2288    /// listener addresses fixed for process-local integration tests. In
2289    /// particular, configured authentication, authorization, and Worker-scope
2290    /// policy are constructed exactly as they are by the binary.
2291    pub async fn start_in_memory_with_config(
2292        addr: SocketAddr,
2293        mut config: TokeiraConfig,
2294    ) -> Result<Self> {
2295        config.infrastructure.storage = ConfigStorageKind::InMemory;
2296        // Bind the inbound Nexus completion listener on an ephemeral loopback port so
2297        // parallel in-memory servers (tests, multi-node harnesses) never collide on the
2298        // fixed default port; the runtime resolves `temporal://system` to the bound port.
2299        config.policy.nexus_completion.http_addr = "127.0.0.1:0".to_string();
2300        config.policy.nexus_completion.system_callback_url = "http://127.0.0.1:0".to_string();
2301        config
2302            .validate()
2303            .context("invalid in-memory server config")?;
2304        let effective_config = Arc::new(config);
2305        let (
2306            (server_task, bound_addr, shutdown_tx, background_cancel, log_broadcast, _recorder),
2307            snapshot_policy,
2308            _node_claim,
2309        ) = build_and_serve(addr, effective_config).await?;
2310        Ok(Self {
2311            bound_addr,
2312            shutdown_tx: Some(shutdown_tx),
2313            server_task: Some(server_task),
2314            background_cancel,
2315            log_broadcast,
2316            snapshot_policy,
2317        })
2318    }
2319
2320    /// Parse TOML and start the in-memory integration facade.
2321    ///
2322    /// Keeping parsing inside this crate lets cross-crate integration tests
2323    /// exercise the public configuration surface without depending directly on
2324    /// the configuration implementation crate.
2325    pub async fn start_in_memory_with_toml(addr: SocketAddr, config: &str) -> Result<Self> {
2326        let config = toml::from_str(config).context("parse in-memory server config")?;
2327        Self::start_in_memory_with_config(addr, config).await
2328    }
2329
2330    /// The socket address the server is bound to after startup.
2331    pub fn bound_addr(&self) -> SocketAddr {
2332        self.bound_addr
2333    }
2334
2335    /// Subscribe to the in-process log-event broadcast. Each subscriber gets a
2336    /// fresh receiver; events emitted before subscription are not replayed.
2337    pub fn log_sink(&self) -> broadcast::Receiver<LogEvent> {
2338        self.log_broadcast.subscribe()
2339    }
2340
2341    /// Signal the server to shut down, wait for the task to exit, and persist
2342    /// the final snapshot when a snapshot policy is configured.
2343    pub async fn shutdown(mut self) -> Result<()> {
2344        if let Some(tx) = self.shutdown_tx.take() {
2345            let _ = tx.send(());
2346        }
2347        self.background_cancel.cancel();
2348        if let Some(task) = self.server_task.take() {
2349            task.await
2350                .context("tokeirad server task panicked")?
2351                .context("tokeirad server task returned an error")?;
2352        }
2353        match self.snapshot_policy.take() {
2354            Some(policy) => finish_snapshot_policy(policy).await,
2355            None => Ok(()),
2356        }
2357    }
2358}
2359
2360impl Drop for TokeiradHandle {
2361    fn drop(&mut self) {
2362        if let Some(tx) = self.shutdown_tx.take() {
2363            let _ = tx.send(());
2364        }
2365        self.background_cancel.cancel();
2366        // Server task is not awaited here; callers who need deterministic
2367        // teardown call `shutdown` explicitly. Tokio will reap the task.
2368    }
2369}
2370
2371fn dsql_auth_config(config: &TokeiraConfig) -> Result<DsqlAuthConfig> {
2372    let dsql = &config.infrastructure.dsql;
2373    let endpoint = dsql
2374        .endpoint
2375        .clone()
2376        .filter(|endpoint| !endpoint.trim().is_empty())
2377        .ok_or_else(|| {
2378            anyhow!("infrastructure.dsql.endpoint must be set when infrastructure.storage is dsql")
2379        })?;
2380    Ok(DsqlAuthConfig {
2381        endpoint,
2382        region: dsql.region.clone(),
2383        admin_role_arn: dsql.admin_role_arn.clone(),
2384        runtime_role_arn: dsql.runtime_role_arn.clone(),
2385        readonly_role_arn: dsql.readonly_role_arn.clone(),
2386    })
2387}
2388
2389fn dsql_pool_config_with_client(
2390    config: &TokeiraConfig,
2391    ddb_client: aws_sdk_dynamodb::Client,
2392) -> (DsqlPoolConfig, aws_sdk_dynamodb::Client) {
2393    let (rate_limiter_table, conn_lease_table) = dsql_coordination_table_names(config);
2394    let pool_config = DsqlPoolConfig {
2395        coordination: DsqlCoordinationConfig {
2396            rate_limiter_table,
2397            conn_lease_table,
2398        },
2399        shard_count: config.infrastructure.placement.shard_count,
2400        projection_partition_count: config.infrastructure.placement.partition_count,
2401        ..DsqlPoolConfig::default()
2402    };
2403    (pool_config, ddb_client)
2404}
2405
2406async fn dsql_pool_config(
2407    config: &TokeiraConfig,
2408    auth: &DsqlAuthConfig,
2409) -> Result<(DsqlPoolConfig, aws_sdk_dynamodb::Client)> {
2410    let region = auth
2411        .resolved_region()
2412        .ok_or_else(|| anyhow!("dsql region must be configured or derivable from endpoint"))?;
2413    let sdk_config = aws_config::defaults(aws_config::BehaviorVersion::latest())
2414        .region(aws_config::Region::new(region))
2415        .load()
2416        .await;
2417    Ok(dsql_pool_config_with_client(
2418        config,
2419        aws_sdk_dynamodb::Client::new(&sdk_config),
2420    ))
2421}
2422
2423fn dsql_coordination_table_names(config: &TokeiraConfig) -> (String, String) {
2424    let dsql = &config.infrastructure.dsql;
2425    let rate_limiter = dsql
2426        .rate_limiter_table
2427        .clone()
2428        .unwrap_or_else(|| format!("{}-dsql-rate-limiter", config.infrastructure.cluster_name));
2429    let conn_lease = dsql
2430        .conn_lease_table
2431        .clone()
2432        .unwrap_or_else(|| format!("{}-dsql-conn-lease", config.infrastructure.cluster_name));
2433    (rate_limiter, conn_lease)
2434}
2435
2436/// Entrypoint the CLI delegates to.
2437///
2438/// Parses `TokeiraConfig` from the CLI arguments, handles `--dump-config`, and
2439/// serves the gRPC stack on `infrastructure.network.grpc_addr` until ctrl-C
2440/// or SIGTERM, then drains gracefully: in-flight requests finish, background
2441/// work is cancelled, and (for in-memory storage with a snapshot policy) the
2442/// final consistent cut is persisted before exit.
2443// CLI output is intentional for --version, --dump-config and generated path config.
2444#[allow(clippy::print_stdout)]
2445pub async fn run_from_cli(cli: Cli) -> Result<()> {
2446    if let Some(command) = cli.command {
2447        return match command {
2448            CliCommand::PeerCredential {
2449                command: PeerCredentialCommand::Generate { out },
2450            } => {
2451                println!("{}", peer::generate(&out)?);
2452                Ok(())
2453            }
2454            CliCommand::Storage { command } => storage_maintenance::run(command).await,
2455        };
2456    }
2457    if cli.version {
2458        println!("{}", render_build_info(cli.verbose, cli.json));
2459        return Ok(());
2460    }
2461
2462    let (effective_config, config_source) = TokeiraConfig::resolve(cli.config.as_deref())?;
2463    if cli.dump_config {
2464        // A TOML comment so the dump stays parseable while naming the winner.
2465        println!("# config source: {config_source}");
2466        println!("{}", effective_config.to_toml()?);
2467        return Ok(());
2468    }
2469    let observability = observability::ObservabilityConfig::from_tokeira_config(&effective_config)?;
2470    let effective_config = Arc::new(effective_config);
2471    let readiness = observability::TokeiradReadiness::new();
2472    let _observability_runtime = observability::install_process_observability(
2473        &observability,
2474        effective_config.clone(),
2475        readiness.registry.clone(),
2476    )
2477    .await?;
2478    log_build_info("tokeirad");
2479    for warning in effective_config.emergency_warnings() {
2480        tracing::warn!("{warning}");
2481    }
2482
2483    let addr: SocketAddr = effective_config
2484        .infrastructure
2485        .network
2486        .grpc_addr
2487        .parse()
2488        .with_context(|| {
2489            format!(
2490                "invalid infrastructure.network.grpc_addr value: {}",
2491                effective_config.infrastructure.network.grpc_addr
2492            )
2493        })?;
2494    info!(config_source = %config_source, "loaded tokeirad configuration");
2495
2496    let (
2497        (
2498            mut server_task,
2499            bound_addr,
2500            shutdown_tx,
2501            background_cancel,
2502            _log_broadcast,
2503            wire_recorder,
2504        ),
2505        snapshot_policy,
2506        node_claim,
2507    ) = build_and_serve(addr, effective_config).await?;
2508    readiness.mark_started();
2509    info!("tokeirad gRPC server listening on {bound_addr}");
2510
2511    // Daemon-only signal handling. The embedded facade (`Engine`) deliberately
2512    // installs no handlers and no process observability: an embedded host owns
2513    // its process lifecycle, and a library that grabs SIGTERM would race the
2514    // host's own shutdown sequence. Only this served entrypoint reacts to
2515    // ctrl-C/SIGTERM, and the reaction is the graceful drain: stop accepting
2516    // connections via the listener's shutdown oneshot, await in-flight
2517    // requests, cancel background work, then take the final snapshot below.
2518    let serve_result = {
2519        let drain_signal = async {
2520            #[cfg(unix)]
2521            {
2522                let mut sigterm =
2523                    tokio::signal::unix::signal(tokio::signal::unix::SignalKind::terminate())
2524                        .context("failed to install the SIGTERM handler")?;
2525                tokio::select! {
2526                    result = tokio::signal::ctrl_c() => result
2527                        .context("failed to listen for ctrl-c")
2528                        .map(|()| "SIGINT"),
2529                    _ = sigterm.recv() => Ok("SIGTERM"),
2530                }
2531            }
2532            #[cfg(not(unix))]
2533            {
2534                tokio::signal::ctrl_c()
2535                    .await
2536                    .context("failed to listen for ctrl-c")
2537                    .map(|()| "ctrl-c")
2538            }
2539        };
2540        tokio::select! {
2541            result = &mut server_task => result
2542                .context("tokeirad server task panicked")?
2543                .context("tokeirad server task returned an error"),
2544            signal = drain_signal => {
2545                match signal {
2546                    Ok(signal) => info!(signal, "shutdown signal received; draining tokeirad"),
2547                    Err(error) => {
2548                        // Signal delivery is broken; a blunt stop is the only
2549                        // honest option left, but say why.
2550                        tracing::warn!(?error, "signal handling failed; shutting down");
2551                    }
2552                }
2553                let _ = shutdown_tx.send(());
2554                (&mut server_task)
2555                    .await
2556                    .context("tokeirad server task panicked")?
2557                    .context("tokeirad server task returned an error")
2558            }
2559        }
2560    };
2561    background_cancel.cancel();
2562    // A node without a controller releases its single-node claim once its
2563    // background work is cancelled and its admitted DSQL work has drained, so
2564    // the next node starts at once. When the claim was lost, the claim's task
2565    // started this drain, and the node exits with that error.
2566    let claim_result = match node_claim {
2567        Some(claim) => claim.shut_down().await,
2568        None => Ok(()),
2569    };
2570    // The final consistent cut runs after the drain regardless of how the
2571    // server exited, so an error exit still leaves the freshest snapshot. A
2572    // snapshot failure must not mask the server's own exit status; it is
2573    // surfaced only after that status is known good.
2574    let snapshot_result = match snapshot_policy {
2575        Some(policy) => finish_snapshot_policy(policy).await,
2576        None => Ok(()),
2577    };
2578
2579    // Tier-2 conformance evidence export. When the wire-coverage recorder is present (the
2580    // conformance flag was set), snapshot the observed `(wire_method, status_code)` set
2581    // after the server has stopped serving and write it as pretty JSON for the Rust report
2582    // (see `.kiro/specs/temporal-functional-conformance`, task 9.x). This runs regardless
2583    // of how the server exited so a clean shutdown after a run still produces evidence; a
2584    // failed export is logged but MUST NOT mask the server's own exit status, so the
2585    // server result is returned after the export attempt. When the recorder is `None`
2586    // nothing is written and there is zero behavioural change.
2587    if let Some(recorder) = wire_recorder {
2588        let out_path = wire_coverage_out_path();
2589        if let Err(error) = export_wire_coverage(recorder.as_ref(), &out_path) {
2590            tracing::warn!(
2591                path = %out_path.display(),
2592                ?error,
2593                "failed to export conformance wire-coverage evidence"
2594            );
2595        }
2596    }
2597
2598    serve_result?;
2599    claim_result?;
2600    snapshot_result?;
2601    Ok(())
2602}
2603
2604/// Snapshot the recorder and write the wire-coverage record to `path` as pretty JSON.
2605///
2606/// Pretty (rather than compact) JSON is deliberate: the evidence file is read by humans
2607/// triaging conformance runs and is diffed across runs, so a stable, line-oriented layout
2608/// is worth the few extra bytes. The recorder's `snapshot()` already sorts rows
2609/// deterministically, so two runs observing the same calls produce byte-identical output.
2610///
2611/// Errors are surfaced with context rather than panicking: a failed export is a loss of
2612/// *evidence*, never a correctness fault, and the caller logs it without disturbing the
2613/// server's exit status.
2614fn export_wire_coverage(recorder: &WireCoverageRecorder, path: &Path) -> Result<()> {
2615    let record = recorder.snapshot();
2616    let row_count = record.rows.len();
2617    let json = serde_json::to_string_pretty(&record)
2618        .context("failed to serialize wire-coverage record to JSON")?;
2619    std::fs::write(path, json).with_context(|| {
2620        format!(
2621            "failed to write wire-coverage evidence to {}",
2622            path.display()
2623        )
2624    })?;
2625    info!(
2626        path = %path.display(),
2627        rows = row_count,
2628        "wrote conformance wire-coverage evidence"
2629    );
2630    Ok(())
2631}
2632
2633fn log_build_info(process: &'static str) {
2634    let info = tokeira_build_info::summary();
2635    tracing::info!(
2636        process,
2637        server_version = tokeira_build_info::SERVER_VERSION,
2638        tokeira_version = info.tokeira_version,
2639        tokeira_git_sha = info.tokeira_git_sha,
2640        tokeira_source_revision = info.tokeira_source_revision,
2641        temporal_proto_version = info.temporal_proto_version,
2642        temporal_server_compat = info.temporal_server_compat,
2643        rust_toolchain = info.rust_toolchain,
2644        source_tree_hash = info.source_tree_hash,
2645        feature_matrix_digest = info.feature_matrix_digest,
2646        sdk_matrix_digest = info.sdk_matrix_digest,
2647        build_mode = info.build_mode,
2648        "tokeira build provenance"
2649    );
2650}
2651
2652fn render_build_info(verbose: bool, json: bool) -> String {
2653    let info = tokeira_build_info::summary();
2654    if json {
2655        return serde_json::json!({
2656            "server_version": tokeira_build_info::SERVER_VERSION,
2657            "tokeira_version": info.tokeira_version,
2658            "tokeira_git_sha": info.tokeira_git_sha,
2659            "source_revision": info.tokeira_source_revision,
2660            "temporal_proto_version": info.temporal_proto_version,
2661            "temporal_server_compat": info.temporal_server_compat,
2662            "rust_toolchain": info.rust_toolchain,
2663            "source_tree_hash": info.source_tree_hash,
2664            "feature_matrix_digest": info.feature_matrix_digest,
2665            "sdk_matrix_digest": info.sdk_matrix_digest,
2666            "build_mode": info.build_mode,
2667        })
2668        .to_string();
2669    }
2670
2671    if verbose {
2672        return [
2673            format!("server_version: {}", tokeira_build_info::SERVER_VERSION),
2674            format!("tokeira_version: {}", info.tokeira_version),
2675            format!("tokeira_git_sha: {}", info.tokeira_git_sha),
2676            format!("tokeira_source_revision: {}", info.tokeira_source_revision),
2677            format!("temporal_proto_version: {}", info.temporal_proto_version),
2678            format!("temporal_server_compat: {}", info.temporal_server_compat),
2679            format!("rust_toolchain: {}", info.rust_toolchain),
2680            format!("source_tree_hash: {}", info.source_tree_hash),
2681            format!("feature_matrix_digest: {}", info.feature_matrix_digest),
2682            format!("sdk_matrix_digest: {}", info.sdk_matrix_digest),
2683            format!("build_mode: {}", info.build_mode),
2684        ]
2685        .join("\n");
2686    }
2687
2688    format!(
2689        "tokeira {}\ngit {}\ntemporal_proto {}\ntemporal_server {}",
2690        tokeira_build_info::SERVER_VERSION,
2691        info.tokeira_git_sha,
2692        info.temporal_proto_version,
2693        info.temporal_server_compat
2694    )
2695}
2696
2697/// What [`build_and_serve`] hands back to the
2698/// caller after the gRPC stack is spawned.
2699///
2700/// The final element — the optional `Arc<WireCoverageRecorder>` — is the Tier-2
2701/// conformance recorder handle. It is `Some` only when the conformance flag is set (see
2702/// [`wire_coverage_enabled`]); the caller snapshots it after the server task completes to
2703/// export wire-coverage evidence. It is carried as a tuple element rather than threaded
2704/// into [`TokeiradHandle`] because only the CLI entrypoint exports the evidence — the
2705/// in-memory test facade ignores it.
2706type ServerStack = (
2707    JoinHandle<Result<()>>,
2708    SocketAddr,
2709    oneshot::Sender<()>,
2710    CancellationToken,
2711    broadcast::Sender<LogEvent>,
2712    Option<Arc<WireCoverageRecorder>>,
2713);
2714
2715#[derive(Clone, Copy, Debug)]
2716enum StackTransport {
2717    Network(SocketAddr),
2718    Embedded,
2719}
2720
2721#[derive(Debug)]
2722enum ConstructedStack {
2723    Network(ServerStack),
2724    Embedded(EmbeddedStack),
2725}
2726
2727struct EmbeddedStack {
2728    component_visibility_store: chasm_extensions::VisibilityQueryStore,
2729    chasm_engine: Arc<tokeira_runtime::chasm::ChasmEngine>,
2730    registry: Arc<tokeira_chasm::Registry>,
2731    service: InProcessGrpcService,
2732    background_cancel: CancellationToken,
2733    log_broadcast: broadcast::Sender<LogEvent>,
2734    recovery_task: Option<JoinHandle<Result<()>>>,
2735    runtime_tasks: RuntimeShutdownHandle,
2736    engine_tasks: RuntimeShutdownHandle,
2737    shard_cleanup: Option<CleanupAction>,
2738}
2739
2740struct StackStartupGuard {
2741    runtime_tasks: RuntimeShutdownHandle,
2742    engine_tasks: RuntimeShutdownHandle,
2743    armed: bool,
2744}
2745
2746impl StackStartupGuard {
2747    fn new(runtime_tasks: RuntimeShutdownHandle, engine_tasks: RuntimeShutdownHandle) -> Self {
2748        Self {
2749            runtime_tasks,
2750            engine_tasks,
2751            armed: true,
2752        }
2753    }
2754
2755    fn disarm(&mut self) {
2756        self.armed = false;
2757    }
2758}
2759
2760impl Drop for StackStartupGuard {
2761    fn drop(&mut self) {
2762        if self.armed {
2763            // Async cleanup is owned by the caller's pool rollback, but every
2764            // background loop must first observe cancellation so it cannot
2765            // retain the partially constructed stack indefinitely.
2766            self.engine_tasks.begin_shutdown();
2767            self.runtime_tasks.begin_shutdown();
2768        }
2769    }
2770}
2771
2772#[derive(Clone, Debug)]
2773struct SelfAssignedShardLease {
2774    shard_id: ShardId,
2775    owner: String,
2776    epoch: ShardEpoch,
2777}
2778
2779impl std::fmt::Debug for EmbeddedStack {
2780    fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
2781        formatter
2782            .debug_struct("EmbeddedStack")
2783            .field("service", &self.service)
2784            .field("background_cancel", &self.background_cancel)
2785            .field("runtime_tasks", &self.runtime_tasks)
2786            .field("engine_tasks", &self.engine_tasks)
2787            .field("has_shard_cleanup", &self.shard_cleanup.is_some())
2788            .finish_non_exhaustive()
2789    }
2790}
2791
2792/// Default maximum identifier length for standalone-activity ids, mirroring
2793/// Temporal's `MaxIDLengthLimit` default of `1000` (`common/dynamicconfig` @
2794/// v1.31.0). Used to validate standalone-activity ids at the edge.
2795const DEFAULT_MAX_ID_LENGTH: usize = 1000;
2796
2797/// Refuse a migration policy a listener-backed distributed node doesn't
2798/// honour, before it connects to anything.
2799///
2800/// A distributed node never migrates the schema it shares with every other
2801/// node: migrating stays the operator's explicit `tkr schema setup` (root
2802/// `AGENTS.md` §4). So an absent or `validate_only` policy selects the startup
2803/// check, and `automatic` is refused rather than quietly read as
2804/// `validate_only`.
2805fn refuse_distributed_migration_policy(config: &TokeiraConfig) -> Result<()> {
2806    match config.infrastructure.dsql.migration_policy {
2807        None | Some(DsqlMigrationPolicy::ValidateOnly) => Ok(()),
2808        Some(DsqlMigrationPolicy::Automatic) => Err(anyhow!(
2809            "infrastructure.dsql.migration_policy = \"automatic\" is not supported by \
2810             listener-backed distributed DSQL: a distributed tokeirad never migrates the \
2811             schema it shares with other nodes. Apply migrations with `tkr schema setup`, \
2812             then set the policy to \"validate_only\" or remove it"
2813        )),
2814    }
2815}
2816
2817/// Check a listener-backed distributed node's DSQL schema against its
2818/// release's schema contract, without changing it, and refuse unless the
2819/// release can run on it.
2820///
2821/// The assessment is the embedded engine's (`apply_embedded_schema`) under the
2822/// validate-only policy. It uses the runner over the compiled-in migrations,
2823/// from which the release's contract is computed; the store's own runner reads
2824/// a migrations directory. Its one connection comes from the store's
2825/// coordinated reservoir, so the check keeps to DSQL's connection limits like
2826/// every other read.
2827async fn check_distributed_schema(dsql_store: &DsqlStore) -> Result<()> {
2828    let contract = MigrationRunner::compatibility_contract();
2829    let decision = {
2830        let mut permit = dsql_store
2831            .connection_director()
2832            .acquire(DbClass::Control)
2833            .await?;
2834        MigrationRunner::embedded()
2835            .assess_connection(
2836                permit.connection()?,
2837                &contract,
2838                SchemaMigrationPolicy::ValidateOnly,
2839            )
2840            .await
2841            .context("failed to read the DSQL schema's migration ledger")?
2842    };
2843    let current = distributed_schema_admission(decision)?;
2844    info!(
2845        schema_version = current,
2846        target_version = contract.target_version,
2847        maximum_readable_version = contract.maximum_readable_version,
2848        "DSQL schema check passed"
2849    );
2850    Ok(())
2851}
2852
2853/// Admit a listener-backed distributed node only to a schema that a
2854/// validate-only assessment found compatible, returning its version.
2855/// Otherwise return the refusal the node exits with: what it found and what
2856/// to do about it.
2857fn distributed_schema_admission(decision: SchemaDecision) -> Result<u32> {
2858    match decision {
2859        SchemaDecision::Compatible { current, .. } => Ok(current),
2860        SchemaDecision::MigrationRequired { current: 0, target } => Err(anyhow!(
2861            "tokeirad refuses to start: the DSQL database has no Tokeira schema. Apply this \
2862             release's migrations, through V{target}, with `tkr schema setup`, then start \
2863             tokeirad"
2864        )),
2865        SchemaDecision::MigrationRequired { current, target } => Err(anyhow!(
2866            "tokeirad refuses to start: the DSQL schema is at V{current}, behind this \
2867             release's target V{target}. Apply the missing migrations with `tkr schema \
2868             setup`, then start tokeirad"
2869        )),
2870        SchemaDecision::Reject(incompatibility) => {
2871            let advice = match incompatibility.category {
2872                SchemaIncompatibilityCategory::FutureSchema => {
2873                    "a later release migrated it; run that release or a later one"
2874                }
2875                SchemaIncompatibilityCategory::BelowMinimum => {
2876                    "it is older than this release migrates from; upgrade it through an \
2877                     earlier release first"
2878                }
2879                _ => {
2880                    "its migration ledger doesn't match this release's migrations; find out \
2881                     what changed it before any node runs on it"
2882                }
2883            };
2884            Err(
2885                anyhow::Error::new(SchemaCompatibilityError::Incompatible(incompatibility))
2886                    .context(format!(
2887                        "tokeirad refuses to start: this release can't run on the DSQL \
2888                         schema, because {advice}"
2889                    )),
2890            )
2891        }
2892        // An assessment under the validate-only policy never initializes or
2893        // migrates; only the automatic policy decides either.
2894        decision @ (SchemaDecision::Initialize { .. } | SchemaDecision::Migrate { .. }) => {
2895            Err(anyhow!(
2896                "tokeirad refuses to start: a validate-only schema assessment decided \
2897                 {decision:?}"
2898            ))
2899        }
2900    }
2901}
2902
2903/// Build the full server stack and start serving on the given address.
2904///
2905/// Storage selection is driven only by `infrastructure.storage`: endpoint
2906/// presence alone is not treated as intent. This keeps failed DSQL writeback
2907/// and explicit in-memory deployments from being conflated. A DSQL node checks
2908/// its schema before it builds anything ([`check_distributed_schema`]), and a
2909/// DSQL node without a controller then takes its cluster's single-node claim
2910/// ([`take_node_claim`]), which it returns for the caller to release.
2911///
2912/// Factored out so both the CLI entrypoint and the in-memory facade share one
2913/// bootstrap path. Deliberately long and sequential: each block mirrors the
2914/// startup dependency order.
2915async fn build_and_serve(
2916    addr: SocketAddr,
2917    effective_config: Arc<TokeiraConfig>,
2918) -> Result<(
2919    ServerStack,
2920    Option<EngineSnapshotPolicy>,
2921    Option<node_claim::NodeClaim>,
2922)> {
2923    // This is the existing single-node ownership decision, shared with peer
2924    // bootstrap. Membership size never grants the no-peer exemption.
2925    let no_controller = effective_config
2926        .infrastructure
2927        .placement
2928        .controller_endpoint
2929        .is_none();
2930    let peer = if no_controller {
2931        None
2932    } else {
2933        Some(peer::bootstrap(
2934            effective_config.infrastructure.peer.as_ref(),
2935        )?)
2936    };
2937    let mut snapshot_inputs = None;
2938    let mut node_claim = None;
2939    let stack = match effective_config.infrastructure.storage {
2940        ConfigStorageKind::InMemory => {
2941            // The listener-backed in-memory server shares the embedded
2942            // facade's snapshot mechanism exactly: restore at boot, periodic
2943            // persistence, and a final cut at shutdown all operate on the one
2944            // in-memory store.
2945            let (store, restored) = restore_snapshot_store(&effective_config).await?;
2946            snapshot_inputs = effective_config
2947                .policy
2948                .snapshot
2949                .clone()
2950                .map(|config| (store.clone(), config));
2951            build_in_memory_stack(
2952                StackTransport::Network(addr),
2953                effective_config,
2954                store,
2955                restored,
2956                ChasmExtensions::default(),
2957                peer,
2958            )
2959            .await
2960        }
2961        ConfigStorageKind::Dsql => {
2962            if effective_config.policy.snapshot.is_some() {
2963                return Err(anyhow!(
2964                    "policy.snapshot is configured but infrastructure.storage is \"dsql\": \
2965                     snapshots are the in-memory store's persistence mechanism, and DSQL \
2966                     storage is already durable — remove policy.snapshot or switch \
2967                     infrastructure.storage to \"memory\""
2968                ));
2969            }
2970            refuse_distributed_migration_policy(&effective_config)?;
2971            let auth = dsql_auth_config(&effective_config)?;
2972            let endpoint = auth.endpoint.clone();
2973            let (pool_config, ddb_client) = dsql_pool_config(&effective_config, &auth).await?;
2974            let dsql_store = DsqlStore::connect(auth, pool_config, ddb_client)
2975                .await
2976                .context("failed to connect DSQL storage backend")?;
2977            // The check runs before anything is built, so a refusal leaves no
2978            // listener, runtime or shard lease behind. Shutting the store down
2979            // closes its connections and releases the DynamoDB connection-slot
2980            // blocks its reservoir took, which dropping it would leave owned.
2981            if let Err(refusal) = check_distributed_schema(&dsql_store).await {
2982                if let Err(error) = dsql_store.shutdown().await {
2983                    tracing::warn!(
2984                        error = %format!("{error:#}"),
2985                        "failed to shut DSQL storage down after refusing its schema"
2986                    );
2987                }
2988                return Err(refusal);
2989            }
2990            // A node without a controller assigns itself every home and writes
2991            // unfenced, so it serves only while it holds the cluster's
2992            // single-node claim. Refusing the claim leaves nothing started, as
2993            // refusing the schema does.
2994            if no_controller {
2995                match take_node_claim(&dsql_store).await {
2996                    Ok(claim) => node_claim = Some(claim),
2997                    Err(refusal) => {
2998                        if let Err(error) = dsql_store.shutdown().await {
2999                            tracing::warn!(
3000                                error = %format!("{error:#}"),
3001                                "failed to shut DSQL storage down after refusing to start"
3002                            );
3003                        }
3004                        return Err(refusal);
3005                    }
3006                }
3007            }
3008            build_dsql_stack(
3009                StackTransport::Network(addr),
3010                effective_config,
3011                dsql_store,
3012                endpoint,
3013                ChasmExtensions::default(),
3014                peer,
3015            )
3016            .await
3017        }
3018    };
3019    let stack = match stack {
3020        Ok(ConstructedStack::Network(stack)) => stack,
3021        Ok(ConstructedStack::Embedded(_)) => {
3022            release_node_claim_after_failed_start(node_claim).await;
3023            return Err(anyhow!(
3024                "network service construction returned an embedded stack"
3025            ));
3026        }
3027        Err(error) => {
3028            release_node_claim_after_failed_start(node_claim).await;
3029            return Err(error);
3030        }
3031    };
3032    // A lost claim cancels the stack's background work, which also stops the
3033    // server: the node drains, and its caller exits with the claim's error.
3034    if let Some(claim) = &node_claim {
3035        claim.drain_on_loss(stack.3.clone());
3036    }
3037    // The policy shares the stack's cancellation token so the periodic task
3038    // dies with the server's other background work; the final consistent cut
3039    // stays with the caller's shutdown sequence.
3040    let snapshot_policy = snapshot_inputs
3041        .map(|(store, config)| EngineSnapshotPolicy::start(store, config, stack.3.clone()));
3042    Ok((stack, snapshot_policy, node_claim))
3043}
3044
3045/// Take the DSQL cluster's single-node claim for a node without a controller,
3046/// after its schema check and before it builds anything (`single-node-claim`
3047/// 2.1–2.5): acquire it, start renewing it, install its admission gate on the
3048/// store's director, and wait out the quiescence after an expired holder. A
3049/// claim another node holds is tried again for 80 s, then refused.
3050async fn take_node_claim(dsql_store: &DsqlStore) -> Result<node_claim::NodeClaim> {
3051    let director = dsql_store.connection_director_arc();
3052    let acquired = node_claim::acquire(&director).await?;
3053    let gate = acquired.gate();
3054    let mut claim = acquired.start(director.clone());
3055    if let Err(error) = director.install_ownership_gate(gate) {
3056        release_node_claim_after_failed_start(Some(claim)).await;
3057        return Err(error.context("failed to install the single-node claim's admission gate"));
3058    }
3059    if let Err(error) = claim.wait_until_serving().await {
3060        release_node_claim_after_failed_start(Some(claim)).await;
3061        return Err(error);
3062    }
3063    Ok(claim)
3064}
3065
3066/// Stop a node's claim after a failed start: it is released as on a clean
3067/// shutdown, unless an expired takeover's quiescence is still running, when it
3068/// is left to expire (`single-node-claim` 2.8).
3069async fn release_node_claim_after_failed_start(claim: Option<node_claim::NodeClaim>) {
3070    if let Some(claim) = claim
3071        && let Err(error) = claim.shut_down().await
3072    {
3073        tracing::warn!(
3074            error = %format!("{error:#}"),
3075            "the single-node claim was lost while the node failed to start"
3076        );
3077    }
3078}
3079
3080async fn build_dsql_stack(
3081    transport: StackTransport,
3082    effective_config: Arc<TokeiraConfig>,
3083    dsql_store: DsqlStore,
3084    endpoint: String,
3085    extensions: ChasmExtensions,
3086    peer: Option<Arc<tokeira_edge::grpc::workflow_offer::PeerOfferEndpoint>>,
3087) -> Result<ConstructedStack> {
3088    let (director, run_repository, projection_log, worker_deployment_repository, _migration_runner) =
3089        dsql_store.into_parts();
3090    // Every repository and visibility surface shares this exact director. The
3091    // embedded caller retains another Arc solely for ordered pool shutdown;
3092    // no second pool or DynamoDB coordinator is constructed here.
3093    let budget_applier: Arc<dyn ConnectionBudgetApplier> = Arc::new(DsqlConnectionBudgetApplier {
3094        director: director.clone(),
3095    });
3096    let chasm_node_repo = Arc::new(tokeira_storage::dsql::DsqlChasmNodeRepository::new(
3097        director.clone(),
3098    ));
3099    let task_queue_config_repository: Arc<dyn TaskQueueConfigRepository> = Arc::new(
3100        tokeira_storage::dsql::DsqlTaskQueueConfigRepository::new(director.clone()),
3101    );
3102    let worker_compute_repository = effective_config.policy.worker_compute.enabled.then(|| {
3103        Arc::new(tokeira_storage::dsql::DsqlWorkerComputeRepository::new(
3104            director.clone(),
3105        )) as Arc<dyn WorkerComputeRepository>
3106    });
3107    let worker_task_provenance: Arc<dyn WorkerTaskProvenanceStore> = Arc::new(
3108        tokeira_storage::dsql::DsqlWorkerTaskProvenanceStore::new(director.clone()),
3109    );
3110    let visibility_store = DsqlVisibilityStore::new(director);
3111    // Before anything here can apply an image (the projectors, the CHASM visibility
3112    // adapter, the visibility repair scanner), copy an earlier release's attribute
3113    // index into its versioned layout. The listener-backed and embedded engines both
3114    // build their stack here; a finished copy returns at once, and an interrupted one
3115    // resumes from its stored position (`bounded-visibility-apply`).
3116    visibility_store
3117        .copy_legacy_attributes()
3118        .await
3119        .context("failed to copy the visibility attribute index")?;
3120    let worker_deployment_repository: Arc<dyn WorkerDeploymentRepository> =
3121        Arc::new(worker_deployment_repository);
3122    build_service_stack_with_storage(
3123        transport,
3124        effective_config,
3125        Arc::new(run_repository),
3126        worker_deployment_repository,
3127        worker_compute_repository,
3128        task_queue_config_repository,
3129        worker_task_provenance,
3130        projection_log,
3131        visibility_store.clone(),
3132        {
3133            let visibility_store = visibility_store.clone();
3134            move || visibility_store.clone()
3135        },
3136        Some(endpoint),
3137        chasm_node_repo,
3138        false,
3139        budget_applier,
3140        extensions,
3141        peer,
3142    )
3143    .await
3144}
3145
3146async fn build_embedded(
3147    effective_config: Arc<TokeiraConfig>,
3148    store: InMemoryStore,
3149    recover_self_assigned_shard: bool,
3150    extensions: ChasmExtensions,
3151) -> Result<EmbeddedStack> {
3152    match build_in_memory_stack(
3153        StackTransport::Embedded,
3154        effective_config,
3155        store,
3156        recover_self_assigned_shard,
3157        extensions,
3158        None,
3159    )
3160    .await?
3161    {
3162        ConstructedStack::Embedded(stack) => Ok(stack),
3163        ConstructedStack::Network(_) => Err(anyhow!(
3164            "embedded service construction returned a network stack"
3165        )),
3166    }
3167}
3168
3169async fn build_in_memory_stack(
3170    transport: StackTransport,
3171    effective_config: Arc<TokeiraConfig>,
3172    store: InMemoryStore,
3173    recover_self_assigned_shard: bool,
3174    extensions: ChasmExtensions,
3175    peer: Option<Arc<tokeira_edge::grpc::workflow_offer::PeerOfferEndpoint>>,
3176) -> Result<ConstructedStack> {
3177    let visibility_store = InMemoryVisibilityStore::default();
3178    let chasm_node_repo: Arc<dyn tokeira_storage::ChasmNodeRepository> =
3179        Arc::new(tokeira_storage::InMemoryChasmNodeStore::new());
3180    #[cfg(test)]
3181    let (chasm_node_repo, visibility_store) = extensions
3182        .test_storage
3183        .clone()
3184        .unwrap_or((chasm_node_repo, visibility_store));
3185    let worker_deployment_repository: Arc<dyn WorkerDeploymentRepository> = Arc::new(store.clone());
3186    let worker_task_provenance: Arc<dyn WorkerTaskProvenanceStore> = Arc::new(store.clone());
3187    let worker_compute_repository = effective_config.policy.worker_compute.enabled.then(|| {
3188        Arc::new(InMemoryWorkerComputeRepository::default()) as Arc<dyn WorkerComputeRepository>
3189    });
3190    build_service_stack_with_storage(
3191        transport,
3192        effective_config,
3193        Arc::new(store.clone()),
3194        worker_deployment_repository,
3195        worker_compute_repository,
3196        Arc::new(store.clone()),
3197        worker_task_provenance,
3198        store,
3199        visibility_store.clone(),
3200        {
3201            let visibility_store = visibility_store.clone();
3202            move || VisibilitySink::new(visibility_store.clone())
3203        },
3204        None,
3205        chasm_node_repo,
3206        recover_self_assigned_shard,
3207        // The in-memory store has no connection reservoir to budget.
3208        Arc::new(NoopConnectionBudgetApplier),
3209        extensions,
3210        peer,
3211    )
3212    .await
3213}
3214
3215/// How often the visibility repair scanner sweeps committed executions to repair any
3216/// projection lost by the best-effort post-commit write (Req 10.11).
3217const VISIBILITY_REPAIR_INTERVAL: std::time::Duration = std::time::Duration::from_secs(30);
3218
3219/// Recovery cadence for disposable CHASM timers and dispatch state.
3220const CHASM_REBUILD_INTERVAL: std::time::Duration = std::time::Duration::from_secs(30);
3221
3222/// Authorization evidence is derived from task deadlines, so cleanup needs no
3223/// operator TTL. One bounded batch per tick prevents expired rows from
3224/// monopolizing the shared DSQL connection budget.
3225const WORKER_TASK_PROVENANCE_CLEANUP_INTERVAL: std::time::Duration =
3226    std::time::Duration::from_secs(30);
3227const WORKER_TASK_PROVENANCE_CLEANUP_BATCH: usize = 256;
3228
3229fn spawn_worker_task_provenance_cleanup(
3230    tasks: &RuntimeShutdownHandle,
3231    store: Arc<dyn WorkerTaskProvenanceStore>,
3232    cancel: CancellationToken,
3233) -> JoinHandle<()> {
3234    tasks.spawn(async move {
3235        let mut interval = tokio::time::interval(WORKER_TASK_PROVENANCE_CLEANUP_INTERVAL);
3236        interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
3237        loop {
3238            tokio::select! {
3239                _ = cancel.cancelled() => break,
3240                _ = interval.tick() => {
3241                    if let Err(error) = store
3242                        .delete_expired(
3243                            time::OffsetDateTime::now_utc(),
3244                            WORKER_TASK_PROVENANCE_CLEANUP_BATCH,
3245                        )
3246                        .await
3247                    {
3248                        tracing::warn!(
3249                            error_kind = worker_task_provenance_cleanup_error_kind(&error),
3250                            "Worker task authorization-evidence cleanup pass failed"
3251                        );
3252                    }
3253                }
3254            }
3255        }
3256    })
3257}
3258
3259fn worker_task_provenance_cleanup_error_kind(
3260    error: &tokeira_storage::WorkerTaskProvenanceError,
3261) -> &'static str {
3262    match error {
3263        tokeira_storage::WorkerTaskProvenanceError::Unavailable { .. } => "unavailable",
3264        tokeira_storage::WorkerTaskProvenanceError::DigestConflict => "conflict",
3265        tokeira_storage::WorkerTaskProvenanceError::Corrupt { .. } => "corrupt",
3266    }
3267}
3268
3269/// Spawn the background visibility repair scanner: it reconstructs each committed
3270/// execution's snapshot from authoritative node state and re-applies it iff-newer, so
3271/// a committed transition can never permanently lack a projection (Req 10.11). The
3272/// snapshot rebuild uses the registered root adapters for every archetype. Runs immediately, then on an interval, until
3273/// `cancel` fires.
3274fn spawn_visibility_repair(
3275    tasks: &RuntimeShutdownHandle,
3276    nodes: Arc<dyn tokeira_storage::ChasmNodeRepository>,
3277    sink: Arc<dyn tokeira_projection::ProjectionSink>,
3278    registry: Arc<tokeira_chasm::Registry>,
3279    partition_count: u32,
3280    cancel: CancellationToken,
3281) {
3282    let rebuild: tokeira_runtime::chasm::SnapshotRebuilder =
3283        Arc::new(move |archetype_id, bytes| {
3284            match registry.visibility_snapshot(archetype_id, bytes) {
3285                Ok(snapshot) => snapshot,
3286                Err(error) => {
3287                    tracing::warn!(
3288                        ?error,
3289                        archetype_id,
3290                        "CHASM visibility rebuild decode failed"
3291                    );
3292                    None
3293                }
3294            }
3295        });
3296    let scanner =
3297        tokeira_runtime::chasm::VisibilityRepairScanner::new(nodes, sink, rebuild, partition_count);
3298    let _repair = tasks.spawn(async move {
3299        let mut interval = tokio::time::interval(VISIBILITY_REPAIR_INTERVAL);
3300        interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
3301        loop {
3302            tokio::select! {
3303                _ = cancel.cancelled() => break,
3304                _ = interval.tick() => {
3305                    if let Err(error) = scanner.repair_once().await {
3306                        tracing::warn!(?error, "visibility repair pass failed");
3307                    }
3308                }
3309            }
3310        }
3311    });
3312}
3313
3314/// Repeat the startup recovery pass so losing an in-memory delivery hint costs
3315/// at most one scan interval. Shutdown cancels any in-flight derived scan.
3316fn spawn_outbox_rebuild(
3317    tasks: &RuntimeShutdownHandle,
3318    scanner: tokeira_runtime::chasm::OutboxRebuildScanner,
3319    cancel: CancellationToken,
3320) {
3321    let _rebuild = tasks.spawn(async move {
3322        let mut interval = tokio::time::interval(CHASM_REBUILD_INTERVAL);
3323        interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
3324        // Startup already ran the first pass before opening admission.
3325        interval.tick().await;
3326        loop {
3327            tokio::select! {
3328                _ = cancel.cancelled() => break,
3329                _ = interval.tick() => {
3330                    tokio::select! {
3331                        _ = cancel.cancelled() => break,
3332                        result = scanner.rebuild_once() => {
3333                            match result {
3334                                Ok(stats) if stats.unserviceable > 0 => tracing::error!(unserviceable = stats.unserviceable, first = ?stats.first_unserviceable, "CHASM rebuild isolated executions; healthy executions were rebuilt"),
3335                                Ok(_) => {},
3336                                Err(error) => tracing::warn!(?error, "CHASM outbox rebuild pass failed"),
3337                            }
3338                        }
3339                    }
3340                }
3341            }
3342        }
3343    });
3344}
3345
3346/// The interval the CHASM timer sweeper ticks at. Activity timeouts (schedule-to-
3347/// start/close, start-to-close, heartbeat) fire within roughly this cadence of their
3348/// deadline; ~200ms is well below the smallest conformance timeout while keeping the
3349/// sweep cheap (a snapshot of the armed-timer map plus a fenced update per due timer).
3350const CHASM_TIMER_SWEEP_INTERVAL: std::time::Duration = std::time::Duration::from_millis(200);
3351
3352/// Spawn the CHASM timer sweeper: it fires armed activity timeouts on a tick by
3353/// delegating to the [`TimeoutEvaluator`](tokeira_runtime::chasm::TimeoutEvaluator)
3354/// (the activity bridge), then re-arms each execution to its state-derived next
3355/// deadline. Runtime-only (clock + loop); all timeout/retry semantics are pure
3356/// (`tokeira-chasm-activity`) behind the evaluator, so the kernel-purity and
3357/// history-authority invariants hold. Runs until `cancel` fires. Gated on standalone
3358/// activities or extension libraries being enabled; generic registered handlers
3359/// drive extension timers while the activity evaluator keeps its existing path.
3360fn spawn_chasm_timer_sweeper(
3361    tasks: &RuntimeShutdownHandle,
3362    engine: Arc<tokeira_runtime::chasm::ChasmEngine>,
3363    evaluator: Arc<dyn tokeira_runtime::chasm::TimeoutEvaluator>,
3364    archetype_id: u32,
3365    cancel: CancellationToken,
3366) {
3367    let sweeper = tokeira_runtime::chasm::ChasmTimerSweeper::new(engine)
3368        .with_evaluator(archetype_id, evaluator);
3369    let _sweeper = tasks.spawn(async move {
3370        let mut interval = tokio::time::interval(CHASM_TIMER_SWEEP_INTERVAL);
3371        interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
3372        loop {
3373            tokio::select! {
3374                _ = cancel.cancelled() => break,
3375                _ = interval.tick() => {
3376                    sweeper.sweep_once().await;
3377                }
3378            }
3379        }
3380    });
3381}
3382
3383async fn build_service_stack_with_storage<R, L, S, V, F>(
3384    transport: StackTransport,
3385    effective_config: Arc<TokeiraConfig>,
3386    run_repository: Arc<R>,
3387    worker_deployment_repository: Arc<dyn WorkerDeploymentRepository>,
3388    worker_compute_repository: Option<Arc<dyn WorkerComputeRepository>>,
3389    task_queue_config_repository: Arc<dyn TaskQueueConfigRepository>,
3390    worker_task_provenance: Arc<dyn WorkerTaskProvenanceStore>,
3391    projection_log: L,
3392    visibility_query_store: V,
3393    projection_sink: F,
3394    dsql_endpoint: Option<String>,
3395    chasm_node_repo: Arc<dyn tokeira_storage::ChasmNodeRepository>,
3396    recover_self_assigned_shard: bool,
3397    budget_applier: Arc<dyn ConnectionBudgetApplier>,
3398    extensions: ChasmExtensions,
3399    peer: Option<Arc<tokeira_edge::grpc::workflow_offer::PeerOfferEndpoint>>,
3400) -> Result<ConstructedStack>
3401where
3402    R: LeaseRepository + RunRepository + 'static,
3403    L: ProjectionLog + Clone + 'static,
3404    S: ProjectionSink + VisibilityStore + 'static,
3405    V: VisibilityStore + Clone + 'static,
3406    F: Fn() -> S + Clone + Send + Sync + 'static,
3407{
3408    // Construction spawns scanners and may seed an Active home. No runtime
3409    // may exist until every physical legacy row has reached its execution home.
3410    tokeira_storage::prepare_execution_placement(run_repository.as_ref()).await?;
3411    // Freeze declarations before constructing the operator wrapper, so later
3412    // namespaces receive the same set as startup. Stateful CHASM admission stays
3413    // in its existing block below, shared by both storage paths.
3414    let has_extensions = !extensions.libraries.is_empty();
3415    let mut registry_builder = tokeira_chasm::Registry::builder();
3416    tokeira_chasm_activity::ActivityLibrary::register(&mut registry_builder)
3417        .map_err(registry_start_error)?;
3418    registry_builder.seal_built_ins();
3419    for library in extensions.libraries {
3420        library(&mut registry_builder).map_err(registry_start_error)?;
3421    }
3422    let registry = Arc::new(registry_builder.build());
3423    let declared = declared_search_attributes(&registry);
3424    // Build the authoritative store first, then wrap it with the
3425    // history-notifying repository used by edge long-poll.
3426    let node_id = IncarnationId::new();
3427    let advertised_addr = match transport {
3428        StackTransport::Network(addr) => addr,
3429        StackTransport::Embedded => SocketAddr::from(([127, 0, 0, 1], 0)),
3430    };
3431    let node_endpoint = configured_node_endpoint(&effective_config, advertised_addr);
3432    let history_waits = HistoryWaitRegistry::default();
3433    let repo = Arc::new(HistoryNotifyingRepository::new(
3434        run_repository,
3435        history_waits.clone(),
3436    ));
3437    let default_namespace = ResolvedNamespace::active("default");
3438    let default_namespace_id = namespace_id_for("default");
3439
3440    // Bootstrap edge-facing namespace/operator state.
3441    let namespaces = Arc::new(InMemoryNamespaceCache::new());
3442    namespaces.insert(default_namespace).await?;
3443    // The Nexus endpoint store is the single source of truth shared by runtime
3444    // dispatch resolution and the OperatorService endpoint-admin CRUD. Seeded empty;
3445    // endpoints are created at runtime via `CreateNexusEndpoint`.
3446    let nexus_store = build_nexus_endpoint_store(
3447        namespaces.as_ref(),
3448        HashMap::<String, BootstrapNexusEndpointConfig>::new(),
3449    )
3450    .await?;
3451    let nexus_registry = NexusEndpointRegistry::new(nexus_store.clone());
3452    let nexus_http_client: Arc<dyn tokeira_runtime::NexusHttpClient> =
3453        Arc::new(HttpNexusClient::new());
3454
3455    // The runtime owns execution orchestration, scanners, brokers, and all
3456    // run-local in-memory coordination such as buffered consistent queries.
3457    let schedule_store = Arc::new(ScheduleStore::default());
3458    let task_queue_config_store = Arc::new(RepositoryBackedTaskQueueConfigStore::new(
3459        task_queue_config_repository,
3460    ));
3461    // A server never fabricates an unset/default policy when durable state was
3462    // merely unavailable: hydrate must succeed before polls can be accepted.
3463    task_queue_config_store
3464        .hydrate()
3465        .await
3466        .context("failed to hydrate task-queue policy")?;
3467    // Agreement is a startup prerequisite, before runtime construction or
3468    // membership can acquire a home. A later incompatible snapshot terminates
3469    // serving instead of silently changing the durable placement function.
3470    let initial_routing_cache = if let Some(endpoint) = &effective_config
3471        .infrastructure
3472        .placement
3473        .controller_endpoint
3474    {
3475        let cache = Arc::new(RoutingCache::new(placement_config(&effective_config)));
3476        tokio::time::timeout(
3477            std::time::Duration::from_secs(10),
3478            tokeira_edge::subscribe_routing_once(cache.clone(), endpoint.clone()),
3479        )
3480        .await
3481        .context("timed out waiting for controller placement snapshot")??;
3482        Some(cache)
3483    } else {
3484        None
3485    };
3486    let mut runtime_config = RuntimeConfig::default();
3487    runtime_config.lane.controller_managed_placement = effective_config
3488        .infrastructure
3489        .placement
3490        .controller_endpoint
3491        .is_some();
3492    let seed_default_shard = !recover_self_assigned_shard
3493        && dsql_endpoint.is_none()
3494        && effective_config
3495            .infrastructure
3496            .placement
3497            .controller_endpoint
3498            .is_none();
3499
3500    let nexus_completion_cfg = effective_config.policy.nexus_completion.clone();
3501    let (
3502        nexus_callback_listener,
3503        in_process_nexus_client,
3504        nexus_completion_client,
3505        resolved_system_callback_url,
3506    ): (
3507        Option<TcpListener>,
3508        Option<Arc<InProcessNexusCompletionClient>>,
3509        Arc<dyn NexusCompletionClient>,
3510        String,
3511    ) = match transport {
3512        StackTransport::Network(_) => {
3513            // Bind before runtime construction because the firing client's system
3514            // callback URL must contain the listener's actual ephemeral port.
3515            let listener = TcpListener::bind(&nexus_completion_cfg.http_addr)
3516                .await
3517                .with_context(|| {
3518                    format!(
3519                        "failed to bind nexus completion listener on {}",
3520                        nexus_completion_cfg.http_addr
3521                    )
3522                })?;
3523            let callback_addr = listener
3524                .local_addr()
3525                .context("failed to resolve bound nexus completion listener address")?;
3526            let callback_url = with_loopback_port(
3527                &nexus_completion_cfg.system_callback_url,
3528                callback_addr.port(),
3529            );
3530            info!(
3531                bind = %callback_addr,
3532                loopback = %callback_url,
3533                "nexus completion callback listener bound"
3534            );
3535            (
3536                Some(listener),
3537                None,
3538                Arc::new(HttpNexusCompletionClient::new()),
3539                callback_url,
3540            )
3541        }
3542        StackTransport::Embedded => {
3543            // A weak runtime attachment closes the system-callback loop without a
3544            // socket and without creating a runtime→client→runtime ownership cycle.
3545            let client = Arc::new(InProcessNexusCompletionClient::new());
3546            (
3547                None,
3548                Some(client.clone()),
3549                client,
3550                EMBEDDED_NEXUS_CALLBACK_BASE.to_owned(),
3551            )
3552        }
3553    };
3554    let nexus_completion_deps = NexusCompletionDeps {
3555        client: nexus_completion_client,
3556        config: NexusCompletionRuntimeConfig {
3557            system_callback_url: resolved_system_callback_url,
3558            retry_initial_interval: time::Duration::milliseconds(
3559                nexus_completion_cfg.retry_initial_interval_ms as i64,
3560            ),
3561            retry_max_interval: time::Duration::milliseconds(
3562                nexus_completion_cfg.retry_max_interval_ms as i64,
3563            ),
3564            retry_backoff_coefficient: nexus_completion_cfg.retry_backoff_coefficient,
3565            retry_max_attempts: nexus_completion_cfg.retry_max_attempts,
3566        },
3567        scanner: CompletionCallbackScannerConfig::default(),
3568    };
3569
3570    let chasm_nexus_client = nexus_completion_deps.client.clone();
3571    let chasm_nexus_config = nexus_completion_deps.config.clone();
3572
3573    let worker_compute_deployment_repository = worker_deployment_repository.clone();
3574    let runtime = TokeiraRuntime::new_with_nexus_and_shards_and_endpoint(
3575        repo.clone(),
3576        runtime_config.lane_count,
3577        runtime_config.lane,
3578        runtime_config.timer_scanner,
3579        runtime_config.workflow_timeout_scanner,
3580        runtime_config.activity_timeout_scanner,
3581        runtime_config.nexus_timeout_scanner,
3582        nexus_registry.clone(),
3583        nexus_http_client.clone(),
3584        // Listener-backed mode posts through HTTP. Embedded mode intercepts the
3585        // reserved system target and invokes the same callback handler inline.
3586        nexus_completion_deps,
3587        effective_config.infrastructure.placement.shard_count,
3588        node_id.to_string(),
3589        node_endpoint.as_authority(),
3590        seed_default_shard,
3591        // Namespace names for the publisher, resolved through the shared edge
3592        // namespace cache: the External-endpoint outbound Nexus metric's tag, and
3593        // the target namespace in external signal and cancel resolving events.
3594        Some(Arc::new(CacheNexusNamespaceResolver {
3595            namespaces: namespaces.clone(),
3596        }) as Arc<dyn NexusNamespaceResolver>),
3597    )
3598    // The edge always exposes Worker Deployment v2 RPCs. Wiring the
3599    // repository here keeps their registry durable for both in-memory and
3600    // DSQL backends instead of falling back to a detached test registry.
3601    .with_queue_placement(
3602        effective_config.infrastructure.placement.partition_count,
3603        effective_config.infrastructure.placement.bundle_count,
3604    )
3605    .with_worker_deployment_repository(worker_deployment_repository)
3606    .with_task_queue_config_store(task_queue_config_store.clone())
3607    .with_delivery_mode_provider(Arc::new(
3608        tokeira_runtime::ConfiguredDeliveryModeProvider::new(
3609            tokeira_runtime::StaticDeliveryPolicy {
3610                enable_fairness: effective_config.policy.task_queues.enable_fairness,
3611            },
3612        ),
3613    ));
3614    runtime.ready().await?;
3615    let runtime = Arc::new(runtime);
3616    let runtime_tasks = runtime.shutdown_handle();
3617    let engine_tasks = RuntimeShutdownHandle::new();
3618    let mut startup_guard = StackStartupGuard::new(runtime_tasks.clone(), engine_tasks.clone());
3619    let background_cancel = engine_tasks.cancellation_token();
3620
3621    let recovered_lease = if recover_self_assigned_shard {
3622        let shard_id = ShardId(0);
3623        let epoch = runtime
3624            .acquire_shard(shard_id)
3625            .await
3626            .context("failed to recover embedded snapshot shard")?;
3627        Some((shard_id, node_id.to_string(), epoch))
3628    } else {
3629        None
3630    };
3631
3632    if matches!(transport, StackTransport::Network(_))
3633        && dsql_endpoint.is_some()
3634        && effective_config
3635            .infrastructure
3636            .placement
3637            .controller_endpoint
3638            .is_none()
3639    {
3640        let _leases = self_assign_dsql_shards(
3641            runtime.as_ref(),
3642            repo.as_ref(),
3643            effective_config.infrastructure.placement.shard_count,
3644            &node_id,
3645            &node_endpoint,
3646            false,
3647        )
3648        .await?;
3649    }
3650
3651    let recovery_task = recovered_lease.map(|(shard_id, owner, epoch)| {
3652        let runtime = runtime.clone();
3653        let repo = repo.clone();
3654        let cancel = background_cancel.clone();
3655        tokio::spawn(async move {
3656            cancel.cancelled().await;
3657            // Drain first so the shard-scoped token stops the lease renewer,
3658            // then advance the durable epoch before a graceful snapshot. A
3659            // crash may leave an active owner in the last interval snapshot;
3660            // boot retires that captured owner before acquiring the new one.
3661            runtime.relinquish_shard(shard_id).await;
3662            match repo
3663                .relinquish_bundle(shard_id, owner, epoch)
3664                .await
3665                .context("failed to release embedded snapshot recovery lease")?
3666            {
3667                LeaseOutcome::Acquired { .. } => Ok(()),
3668                LeaseOutcome::Rejected {
3669                    current_owner,
3670                    current_epoch,
3671                } => Err(anyhow!(
3672                    "embedded snapshot recovery lease was fenced by owner {current_owner} at epoch {}",
3673                    current_epoch.0
3674                )),
3675                LeaseOutcome::Renewed { epoch } => Err(anyhow!(
3676                    "embedded snapshot recovery lease unexpectedly renewed at epoch {}",
3677                    epoch.0
3678                )),
3679            }
3680        })
3681    });
3682    let task_queue_config_refresh = task_queue_config_store
3683        .clone()
3684        .spawn_refresh(background_cancel.clone());
3685    let _task_queue_config_refresh = engine_tasks.track_join(task_queue_config_refresh);
3686    let _worker_task_provenance_cleanup = spawn_worker_task_provenance_cleanup(
3687        &engine_tasks,
3688        worker_task_provenance.clone(),
3689        background_cancel.clone(),
3690    );
3691
3692    let _worker_compute_service = if effective_config.policy.worker_compute.enabled {
3693        let controller_repository = worker_compute_repository
3694            .clone()
3695            .context("worker-compute enabled without a controller repository")?;
3696        let (observation_sender, observation_receiver) =
3697            mpsc::channel(OBSERVATION_CHANNEL_CAPACITY);
3698        let observation_sink = Arc::new(ChannelDemandObservationSink::new(observation_sender));
3699        runtime
3700            .broker()
3701            .set_demand_observation_sink(observation_sink.clone());
3702        runtime
3703            .activity_broker()
3704            .set_demand_observation_sink(observation_sink.clone());
3705        runtime
3706            .nexus_task_broker()
3707            .set_demand_observation_sink(observation_sink);
3708
3709        let (reconcile_sender, reconcile_receiver) = mpsc::channel(RECONCILE_CHANNEL_CAPACITY);
3710        runtime
3711            .deployment_registry()
3712            .expect("Worker Deployment repository was installed above")
3713            .set_worker_compute_reconcile_sink(Arc::new(ChannelWorkerComputeReconcileSink::new(
3714                reconcile_sender,
3715            )));
3716
3717        let clock = Arc::new(SystemWorkerComputeClock);
3718        let catalog = Arc::new(WorkerComputeNamespaceCatalogAdapter::new(
3719            namespaces.clone(),
3720        ));
3721        let reconciler = WorkerComputeReconciler::new(
3722            worker_compute_deployment_repository,
3723            controller_repository.clone(),
3724            node_id,
3725        );
3726        let sampler = runtime.worker_compute_queue_sampler(controller_repository.clone(), node_id);
3727        let provider = Arc::new(NexusWorkerComputeProvider::new(
3728            nexus_registry,
3729            nexus_http_client,
3730            runtime.nexus_task_broker(),
3731        ));
3732        let outbox = WorkerComputeOutbox::new(
3733            controller_repository.clone(),
3734            provider,
3735            clock.clone(),
3736            node_id,
3737        );
3738        let shard_runtime = runtime.clone();
3739        let active_shards: tokeira_runtime::WorkerComputeActiveShards =
3740            Arc::new(move || shard_runtime.active_shards());
3741        let service = WorkerComputeControllerService::new(
3742            catalog,
3743            controller_repository,
3744            reconciler,
3745            sampler,
3746            outbox,
3747            clock,
3748            active_shards,
3749            observation_receiver,
3750            reconcile_receiver,
3751        );
3752        let service_cancel = background_cancel.clone();
3753        Some(engine_tasks.spawn(async move {
3754            if let Err(error) = service.run(service_cancel).await {
3755                tracing::warn!(?error, "worker-compute controller service exited");
3756            }
3757        }))
3758    } else {
3759        None
3760    };
3761
3762    let membership_client = effective_config
3763        .infrastructure
3764        .placement
3765        .controller_endpoint
3766        .clone()
3767        .map(|controller_endpoint| {
3768            runtime.spawn_membership_client(
3769                membership_config(
3770                    &effective_config,
3771                    node_id,
3772                    node_endpoint.clone(),
3773                    controller_endpoint,
3774                ),
3775                budget_applier,
3776                background_cancel.clone(),
3777            )
3778        });
3779    if let Some(membership_client) = membership_client {
3780        let _membership_client = engine_tasks.track_join(membership_client);
3781    }
3782    let schedule_engine = run_schedule_engine(
3783        schedule_store.clone(),
3784        runtime.clone(),
3785        ScheduleEngineConfig::default(),
3786        background_cancel.clone(),
3787    );
3788    let _schedule_engine = engine_tasks.track_join(schedule_engine);
3789
3790    let authorization = build_authorization_stack(&effective_config).await?;
3791    let interceptors = Arc::new(EdgeInterceptors::configured(
3792        namespaces.clone(),
3793        authorization.grpc,
3794        authorization.principal_attribution,
3795    ));
3796    let router: Arc<dyn tokeira_edge::EdgeRouter> = if let Some(controller_endpoint) =
3797        effective_config
3798            .infrastructure
3799            .placement
3800            .controller_endpoint
3801            .clone()
3802    {
3803        let routing_cache = initial_routing_cache
3804            .expect("controller snapshot validated before runtime construction");
3805        let routing_config = EdgeRoutingConfig {
3806            controller_endpoint,
3807            max_retries: effective_config
3808                .infrastructure
3809                .placement
3810                .routing_max_retries,
3811        };
3812        let subscription_cache = routing_cache.clone();
3813        let subscription_cancel = background_cancel.clone();
3814        let routing_shutdown = runtime.shutdown_handle();
3815        let routing_cancel = background_cancel.clone();
3816        let _routing_subscription = engine_tasks.spawn(async move {
3817            if let Err(error) =
3818                run_routing_subscription(subscription_cache, routing_config, subscription_cancel)
3819                    .await
3820            {
3821                tracing::error!(
3822                    ?error,
3823                    "routing placement became incompatible; stopping node"
3824                );
3825                routing_shutdown.begin_shutdown();
3826                routing_cancel.cancel();
3827            }
3828        });
3829        runtime.set_workflow_task_starter(Arc::new(
3830            tokeira_edge::grpc::workflow_dispatch::RemoteWorkflowTaskStarter::new(
3831                routing_cache.clone(),
3832            ),
3833        ));
3834        if let Some(peer) = &peer {
3835            runtime.set_activity_task_starter(Arc::new(
3836                tokeira_edge::grpc::activity_dispatch::RemoteActivityTaskStarter::new(
3837                    routing_cache.clone(),
3838                    peer.clone(),
3839                ),
3840            ));
3841            runtime.set_outbound_delivery_transport(Arc::new(
3842                tokeira_edge::grpc::outbound_delivery::RemoteOutboundDelivery::new(
3843                    routing_cache.clone(),
3844                    peer.clone(),
3845                ),
3846            ));
3847        }
3848        Arc::new(CacheBackedRouter::new(routing_cache, node_id))
3849    } else {
3850        Arc::new(LocalOnlyRouter)
3851    };
3852    let workflow_broker = runtime.broker();
3853    let buffered_queries = runtime.buffered_queries();
3854    let worker_registry = runtime.worker_registry();
3855    let nexus_task_broker = runtime.nexus_task_broker();
3856    let runtime_adapter = Arc::new(RuntimeAdapter::new(runtime.clone()));
3857    let callback_runtime = runtime_adapter.clone() as Arc<dyn WorkflowRuntimeApi>;
3858    if let Some(client) = in_process_nexus_client {
3859        client.attach(&callback_runtime)?;
3860    }
3861    if let Some(listener) = nexus_callback_listener {
3862        // Listener-backed and embedded completion both delegate to this same
3863        // `WorkflowRuntimeApi`; only the transport around the callback differs.
3864        spawn_nexus_callback_server(listener, callback_runtime, background_cancel.clone());
3865    }
3866    let resolver = Arc::new(StoreExecutionResolver::new(repo.clone()));
3867    tokeira_projection::seed_predefined_search_attributes(
3868        &visibility_query_store,
3869        default_namespace_id,
3870    )
3871    .await
3872    .context("failed to seed Temporal predefined search attributes")?;
3873    let operator_visibility_store = visibility_query_store.clone();
3874    let visibility = Arc::new(VisibilityQueryService::new(visibility_query_store.clone()));
3875    let long_polls = LongPollGate::new(LongPollConfig::default());
3876    let operator_api = Arc::new(
3877        VisibilityRegistryOperatorApi::new(
3878            InMemoryOperatorApi::new("tokeira-local", tokeira_build_info::SERVER_VERSION),
3879            operator_visibility_store,
3880        )
3881        .with_declared_search_attributes(declared.clone()),
3882    );
3883
3884    // One projector per partition in the deployment: this node runs a partition's
3885    // projector while it holds the partition's home bundle, as it serves the
3886    // partition's task queues (`projection-passes` criterion 2.5).
3887    let projectors = ProjectorSupervisor {
3888        log: projection_log.clone(),
3889        sink: projection_sink.clone(),
3890        partitions: effective_config.infrastructure.placement.partition_count,
3891        fanout: 1,
3892        page_size: tokeira_projection::PAGE_SIZE,
3893        homes: {
3894            let runtime = runtime.clone();
3895            move |partition| {
3896                runtime.partition_home(partition).map(|home| ProjectorHome {
3897                    generation: home.generation,
3898                    cancel: home.cancel,
3899                })
3900            }
3901        },
3902    };
3903    let _projectors = engine_tasks.spawn(projectors.run(background_cancel.clone()));
3904
3905    let nexus_http_waiters = tokeira_edge::nexus_http::NexusHttpWaiterRegistry::default();
3906    let workflow_service =
3907        WorkflowService::new_with_stores_and_buffered_queries_and_history_wait_registry(
3908            runtime_adapter.clone(),
3909            resolver,
3910            visibility,
3911            repo.clone(),
3912            operator_api.clone(),
3913            namespaces.clone(),
3914            interceptors.clone(),
3915            PollerRegistry::default(),
3916            PendingQueryStore::default(),
3917            buffered_queries,
3918            workflow_broker,
3919            nexus_task_broker.clone(),
3920            long_polls,
3921            router,
3922            history_waits,
3923            worker_registry,
3924            runtime.heartbeat_store(),
3925            schedule_store,
3926            task_queue_config_store,
3927            Arc::new(tokeira_runtime::BatchOperationStore::default()),
3928        )
3929        .with_nexus_http_waiters(nexus_http_waiters.clone())
3930        .with_worker_task_provenance(worker_task_provenance)
3931        .with_scoped_worker_sessions(ScopedWorkerSessionRegistry::default())
3932        .with_worker_deployment_runtime(runtime_adapter);
3933    // The Nexus endpoint admin shares the dispatch store, gated through the operator
3934    // interceptor (create/update/delete = OperatorWrite, get/list = OperatorRead).
3935    // Limits come from config (raise, never hardcode); the namespace resolver backs
3936    // the Worker-target existence check.
3937    let nexus_endpoint_limits = {
3938        let cfg = &effective_config.policy.nexus_endpoint_limits;
3939        tokeira_edge::nexus_endpoint::NexusEndpointLimits {
3940            name_max_length: cfg.name_max_length,
3941            external_url_max_length: cfg.external_url_max_length,
3942            description_max_size: cfg.description_max_size,
3943            task_queue_max_length: cfg.task_queue_max_length,
3944            list_default_page_size: cfg.list_default_page_size,
3945            list_max_page_size: cfg.list_max_page_size,
3946        }
3947    };
3948    let nexus_endpoint_admin = Arc::new(tokeira_edge::nexus_endpoint::NexusEndpointAdmin::new(
3949        nexus_store.clone(),
3950        Arc::new(tokeira_edge::nexus_endpoint::CacheNamespaceResolver::new(
3951            namespaces.clone(),
3952        )),
3953        nexus_endpoint_limits,
3954    ));
3955    let dispatch_grpc = tokeira_edge::grpc::workflow_dispatch::WorkflowTaskDispatchGrpc::new(
3956        runtime.clone(),
3957        interceptors.clone(),
3958        workflow_service.clone(),
3959    )
3960    .with_peer(peer.clone());
3961    let activity_dispatch_grpc =
3962        tokeira_edge::grpc::activity_dispatch::ActivityTaskDispatchGrpc::new(
3963            runtime.clone(),
3964            interceptors.clone(),
3965            peer.clone(),
3966        );
3967    let outbound_grpc =
3968        tokeira_edge::grpc::outbound_delivery::OutboundDeliveryGrpc::new(runtime.clone(), peer);
3969    let operator_service = OperatorService::new(operator_api, interceptors)
3970        .with_nexus_endpoints(nexus_endpoint_admin)
3971        .with_namespace_deletion(namespaces.clone(), Arc::new(workflow_service.clone()));
3972    let production_nexus_authorizer = authorization.nexus;
3973    let nexus_http_authorizer: Arc<dyn tokeira_edge::nexus_http::NexusHttpAuthorizer> =
3974        match harness::installed().and_then(|hooks| hooks.wrap_nexus_http_authorizer.as_ref()) {
3975            Some(wrap) => wrap(production_nexus_authorizer),
3976            None => production_nexus_authorizer,
3977        };
3978    let nexus_http_layer =
3979        NexusHttpLayer::new(Arc::new(tokeira_edge::nexus_http::NexusHttpHandler::new(
3980            namespaces.clone(),
3981            nexus_store,
3982            nexus_task_broker,
3983            nexus_http_waiters,
3984            nexus_http_authorizer,
3985        )));
3986    let http_api_layer = HttpApiLayer::new(
3987        HttpApiCatalog::pinned().context("failed to build Temporal HTTP API route catalog")?,
3988        HttpApiPolicy::new(
3989            effective_config.policy.http_api.allowed_hosts.clone(),
3990            effective_config
3991                .policy
3992                .http_api
3993                .additional_forwarded_headers
3994                .clone(),
3995        )
3996        .context("failed to compile Temporal HTTP API policy")?,
3997    );
3998
3999    // Wire the standalone-activity (CHASM) bridge onto the gRPC adapter: a CHASM
4000    // engine over the backend's node repository, an activity-library registry, and
4001    // a shared dispatch queue the engine routes committed dispatch tasks into and a
4002    // worker poll drains. The enable gate is operator config — off by default, so an
4003    // unconfigured server matches the `v1.31.0` baseline (RPCs answer
4004    // `UNIMPLEMENTED`); enabling it is a declared deviation (`AGENTS §8`).
4005    let (workflow_grpc, chasm_engine) = {
4006        // Capture what the visibility repair scanner needs before the engine consumes
4007        // the node repo + registry: the activity id selects the legacy evaluator;
4008        // registered root adapters rebuild visibility for every archetype.
4009        let repair_archetype = registry.archetype_id(
4010            <tokeira_chasm_activity::ActivityExecution as tokeira_chasm::Component>::FQN,
4011        );
4012        // Finish pointer migration before the CHASM engine can admit a start. The
4013        // driver skips completed backfills and preserves any newer scoped pointer.
4014        tokeira_storage::run_current_execution_backfill(
4015            chasm_node_repo.as_ref(),
4016            repair_archetype.context("activity CHASM archetype is not registered")?,
4017            500,
4018        )
4019        .await
4020        .context("failed to backfill CHASM current executions")?;
4021        check_registered_archetypes(&chasm_node_repo.distinct_archetypes().await?, &registry)
4022            .map_err(registry_start_error)?;
4023        let repair_nodes = chasm_node_repo.clone();
4024        let dispatch_queue = Arc::new(tokeira_edge::chasm_activity::ActivityDispatchQueue::new());
4025        let multiplexer = Arc::new(tokeira_runtime::chasm::DispatchMultiplexer::default());
4026        // Standalone activities flow into the shared visibility index via the
4027        // engine→projection adapter, post-commit and off the correctness path
4028        // (spec task 24.2). It reuses the same projection apply path as the workflow
4029        // visibility worker, so both archetypes land in one logical index.
4030        let chasm_visibility_sink =
4031            Arc::new(tokeira_runtime::chasm::ProjectionVisibilitySink::new(
4032                Arc::new(projection_sink()),
4033                effective_config.infrastructure.placement.partition_count,
4034            ));
4035        let mut chasm_engine = tokeira_runtime::chasm::ChasmEngine::new(
4036            chasm_node_repo.clone(),
4037            registry.clone(),
4038            multiplexer.clone(),
4039            chasm_visibility_sink,
4040        );
4041        if let Some(clock) = extensions.clock {
4042            chasm_engine = chasm_engine.with_clock(clock);
4043        }
4044        let chasm_engine = Arc::new(chasm_engine);
4045        let activity_config = tokeira_chasm_activity::ActivityConfig {
4046            enable_standalone: effective_config
4047                .policy
4048                .compatibility
4049                .enable_standalone_activities,
4050            enable_callbacks: effective_config
4051                .policy
4052                .compatibility
4053                .enable_standalone_activity_callbacks,
4054            ..tokeira_chasm_activity::ActivityConfig::default()
4055        };
4056        let standalone_enabled = activity_config.enable_standalone;
4057        // Keep an engine handle for the timer sweeper before the bridge takes it.
4058        let sweeper_engine = chasm_engine.clone();
4059        let dispatch_executor = Arc::new(
4060            tokeira_edge::chasm_executors::ActivityDispatchExecutor::new(
4061                Arc::downgrade(&chasm_engine),
4062                dispatch_queue,
4063            ),
4064        );
4065        let activity_bridge = Arc::new(
4066            tokeira_edge::chasm_activity::ActivityBridge::new(
4067                chasm_engine.clone(),
4068                activity_config.clone(),
4069                DEFAULT_MAX_ID_LENGTH,
4070            )
4071            .with_dispatch_executor(dispatch_executor.clone()),
4072        );
4073        // Weak engine handles break the ownership cycle. Register every role before
4074        // rebuild/serve; extension executors join the same multiplexer below.
4075        multiplexer
4076            .register(dispatch_executor)
4077            .map_err(registry_start_error)?;
4078        multiplexer
4079            .register(Arc::new(
4080                tokeira_edge::chasm_executors::StartActivityExecutor::new(
4081                    Arc::downgrade(&chasm_engine),
4082                    activity_config,
4083                    DEFAULT_MAX_ID_LENGTH,
4084                ),
4085            ))
4086            .map_err(registry_start_error)?;
4087        multiplexer
4088            .register(Arc::new(
4089                tokeira_edge::chasm_executors::DeliverCallbackExecutor::new(
4090                    Arc::downgrade(&chasm_engine),
4091                    chasm_nexus_client,
4092                    chasm_nexus_config,
4093                    namespaces.clone(),
4094                ),
4095            ))
4096            .map_err(registry_start_error)?;
4097        for executor in extensions.executors {
4098            multiplexer
4099                .register(executor)
4100                .map_err(registry_start_error)?;
4101        }
4102        // Resolve before registering: projection registration alone silently
4103        // preserves an existing id even if its type differs from the declaration.
4104        for namespace in namespaces.list_all().await? {
4105            seed_declared_search_attributes(
4106                &visibility_query_store,
4107                namespace_id_for(&namespace.name),
4108                &declared,
4109            )
4110            .await
4111            .map_err(seed_start_error)?;
4112        }
4113        let rebuild = tokeira_runtime::chasm::OutboxRebuildScanner::new(
4114            chasm_node_repo,
4115            chasm_engine.clone(),
4116            multiplexer.clone(),
4117        );
4118        let rebuild_stats = rebuild
4119            .rebuild_once()
4120            .await
4121            .context("failed to rebuild CHASM outboxes before serving")?;
4122        check_rebuilt_outboxes(rebuild_stats)?;
4123        spawn_outbox_rebuild(&engine_tasks, rebuild, background_cancel.clone());
4124        // Spawn the visibility repair scanner (Req 10.11): a committed transition can
4125        // never permanently lack a projection — the scanner rebuilds each execution's
4126        // snapshot from authoritative node state and re-applies it iff-newer. This is
4127        // the AGENTS-3 sweeper shape (derived effect reconstructed from authoritative
4128        // state), the durability backstop for the best-effort post-commit write (24.2).
4129        spawn_visibility_repair(
4130            &engine_tasks,
4131            repair_nodes,
4132            Arc::new(projection_sink()),
4133            registry.clone(),
4134            effective_config.infrastructure.placement.partition_count,
4135            background_cancel.clone(),
4136        );
4137        // Spawn the CHASM timer sweeper (`chasm-activity-timeouts-and-retry`): nothing
4138        // else fires armed activity timeouts. A conformance harness must keep it ready
4139        // even when the boot-time default is off, because the corpus enables
4140        // `activity.enableStandalone` live after server startup. Production retains
4141        // the configured gate; registered extension libraries also need generic timers.
4142        let sweeper_forced =
4143            harness::installed().is_some_and(|hooks| hooks.force_chasm_timer_sweeper);
4144        if standalone_enabled || sweeper_forced || has_extensions {
4145            spawn_chasm_timer_sweeper(
4146                &engine_tasks,
4147                sweeper_engine,
4148                activity_bridge.clone(),
4149                repair_archetype.context("activity CHASM archetype is not registered")?,
4150                background_cancel.clone(),
4151            );
4152        }
4153        (
4154            WorkflowServiceGrpc::new(workflow_service.clone()).with_chasm_activity(activity_bridge),
4155            chasm_engine,
4156        )
4157    };
4158    // Minimal AdminService (DescribeMutableState) shares the WorkflowService's
4159    // run repository — the reset conformance suite reads a run's ResetRunId/status.
4160    let admin_grpc = AdminServiceGrpc::new(workflow_service);
4161    let operator_grpc = OperatorServiceGrpc::new(operator_service);
4162
4163    // Embedded startup defers self-assignment until every other fallible stack
4164    // construction step has completed. Admission is not opened until all
4165    // configured leases are held, and the returned cleanup owns the exact
4166    // `(shard, owner, epoch)` tuples needed for conditional release.
4167    let shard_cleanup = if matches!(transport, StackTransport::Embedded)
4168        && dsql_endpoint.is_some()
4169        && effective_config
4170            .infrastructure
4171            .placement
4172            .controller_endpoint
4173            .is_none()
4174    {
4175        let leases = self_assign_dsql_shards(
4176            runtime.as_ref(),
4177            repo.as_ref(),
4178            effective_config.infrastructure.placement.shard_count,
4179            &node_id,
4180            &node_endpoint,
4181            true,
4182        )
4183        .await?;
4184        Some(self_assigned_shard_cleanup(
4185            runtime.clone(),
4186            repo.clone(),
4187            leases,
4188        ))
4189    } else {
4190        None
4191    };
4192
4193    match dsql_endpoint {
4194        Some(endpoint) => info!(%endpoint, "storage backend: dsql"),
4195        None => info!("storage backend: in-memory"),
4196    }
4197
4198    let log_broadcast = broadcast::Sender::<LogEvent>::new(LOG_BROADCAST_CAPACITY);
4199    let addr = match transport {
4200        StackTransport::Embedded => {
4201            startup_guard.disarm();
4202            return Ok(ConstructedStack::Embedded(EmbeddedStack {
4203                component_visibility_store: chasm_extensions::VisibilityQueryStore(Arc::new(
4204                    visibility_query_store,
4205                )),
4206                chasm_engine,
4207                registry,
4208                service: InProcessGrpcService::new(workflow_grpc, operator_grpc, admin_grpc),
4209                background_cancel,
4210                log_broadcast,
4211                recovery_task,
4212                runtime_tasks,
4213                engine_tasks,
4214                shard_cleanup,
4215            }));
4216        }
4217        StackTransport::Network(addr) => addr,
4218    };
4219
4220    // Both reflection protocols are served: `v1alpha` is what the previous
4221    // stack offered and older tooling still asks for, `v1` is the current
4222    // protocol newer tooling tries first.
4223    let reflection_v1 = tonic_reflection::server::Builder::configure()
4224        .register_encoded_file_descriptor_set(tokeira_proto::public::FILE_DESCRIPTOR_SET)
4225        .build_v1()
4226        .context("failed to build gRPC reflection service")?;
4227    let reflection_v1alpha = tonic_reflection::server::Builder::configure()
4228        .register_encoded_file_descriptor_set(tokeira_proto::public::FILE_DESCRIPTOR_SET)
4229        .build_v1alpha()
4230        .context("failed to build gRPC reflection service")?;
4231
4232    let listener = TcpListener::bind(addr)
4233        .await
4234        .with_context(|| format!("failed to bind tokeirad gRPC listener on {addr}"))?;
4235    let bound_addr = listener
4236        .local_addr()
4237        .context("failed to resolve bound local address for tokeirad gRPC listener")?;
4238
4239    let (shutdown_tx, shutdown_rx) = oneshot::channel();
4240
4241    // Tier-2 functional conformance (see `.kiro/specs/temporal-functional-conformance`)
4242    // captures every served `(wire_method, status_code)` at the transport boundary. The
4243    // capturing tower layer is mounted ONLY when the conformance flag is set, so a normal
4244    // production server never installs it and pays zero per-call cost.
4245    //
4246    // The recorder is constructed here, BEFORE the server task is spawned, and an
4247    // `Arc::clone` is moved into the task for the layer while the original handle is
4248    // returned to the caller. This is what lets the caller snapshot the observed coverage
4249    // after the server has shut down (the layer's clone is dropped with the task, but the
4250    // returned `Arc` keeps the counts alive) and export it as JSON evidence — see
4251    // [`run_from_cli`]. When the flag is off the handle is `None`: nothing is mounted,
4252    // nothing is snapshotted, and there is zero behavioural change.
4253    let wire_coverage_recorder = wire_coverage_enabled().then(|| {
4254        info!("conformance wire-coverage recorder enabled");
4255        Arc::new(WireCoverageRecorder::new())
4256    });
4257    let server_recorder = wire_coverage_recorder.clone();
4258
4259    let server_cancel = background_cancel.clone();
4260    let server_task = tokio::spawn(async move {
4261        let shutdown_signal = async move {
4262            tokio::select! {
4263                _ = shutdown_rx => {},
4264                _ = server_cancel.cancelled() => {},
4265            }
4266        };
4267
4268        // The conformance layer changes the server's tower-stack type, so the flagged and
4269        // unflagged paths are distinct server builds rather than a conditionally-mutated
4270        // builder. They are otherwise identical; only the extra `.layer()` differs.
4271        match server_recorder {
4272            Some(recorder) => {
4273                Server::builder()
4274                    .accept_http1(true)
4275                    .layer(nexus_http_layer.clone())
4276                    .layer(http_api_layer.clone())
4277                    .layer(CorsLayer::permissive())
4278                    .layer(GrpcWebLayer::new())
4279                    .layer(WireCoverageLayer::new(recorder))
4280                    .add_service(workflow_grpc.into_service())
4281                    .add_service(operator_grpc.into_service())
4282                    .add_service(admin_grpc.into_service())
4283                    .add_service(dispatch_grpc.into_service())
4284                    .add_service(activity_dispatch_grpc.into_service())
4285                    .add_service(outbound_grpc.into_service())
4286                    .add_service(reflection_v1)
4287                    .add_service(reflection_v1alpha)
4288                    .serve_with_incoming_shutdown(TcpListenerStream::new(listener), shutdown_signal)
4289                    .await
4290            }
4291            None => {
4292                Server::builder()
4293                    .accept_http1(true)
4294                    .layer(nexus_http_layer)
4295                    .layer(http_api_layer)
4296                    .layer(CorsLayer::permissive())
4297                    .layer(GrpcWebLayer::new())
4298                    .add_service(workflow_grpc.into_service())
4299                    .add_service(operator_grpc.into_service())
4300                    .add_service(admin_grpc.into_service())
4301                    .add_service(dispatch_grpc.into_service())
4302                    .add_service(activity_dispatch_grpc.into_service())
4303                    .add_service(outbound_grpc.into_service())
4304                    .add_service(reflection_v1)
4305                    .add_service(reflection_v1alpha)
4306                    .serve_with_incoming_shutdown(TcpListenerStream::new(listener), shutdown_signal)
4307                    .await
4308            }
4309        }
4310        .with_context(|| format!("failed to serve gRPC transport on {bound_addr}"))?;
4311        Ok::<(), anyhow::Error>(())
4312    });
4313
4314    // Harness-owned background work (the conformance dynamic-config control
4315    // listener, spec `.kiro/specs/conformance-config-override/`) spawns at the
4316    // same point the stack's own background tasks do, tied to the same
4317    // cancellation token so a graceful drain also stops the harness side.
4318    // Production installs no hooks, so this is a no-op outside a harness.
4319    if let Some(task) = harness::installed().and_then(|hooks| hooks.background_task.as_ref()) {
4320        task(background_cancel.clone());
4321    }
4322
4323    startup_guard.disarm();
4324    Ok(ConstructedStack::Network((
4325        server_task,
4326        bound_addr,
4327        shutdown_tx,
4328        background_cancel,
4329        log_broadcast,
4330        wire_coverage_recorder,
4331    )))
4332}
4333
4334/// Replace the port in a `scheme://host[:port]` loopback base URL with the actually-bound
4335/// listener `port`. This keeps `system_callback_url` (scheme + host from config) pointed at
4336/// the listener even when `http_addr` requested an ephemeral `:0` port.
4337fn with_loopback_port(base: &str, port: u16) -> String {
4338    let trimmed = base.trim_end_matches('/');
4339    match trimmed.rsplit_once(':') {
4340        // Only treat the suffix as a port when it is all digits — otherwise the match is
4341        // the `:` inside the scheme (`http://host` with no port) and we append instead.
4342        Some((prefix, suffix))
4343            if !suffix.is_empty() && suffix.bytes().all(|b| b.is_ascii_digit()) =>
4344        {
4345            format!("{prefix}:{port}")
4346        }
4347        _ => format!("{trimmed}:{port}"),
4348    }
4349}
4350
4351/// Serve the inbound `POST /nexus/callback` completion endpoint on `listener` until
4352/// `cancel` fires. Each request is delegated to [`handle_nexus_callback`], which resolves
4353/// the originator workflow's pending operation — the inbound half of the async-completion
4354/// loopback the runtime's firing client drives.
4355fn spawn_nexus_callback_server(
4356    listener: TcpListener,
4357    runtime: Arc<dyn WorkflowRuntimeApi>,
4358    cancel: CancellationToken,
4359) {
4360    tokio::spawn(async move {
4361        loop {
4362            let stream = tokio::select! {
4363                _ = cancel.cancelled() => break,
4364                accept = listener.accept() => match accept {
4365                    Ok((stream, _peer)) => stream,
4366                    Err(error) => {
4367                        tracing::warn!(?error, "nexus callback listener accept failed");
4368                        continue;
4369                    }
4370                },
4371            };
4372            let io = TokioIo::new(stream);
4373            let runtime = runtime.clone();
4374            let conn_cancel = cancel.clone();
4375            tokio::spawn(async move {
4376                let service = service_fn(move |request| {
4377                    let runtime = runtime.clone();
4378                    async move {
4379                        Ok::<_, Infallible>(
4380                            nexus_callback_response(runtime.as_ref(), request).await,
4381                        )
4382                    }
4383                });
4384                let conn = hyper::server::conn::http1::Builder::new().serve_connection(io, service);
4385                tokio::pin!(conn);
4386                tokio::select! {
4387                    result = conn.as_mut() => {
4388                        if let Err(error) = result {
4389                            tracing::debug!(?error, "nexus callback connection closed with error");
4390                        }
4391                    }
4392                    _ = conn_cancel.cancelled() => {
4393                        conn.as_mut().graceful_shutdown();
4394                        let _ = conn.await;
4395                    }
4396                }
4397            });
4398        }
4399    });
4400}
4401
4402/// Outcome of reading a request body under a byte cap (see [`collect_bounded_body`]).
4403#[derive(Debug)]
4404enum BoundedBody {
4405    /// The body fit within the cap and was fully buffered.
4406    Collected(Bytes),
4407    /// The body exceeded the cap; collection stopped early. Maps to `413`.
4408    TooLarge,
4409    /// The underlying stream errored before completing. Maps to `400`.
4410    ReadFailed(Box<dyn std::error::Error + Send + Sync>),
4411}
4412
4413/// Buffer `body` into memory, refusing to accumulate more than `limit` bytes.
4414///
4415/// [`Limited`] stops polling once the running total would exceed `limit` and surfaces a
4416/// downcastable [`LengthLimitError`], letting the caller reject an oversized body with
4417/// `413` *before* it is fully read — the bound that keeps the network-exposed
4418/// `/nexus/callback` listener from being an unauthenticated memory-exhaustion vector.
4419async fn collect_bounded_body<B>(body: B, limit: usize) -> BoundedBody
4420where
4421    B: hyper::body::Body<Data = Bytes>,
4422    B::Error: Into<Box<dyn std::error::Error + Send + Sync>>,
4423{
4424    match Limited::new(body, limit).collect().await {
4425        Ok(collected) => BoundedBody::Collected(collected.to_bytes()),
4426        Err(error) if error.downcast_ref::<LengthLimitError>().is_some() => BoundedBody::TooLarge,
4427        Err(error) => BoundedBody::ReadFailed(error),
4428    }
4429}
4430
4431/// Adapt a hyper request to the transport-agnostic edge handler. Non-`POST` requests and
4432/// any path other than `/nexus/callback` are 404; the handler owns all other status
4433/// mapping (decode failures → 400, accepted → 200, not-found/stale → 404, internal → 503).
4434async fn nexus_callback_response(
4435    runtime: &dyn WorkflowRuntimeApi,
4436    request: Request<Incoming>,
4437) -> Response<Full<Bytes>> {
4438    let (parts, body) = request.into_parts();
4439
4440    if parts.method != Method::POST || parts.uri.path() != NEXUS_CALLBACK_PATH {
4441        return nexus_response(StatusCode::NOT_FOUND, None);
4442    }
4443
4444    let header = |name: &str| {
4445        parts
4446            .headers
4447            .get(name)
4448            .and_then(|value| value.to_str().ok())
4449            .map(str::to_owned)
4450    };
4451    let token = header(TEMPORAL_CALLBACK_TOKEN_HEADER);
4452    let operation_token = header(NEXUS_OPERATION_TOKEN_HEADER);
4453    let state = header(NEXUS_OPERATION_STATE_HEADER);
4454    let content_type = header(hyper::header::CONTENT_TYPE.as_str());
4455
4456    // Bound the body before buffering it. This listener is network-exposed (default
4457    // bind is loopback, but operators may widen it) and reads the whole body *before*
4458    // `handle_nexus_callback` validates the callback token, so an unbounded `collect()`
4459    // is an unauthenticated memory-exhaustion vector — and the process runs
4460    // `panic = "abort"`, so an OOM abort drops every in-flight workflow on the node.
4461    // Mirror the cap the caller-facing Nexus transport already enforces.
4462    let body_bytes = match collect_bounded_body(body, MAX_NEXUS_PAYLOAD_BYTES).await {
4463        BoundedBody::Collected(bytes) => bytes,
4464        BoundedBody::TooLarge => {
4465            tracing::debug!(
4466                limit = MAX_NEXUS_PAYLOAD_BYTES,
4467                "nexus callback body too large"
4468            );
4469            return nexus_response(
4470                StatusCode::PAYLOAD_TOO_LARGE,
4471                Some(b"request body too large".to_vec()),
4472            );
4473        }
4474        BoundedBody::ReadFailed(error) => {
4475            tracing::debug!(?error, "failed to read nexus callback request body");
4476            return nexus_response(
4477                StatusCode::BAD_REQUEST,
4478                Some(b"failed to read request body".to_vec()),
4479            );
4480        }
4481    };
4482
4483    let CallbackResponse { status, body } = handle_nexus_callback(
4484        runtime,
4485        token.as_deref(),
4486        operation_token.as_deref(),
4487        state.as_deref(),
4488        content_type.as_deref(),
4489        &body_bytes,
4490    )
4491    .await;
4492
4493    let status = StatusCode::from_u16(status).unwrap_or(StatusCode::INTERNAL_SERVER_ERROR);
4494    nexus_response(status, body)
4495}
4496
4497fn nexus_response(status: StatusCode, body: Option<Vec<u8>>) -> Response<Full<Bytes>> {
4498    let mut response = Response::new(Full::new(Bytes::from(body.unwrap_or_default())));
4499    *response.status_mut() = status;
4500    response
4501}
4502
4503/// Broadcast buffer for [`LogEvent`]s fanned out to test harnesses.
4504///
4505/// Dimensioned so a slow consumer does not drop observation events under
4506/// typical integration-test cadence (one RPC per worker tick at ~30s).
4507const LOG_BROADCAST_CAPACITY: usize = 256;
4508
4509/// Environment flag selecting the Tier-2 wire-coverage capture layer.
4510///
4511/// Set during a functional-conformance run (see
4512/// `.kiro/specs/temporal-functional-conformance`) so the gRPC server mounts the
4513/// `WireCoverageLayer`. Read as a single env var rather than threaded through
4514/// `TokeiraConfig` because this is a conformance-harness concern, not a server-config
4515/// surface, and must add nothing to the production config schema.
4516const WIRE_COVERAGE_ENV: &str = "TOKEIRA_CONFORMANCE_WIRE_COVERAGE";
4517
4518/// Environment variable naming the file the wire-coverage evidence is written to.
4519///
4520/// Read only when the wire-coverage layer is active (see [`WIRE_COVERAGE_ENV`]). Kept a
4521/// separate var from the enable flag so an operator can leave the output location pinned
4522/// across runs while toggling capture on and off, and so the default path is never
4523/// silently inferred from the enable value. Like [`WIRE_COVERAGE_ENV`] it is read as a
4524/// single env var rather than threaded through `TokeiraConfig`, because the export path is
4525/// a conformance-harness concern, not a production server-config surface.
4526const WIRE_COVERAGE_OUT_ENV: &str = "TOKEIRA_CONFORMANCE_WIRE_COVERAGE_OUT";
4527
4528/// Default wire-coverage evidence path used when capture is on but
4529/// [`WIRE_COVERAGE_OUT_ENV`] is unset.
4530///
4531/// A repo-relative `./wire-coverage.json` is chosen so an operator who enables capture
4532/// without naming a path still gets evidence in the working directory rather than nothing;
4533/// the report can then be pointed at the default. It is never used when capture is off.
4534const WIRE_COVERAGE_DEFAULT_OUT: &str = "./wire-coverage.json";
4535
4536/// Whether the Tier-2 wire-coverage layer should be mounted.
4537///
4538/// True only when [`WIRE_COVERAGE_ENV`] is present and not one of the empty/false-y
4539/// values, so an accidentally-exported empty variable does not silently enable capture in
4540/// production. Any other value (e.g. `1`, `true`, `on`) enables it.
4541fn wire_coverage_enabled() -> bool {
4542    match std::env::var(WIRE_COVERAGE_ENV) {
4543        Ok(value) => {
4544            let trimmed = value.trim();
4545            !trimmed.is_empty()
4546                && !trimmed.eq_ignore_ascii_case("0")
4547                && !trimmed.eq_ignore_ascii_case("false")
4548        }
4549        Err(_) => false,
4550    }
4551}
4552
4553/// Resolve the path the wire-coverage evidence is written to.
4554///
4555/// Returns the value of [`WIRE_COVERAGE_OUT_ENV`] when it is set to a non-empty value,
4556/// otherwise [`WIRE_COVERAGE_DEFAULT_OUT`]. An empty or whitespace-only override is
4557/// treated as unset rather than as a request to write to a blank path, mirroring the
4558/// false-y handling in [`wire_coverage_enabled`] so an accidentally-exported empty
4559/// variable falls back to the documented default instead of erroring.
4560fn wire_coverage_out_path() -> PathBuf {
4561    match std::env::var(WIRE_COVERAGE_OUT_ENV) {
4562        Ok(value) if !value.trim().is_empty() => PathBuf::from(value.trim()),
4563        _ => PathBuf::from(WIRE_COVERAGE_DEFAULT_OUT),
4564    }
4565}
4566
4567/// Build the shared Nexus endpoint store, seeding any bootstrap-configured
4568/// endpoints. The same store backs both runtime dispatch resolution (via
4569/// [`NexusEndpointRegistry`]) and the OperatorService endpoint-admin CRUD, so a
4570/// `CreateNexusEndpoint` at runtime is immediately resolvable for dispatch and a
4571/// seeded endpoint is visible to admin reads. Worker targets resolve the namespace
4572/// name to its id up front (failing if the namespace is unknown), storing both so
4573/// reads echo the name and dispatch routes on the id.
4574async fn build_nexus_endpoint_store(
4575    namespaces: &dyn NamespaceCache,
4576    configs: HashMap<String, BootstrapNexusEndpointConfig>,
4577) -> Result<Arc<InMemoryNexusEndpointStore>> {
4578    let store = Arc::new(InMemoryNexusEndpointStore::new());
4579    let now = std::time::SystemTime::now()
4580        .duration_since(std::time::UNIX_EPOCH)
4581        .map(|d| d.as_nanos() as i64)
4582        .unwrap_or(0);
4583    for (endpoint_name, config) in configs {
4584        let target = match config.target {
4585            BootstrapNexusEndpointTarget::External { address } => {
4586                NexusEndpointSpecTarget::External { url: address }
4587            }
4588            BootstrapNexusEndpointTarget::Worker {
4589                namespace_name,
4590                task_queue,
4591            } => {
4592                let namespace = namespaces.get(&namespace_name).await?.ok_or_else(|| {
4593                    anyhow!(
4594                        "failed to register nexus worker endpoint `{endpoint_name}`: namespace `{namespace_name}` not found"
4595                    )
4596                })?;
4597                NexusEndpointSpecTarget::Worker {
4598                    namespace_name: namespace.name.clone(),
4599                    namespace_id: namespace_id_for(&namespace.name).0.to_string(),
4600                    task_queue,
4601                }
4602            }
4603        };
4604        store
4605            .create(
4606                NexusEndpointSpec {
4607                    name: endpoint_name.clone(),
4608                    description: Vec::new(),
4609                    target,
4610                },
4611                now,
4612            )
4613            .map_err(|e| anyhow!("failed to seed nexus endpoint `{endpoint_name}`: {e}"))?;
4614    }
4615    Ok(store)
4616}
4617
4618async fn self_assign_dsql_shards<R>(
4619    runtime: &TokeiraRuntime<R>,
4620    lease_repository: &R,
4621    shard_count: u32,
4622    node_id: &IncarnationId,
4623    node_endpoint: &NodeEndpoint,
4624    require_all: bool,
4625) -> Result<Vec<SelfAssignedShardLease>>
4626where
4627    R: LeaseRepository + RunRepository + 'static,
4628{
4629    // In single-node compose mode (no controller), any existing leases are stale
4630    // by definition — there is no other node. Relinquish all existing leases first
4631    // so try_acquire_bundle succeeds regardless of prior owner or expiry.
4632    let existing_leases = lease_repository.list_bundle_leases().await?;
4633    for lease in &existing_leases {
4634        if let Some(owner) = &lease.owner_node_id {
4635            let _ = lease_repository
4636                .relinquish_bundle(lease.bundle_id, owner.clone(), lease.epoch)
4637                .await;
4638        }
4639    }
4640
4641    let mut acquired = Vec::new();
4642    let mut recovered = SweepResult::default();
4643    let owner = node_id.to_string();
4644    for shard_index in 0..shard_count {
4645        let shard_id = ShardId(shard_index);
4646        match lease_repository
4647            .try_acquire_bundle(shard_id, owner.clone(), node_endpoint.as_authority())
4648            .await
4649        {
4650            Ok(LeaseOutcome::Acquired { epoch } | LeaseOutcome::Renewed { epoch }) => {
4651                // Being the only owner removes the need for a lease renewer,
4652                // not for recovery: the previous process's offered workflow
4653                // tasks and timeout tracking ended with it, so the shard is
4654                // swept before it admits anything (runtime-sweeper-recovery
4655                // Requirement 11.6).
4656                match runtime.recover_self_assigned_shard(shard_id, epoch).await {
4657                    Ok(swept) => {
4658                        recovered.workflow_tasks_republished += swept.workflow_tasks_republished;
4659                        recovered.activity_tasks_republished += swept.activity_tasks_republished;
4660                        recovered.due_timers_injected += swept.due_timers_injected;
4661                        recovered.activity_tracking_entries_reconstructed +=
4662                            swept.activity_tracking_entries_reconstructed;
4663                        acquired.push(SelfAssignedShardLease {
4664                            shard_id,
4665                            owner: owner.clone(),
4666                            epoch,
4667                        });
4668                    }
4669                    Err(error) => {
4670                        // The runtime has dropped the shard; release its lease.
4671                        let _ = lease_repository
4672                            .relinquish_bundle(shard_id, owner.clone(), epoch)
4673                            .await;
4674                        if require_all {
4675                            rollback_self_assigned_shards(runtime, lease_repository, &acquired)
4676                                .await;
4677                            return Err(error).context(format!(
4678                                "failed to recover embedded DSQL shard {shard_index}"
4679                            ));
4680                        }
4681                        tracing::warn!(
4682                            shard_index,
4683                            ?error,
4684                            "failed to recover self-assigned shard; leaving it unowned"
4685                        );
4686                    }
4687                }
4688            }
4689            Ok(LeaseOutcome::Rejected {
4690                current_owner,
4691                current_epoch,
4692            }) => {
4693                if require_all {
4694                    rollback_self_assigned_shards(runtime, lease_repository, &acquired).await;
4695                    return Err(anyhow!(
4696                        "embedded DSQL shard {shard_index} is held by owner {current_owner} at epoch {}",
4697                        current_epoch.0
4698                    ));
4699                }
4700                tracing::warn!(
4701                    shard_index,
4702                    %current_owner,
4703                    current_epoch = current_epoch.0,
4704                    "failed to self-assign shard: lease is held by another owner"
4705                );
4706            }
4707            Err(error) => {
4708                if require_all {
4709                    rollback_self_assigned_shards(runtime, lease_repository, &acquired).await;
4710                    return Err(error).context(format!(
4711                        "failed to self-assign embedded DSQL shard {shard_index}"
4712                    ));
4713                }
4714                tracing::warn!(shard_index, ?error, "failed to self-assign shard");
4715            }
4716        }
4717    }
4718    info!(
4719        acquired = acquired.len(),
4720        shard_count,
4721        workflow_tasks_republished = recovered.workflow_tasks_republished,
4722        activity_tasks_republished = recovered.activity_tasks_republished,
4723        due_timers_injected = recovered.due_timers_injected,
4724        activity_tracking_entries_reconstructed = recovered.activity_tracking_entries_reconstructed,
4725        "self-assigned and recovered DSQL shards (no controller)"
4726    );
4727    Ok(acquired)
4728}
4729
4730async fn rollback_self_assigned_shards<R>(
4731    runtime: &TokeiraRuntime<R>,
4732    lease_repository: &R,
4733    leases: &[SelfAssignedShardLease],
4734) where
4735    R: LeaseRepository + RunRepository + 'static,
4736{
4737    for lease in leases.iter().rev() {
4738        runtime.relinquish_shard(lease.shard_id).await;
4739        let _ = lease_repository
4740            .relinquish_bundle(lease.shard_id, lease.owner.clone(), lease.epoch)
4741            .await;
4742    }
4743}
4744
4745fn self_assigned_shard_cleanup<R>(
4746    runtime: Arc<TokeiraRuntime<R>>,
4747    lease_repository: Arc<R>,
4748    leases: Vec<SelfAssignedShardLease>,
4749) -> CleanupAction
4750where
4751    R: LeaseRepository + RunRepository + 'static,
4752{
4753    Box::new(move |deadline| {
4754        Box::pin(async move {
4755            let mut first_error = None;
4756            for lease in leases.into_iter().rev() {
4757                runtime.relinquish_shard(lease.shard_id).await;
4758                let now = Instant::now();
4759                if now >= deadline {
4760                    first_error.get_or_insert_with(|| {
4761                        anyhow!("deadline elapsed while relinquishing embedded DSQL shards")
4762                    });
4763                    continue;
4764                }
4765                let release = tokio::time::timeout(
4766                    deadline.saturating_duration_since(now),
4767                    lease_repository.relinquish_bundle(lease.shard_id, lease.owner, lease.epoch),
4768                )
4769                .await;
4770                match release {
4771                    Ok(Ok(LeaseOutcome::Acquired { .. })) => {}
4772                    Ok(Ok(outcome)) => {
4773                        first_error.get_or_insert_with(|| {
4774                            anyhow!("embedded DSQL shard release was fenced: {outcome:?}")
4775                        });
4776                    }
4777                    Ok(Err(error)) => {
4778                        first_error.get_or_insert(error);
4779                    }
4780                    Err(_) => {
4781                        first_error.get_or_insert_with(|| {
4782                            anyhow!("deadline elapsed while relinquishing embedded DSQL shard")
4783                        });
4784                    }
4785                }
4786            }
4787            match first_error {
4788                Some(error) => Err(error),
4789                None => Ok(()),
4790            }
4791        })
4792    })
4793}
4794
4795fn configured_node_endpoint(config: &TokeiraConfig, listen_addr: SocketAddr) -> NodeEndpoint {
4796    NodeEndpoint {
4797        host: config.infrastructure.placement.node_host.clone(),
4798        port: config
4799            .infrastructure
4800            .placement
4801            .node_port
4802            .unwrap_or_else(|| listen_addr.port()),
4803    }
4804}
4805
4806fn placement_config(config: &TokeiraConfig) -> PlacementConfig {
4807    let placement = &config.infrastructure.placement;
4808    PlacementConfig {
4809        shard_count: placement.shard_count,
4810        bundle_count: placement.bundle_count,
4811        partition_count: placement.partition_count,
4812        hash_version: placement.hash_version,
4813    }
4814}
4815
4816fn membership_config(
4817    config: &TokeiraConfig,
4818    node_id: IncarnationId,
4819    node_endpoint: NodeEndpoint,
4820    controller_endpoint: String,
4821) -> MembershipConfig {
4822    let placement = &config.infrastructure.placement;
4823    MembershipConfig {
4824        controller_endpoint,
4825        heartbeat_interval: std::time::Duration::from_millis(placement.heartbeat_interval_ms),
4826        reconnect_base_delay: std::time::Duration::from_millis(placement.reconnect_base_delay_ms),
4827        reconnect_max_delay: std::time::Duration::from_millis(placement.reconnect_max_delay_ms),
4828        node_id,
4829        node_endpoint,
4830        zone: None,
4831        version: tokeira_build_info::TOKEIRA_VERSION.to_string(),
4832        build_id: tokeira_build_info::TOKEIRA_GIT_SHA.to_string(),
4833    }
4834}
4835
4836/// Budget applier for stacks without a connection reservoir (the in-memory
4837/// store): directives are accepted and discarded, and no headroom is reported.
4838#[derive(Debug)]
4839struct NoopConnectionBudgetApplier;
4840
4841impl ConnectionBudgetApplier for NoopConnectionBudgetApplier {
4842    fn apply_budget(
4843        &self,
4844        _rate_per_second: f64,
4845        _capacity: u64,
4846        _max_reservoir_size: u32,
4847    ) -> Result<()> {
4848        Ok(())
4849    }
4850
4851    fn available_connections(&self) -> u32 {
4852        0
4853    }
4854}
4855
4856/// Applies controller connection budgets to the DSQL connection director the
4857/// whole stack shares, and reports its warm headroom in heartbeats.
4858///
4859/// Only the reservoir cap is applied. The directive's `rate_per_second` and
4860/// `capacity` describe this node's share of the cluster creation rate, but on
4861/// the distributed path the creation-rate limiter is the DynamoDB token bucket
4862/// every node shares, which already enforces the cluster rate as one row: a
4863/// node writing its per-node share into that row would replace the cluster
4864/// rate with a fraction of it. A process-local governor in front of the shared
4865/// bucket is the mechanism the share would drive, and does not exist yet.
4866#[derive(Debug)]
4867struct DsqlConnectionBudgetApplier {
4868    director: Arc<DsqlConnectionDirector>,
4869}
4870
4871impl ConnectionBudgetApplier for DsqlConnectionBudgetApplier {
4872    fn apply_budget(
4873        &self,
4874        rate_per_second: f64,
4875        capacity: u64,
4876        max_reservoir_size: u32,
4877    ) -> Result<()> {
4878        let retired = self.director.apply_connection_budget(max_reservoir_size)?;
4879        tracing::debug!(
4880            rate_per_second,
4881            capacity,
4882            max_reservoir_size,
4883            reservoir_target = self.director.reservoir_target(),
4884            retired,
4885            "applied controller connection budget to the DSQL reservoir"
4886        );
4887        Ok(())
4888    }
4889
4890    fn available_connections(&self) -> u32 {
4891        u32::try_from(self.director.ready_connections()).unwrap_or(u32::MAX)
4892    }
4893}
4894
4895/// Resolves executions and builds describe responses straight from the run
4896/// repository. The namespace is taken from each request (via `namespace_id_for`),
4897/// not bound at construction — a single server serves every namespace, so freezing
4898/// it to one (e.g. `default`) would make query/describe miss every other namespace's
4899/// runs while history (which derives the namespace per request) still found them.
4900struct StoreExecutionResolver<R> {
4901    repo: Arc<R>,
4902}
4903
4904impl<R> StoreExecutionResolver<R> {
4905    fn new(repo: Arc<R>) -> Self {
4906        Self { repo }
4907    }
4908}
4909
4910fn apply_reported_problem_search_attribute(
4911    search_attributes: &mut tokeira_types::SearchAttributes,
4912    problem: Option<WorkflowTaskReportedProblem>,
4913) {
4914    let Some(problem) = problem else {
4915        return;
4916    };
4917    // v1.31.0 exposes exactly these two KeywordList elements, derived from the
4918    // last NON-TRANSIENT WFT problem once AttemptsSinceLastSuccess reaches the
4919    // configured threshold — a `Failed`-flavored pair or a `TimedOut` pair,
4920    // per the persisted `LastWorkflowTaskFailure` oneof
4921    // (`mutable_state_impl.go:6478-6491 @ v1.31.0`; the timed-out cause
4922    // renders `TimeoutType.String()`, which is "StartToClose" for the only
4923    // type v1.31.0 ever stores).
4924    let entries = match &problem.problem {
4925        tokeira_kernel::WorkflowTaskProblem::Failed(cause) => vec![
4926            "category=WorkflowTaskFailed".to_string(),
4927            format!("cause=WorkflowTaskFailedCause{}", cause.as_str()),
4928        ],
4929        tokeira_kernel::WorkflowTaskProblem::TimedOutStartToClose => vec![
4930            "category=WorkflowTaskTimedOut".to_string(),
4931            "cause=WorkflowTaskTimedOutCauseStartToClose".to_string(),
4932        ],
4933    };
4934    search_attributes.0.insert(
4935        "TemporalReportedProblems".to_string(),
4936        SearchAttrValue::KeywordList(entries),
4937    );
4938}
4939
4940#[async_trait]
4941impl<R> ExecutionResolver for StoreExecutionResolver<R>
4942where
4943    R: RunRepository + 'static,
4944{
4945    async fn current_run_key(
4946        &self,
4947        namespace: &str,
4948        workflow_id: &str,
4949    ) -> Result<Option<tokeira_types::RunKey>> {
4950        let namespace_id = namespace_id_for(namespace);
4951        let result = self
4952            .repo
4953            .resolve_execution(&ExecutionRef {
4954                namespace_id,
4955                workflow_id: WorkflowId(workflow_id.to_string()),
4956                run_id: None,
4957            })
4958            .await?;
4959        if result.is_some() {
4960            return Ok(result);
4961        }
4962        self.repo
4963            .find_latest_run(namespace_id, &WorkflowId(workflow_id.to_string()))
4964            .await
4965    }
4966
4967    async fn describe_execution(
4968        &self,
4969        namespace: &str,
4970        workflow_id: &str,
4971        run_id: Option<tokeira_types::RunId>,
4972    ) -> Result<Option<WorkflowExecutionDescription>> {
4973        let namespace_id = namespace_id_for(namespace);
4974        let run_key = match self
4975            .repo
4976            .resolve_execution(&ExecutionRef {
4977                namespace_id,
4978                workflow_id: WorkflowId(workflow_id.to_string()),
4979                run_id,
4980            })
4981            .await?
4982        {
4983            Some(rk) => rk,
4984            None if run_id.is_none() => {
4985                match self
4986                    .repo
4987                    .find_latest_run(namespace_id, &WorkflowId(workflow_id.to_string()))
4988                    .await?
4989                {
4990                    Some(rk) => rk,
4991                    None => return Ok(None),
4992                }
4993            }
4994            None => return Ok(None),
4995        };
4996
4997        let (loaded, stats) = self.repo.load_run_with_stats(run_key).await?;
4998        match loaded {
4999            LoadedRun::Existing(state) => {
5000                // Describe surfaces history size and external-payload statistics
5001                // (`describeworkflow/api.go:126,166 @ v1.31.0`). Both are
5002                // state-maintained: the History Size is the statistic storage
5003                // advances with every committed batch, the same number the
5004                // continue-as-new advice and visibility read, and the payload
5005                // statistics accumulate on the kernel state. No history read
5006                // happens here (continue-as-new-advice, Requirements 7.1, 7.2).
5007                let history_size_bytes = stats.history_size_bytes;
5008                let external_payload_count = state.external_payload_count;
5009                let external_payload_size_bytes = state.external_payload_size_bytes;
5010                // Derived straight from committed kernel state: the counter
5011                // and problem identity live on the run, the threshold is read
5012                // live at derive time (Tier 3.22 `DynamicConfigChanges`).
5013                let reported_problem = reported_problem_from_state(&state);
5014                let versioning_info = state.versioning_info.clone();
5015                let most_recent_worker_version_stamp = versioning_info
5016                    .as_ref()
5017                    .and_then(|info| info.most_recent_worker_version_stamp.clone());
5018                let pending_nexus_operations =
5019                    tokeira_edge::translate::describe_pending_nexus_operations(&state);
5020                let mut search_attributes = state.search_attributes;
5021                apply_reported_problem_search_attribute(&mut search_attributes, reported_problem);
5022                if let Some(build_ids) = versioning_info
5023                    .as_ref()
5024                    .map(|info| &info.build_id_search_attributes)
5025                    .filter(|build_ids| !build_ids.is_empty())
5026                {
5027                    search_attributes.0.insert(
5028                        "BuildIds".to_string(),
5029                        tokeira_types::SearchAttrValue::KeywordList(build_ids.clone()),
5030                    );
5031                }
5032                Ok(Some(WorkflowExecutionDescription {
5033                    namespace: namespace.to_string(),
5034                    workflow_id: state.workflow_id.0,
5035                    run_key: state.run_key,
5036                    run_id: state.run_id,
5037                    workflow_type: state.workflow_type.0,
5038                    task_queue: state.task_queue.0.clone(),
5039                    status: state.status,
5040                    start_time: Some(state.started_at),
5041                    close_time: state.closed_at,
5042                    // v1.31.0 ExecutionTime = this run's StartTime + FirstWorkflowTaskBackoff
5043                    // (mutable_state_impl.go:2859) — NOT the chain's first-run start. tokeira
5044                    // carries that backoff (client start delay / workflow-retry backoff) as
5045                    // `workflow_start_delay`; TestWorkflowRetry asserts start + backoff per attempt.
5046                    execution_time: deadline_after(
5047                        state.started_at,
5048                        state.workflow_start_delay.unwrap_or_default(),
5049                    ),
5050                    execution_config: tokeira_edge::translate::ExecutionConfigDescription {
5051                        task_queue: state.task_queue.0.clone(),
5052                        workflow_execution_timeout: state.workflow_execution_timeout,
5053                        workflow_run_timeout: state.workflow_run_timeout,
5054                        default_workflow_task_timeout: state.workflow_task_timeout,
5055                        // Describe returns start-event metadata verbatim, even after later
5056                        // transitions (`describeworkflow/api.go:98-110 @ v1.31.0`). The kernel
5057                        // summary is replayed from that event, so no history scan is needed here.
5058                        user_metadata: state.user_metadata.map(|metadata| {
5059                            tokeira_edge::translate::UserMetadata {
5060                                summary: metadata.summary,
5061                                details: metadata.details,
5062                            }
5063                        }),
5064                    },
5065                    history_length: state.last_event_id,
5066                    history_size_bytes,
5067                    state_transition_count: state.transition_seq.0 as i64,
5068                    parent_namespace_id: state
5069                        .parent_namespace_id
5070                        .map(|namespace_id| namespace_id.0.to_string()),
5071                    parent_workflow_id: state.parent_workflow_id.clone(),
5072                    parent_run_id: state.parent_run_id,
5073                    root_workflow_id: state.root_workflow_id.clone(),
5074                    root_run_id: state.root_run_id,
5075                    first_run_id: state.first_execution_run_id,
5076                    memo: state.memo,
5077                    search_attributes,
5078                    pending_activities: state
5079                        .activities
5080                        .values()
5081                        .map(
5082                            |activity| tokeira_edge::translate::PendingActivityDescription {
5083                                activity_id: activity.activity_id.clone(),
5084                                activity_type: activity.activity_type.clone(),
5085                                is_started: activity.started_at.is_some(),
5086                                cancel_requested: activity.cancel_requested,
5087                                attempt: activity.attempt,
5088                                maximum_attempts: activity
5089                                    .retry_policy
5090                                    .as_ref()
5091                                    .map(|policy| policy.maximum_attempts)
5092                                    .unwrap_or_default(),
5093                                scheduled_at: activity.scheduled_at,
5094                                started_at: activity.started_at,
5095                                last_failure: activity.last_failure.clone(),
5096                                heartbeat_details: activity.heartbeat_details.clone(),
5097                                last_heartbeat_at: activity.last_heartbeat_at,
5098                                last_worker_identity: {
5099                                    // `LastWorkerIdentity = StartedIdentity`, falling
5100                                    // back to `RetryLastWorkerIdentity` when empty and
5101                                    // a retry policy exists (`GetPendingActivityInfo`,
5102                                    // workflow/activity.go:159-166 @ v1.31.0).
5103                                    let started = activity
5104                                        .started_identity
5105                                        .clone()
5106                                        .map(|identity| identity.0)
5107                                        .unwrap_or_default();
5108                                    if started.is_empty() && activity.retry_policy.is_some() {
5109                                        activity
5110                                            .retry_last_worker_identity
5111                                            .clone()
5112                                            .map(|identity| identity.0)
5113                                            .unwrap_or_default()
5114                                    } else {
5115                                        started
5116                                    }
5117                                },
5118                                paused: activity.pause_info.is_some(),
5119                                pause_info: activity.pause_info.as_ref().map(|info| {
5120                                    tokeira_edge::translate::PauseInfoDescription {
5121                                        identity: info.identity.clone(),
5122                                        paused_time: info.pause_time,
5123                                        reason: info.reason.clone(),
5124                                        rule_id: info.rule_id.clone(),
5125                                    }
5126                                }),
5127                                activity_options: tokeira_edge::translate::ActivityOptions {
5128                                    task_queue: Some(activity.task_queue.0.clone()),
5129                                    schedule_to_close_timeout: activity.schedule_to_close_timeout,
5130                                    schedule_to_start_timeout: activity.schedule_to_start_timeout,
5131                                    start_to_close_timeout: activity.start_to_close_timeout,
5132                                    heartbeat_timeout: activity.heartbeat_timeout,
5133                                    retry_policy: activity.retry_policy.clone(),
5134                                    priority: activity.priority.as_ref().map(|priority| {
5135                                        tokeira_edge::translate::Priority {
5136                                            priority_key: priority.priority_key,
5137                                            fairness_key: priority.fairness_key.clone(),
5138                                            fairness_weight: priority.fairness_weight,
5139                                        }
5140                                    }),
5141                                },
5142                            },
5143                        )
5144                        .collect(),
5145                    pending_children: state
5146                        .children
5147                        .values()
5148                        .map(|child| tokeira_edge::translate::PendingChildDescription {
5149                            workflow_id: child.child_workflow_id.0.clone(),
5150                            run_id: child
5151                                .child_run_id
5152                                .as_ref()
5153                                .map(|run_id| run_id.0.to_string()),
5154                            workflow_type: String::new(),
5155                            initiated_event_id: child.initiated_event_id,
5156                            parent_close_policy: child.parent_close_policy,
5157                        })
5158                        .collect(),
5159                    pending_workflow_task: state.pending_workflow_task.as_ref().map(|task| {
5160                        tokeira_edge::translate::PendingWorkflowTaskDescription {
5161                            is_started: task.started_event_id.is_some(),
5162                            scheduled_at: task.scheduled_at,
5163                            started_at: task.started_at,
5164                            attempt: task.attempt,
5165                        }
5166                    }),
5167                    callbacks: state.completion_callbacks.clone(),
5168                    pending_nexus_operations,
5169                    pause_info: state.pause_info.as_ref().map(|info| {
5170                        tokeira_edge::translate::PauseInfoDescription {
5171                            identity: info.identity.clone(),
5172                            paused_time: info.pause_time,
5173                            reason: info.reason.clone(),
5174                            rule_id: None,
5175                        }
5176                    }),
5177                    execution_expiration_time: state.workflow_execution_timeout.map(|timeout| {
5178                        deadline_after(
5179                            state.first_run_started_at.unwrap_or(state.started_at),
5180                            timeout,
5181                        )
5182                    }),
5183                    run_expiration_time: state
5184                        .workflow_run_timeout
5185                        .map(|timeout| deadline_after(state.started_at, timeout)),
5186                    cancel_requested: state.cancel_requested,
5187                    original_start_time: state.first_run_started_at.unwrap_or(state.started_at),
5188                    versioning_info: versioning_info
5189                        .filter(|info| info.has_execution_versioning_info()),
5190                    worker_deployment_name: state.worker_deployment_name.clone(),
5191                    priority: state.priority.as_ref().map(|priority| {
5192                        tokeira_edge::translate::Priority {
5193                            priority_key: priority.priority_key,
5194                            fairness_key: priority.fairness_key.clone(),
5195                            fairness_weight: priority.fairness_weight,
5196                        }
5197                    }),
5198                    auto_reset_points: state.auto_reset_points.clone(),
5199                    most_recent_worker_version_stamp,
5200                    request_id_infos: state.request_id_infos.clone(),
5201                    external_payload_count,
5202                    external_payload_size_bytes,
5203                }))
5204            }
5205            LoadedRun::Absent => Err(anyhow!("resolved run missing from storage: {:?}", run_key)),
5206        }
5207    }
5208}
5209
5210#[doc(hidden)]
5211pub fn __cli_parse() -> Cli {
5212    Cli::parse()
5213}
5214
5215#[cfg(test)]
5216// Edition-2024 `std::env` mutation in tests — each site carries its SAFETY comment.
5217#[allow(unsafe_code)]
5218mod tests {
5219    use super::*;
5220    use metrics::with_local_recorder;
5221    use metrics_util::debugging::DebuggingRecorder;
5222    use opentelemetry::trace::TracerProvider as _;
5223    use opentelemetry_sdk::{
5224        error::{OTelSdkError, OTelSdkResult},
5225        trace::{SdkTracerProvider, SpanData, SpanExporter},
5226    };
5227    use proptest::prelude::*;
5228    use tracing_subscriber::layer::SubscriberExt as _;
5229    // Endpoint target/config types are referenced only by the Nexus endpoint store
5230    // tests below; importing them here (not at crate scope) keeps the lib build clean.
5231    use tokeira_runtime::{EndpointTarget, NexusEndpointConfig};
5232
5233    // `bounded-visibility-apply`: the listener-backed and embedded engines both build
5234    // their DSQL stack in `build_dsql_stack`, which copies the attribute index before it
5235    // builds anything that can apply an image (the projectors, the CHASM visibility
5236    // adapter and the visibility repair scanner all come from
5237    // `build_service_stack_with_storage`).
5238    #[test]
5239    fn the_dsql_stack_copies_the_attribute_index_before_anything_applies_an_image() {
5240        let source = include_str!("lib.rs");
5241        assert_eq!(
5242            source.matches("build_dsql_stack(\n").count(),
5243            3,
5244            "the definition, build_and_serve's DSQL branch and the embedded engine"
5245        );
5246        let start = source
5247            .find("async fn build_dsql_stack(")
5248            .expect("build_dsql_stack is defined");
5249        let body = &source[start..];
5250        let end = body[1..]
5251            .find("\nasync fn ")
5252            .map_or(body.len(), |end| end + 1);
5253        let body = &body[..end];
5254        let copy = body
5255            .find(".copy_legacy_attributes()")
5256            .expect("build_dsql_stack copies the attribute index");
5257        let services = body
5258            .find("build_service_stack_with_storage(")
5259            .expect("build_dsql_stack builds the services");
5260        assert!(
5261            copy < services,
5262            "the copy runs before the services are built"
5263        );
5264    }
5265
5266    #[derive(Clone, Copy, Debug, PartialEq, Eq)]
5267    enum StartupModelPhase {
5268        Cluster,
5269        Pool,
5270        Schema,
5271        Ownership,
5272        Runtime,
5273    }
5274
5275    #[derive(Clone, Copy, Debug, PartialEq, Eq)]
5276    enum StartupModelEvent {
5277        Attempt(StartupModelPhase),
5278        Complete(StartupModelPhase),
5279        ReleaseOwnership,
5280        ClosePool,
5281        ReturnHandle,
5282    }
5283
5284    const STARTUP_MODEL_PHASES: [StartupModelPhase; 5] = [
5285        StartupModelPhase::Cluster,
5286        StartupModelPhase::Pool,
5287        StartupModelPhase::Schema,
5288        StartupModelPhase::Ownership,
5289        StartupModelPhase::Runtime,
5290    ];
5291
5292    fn startup_model(failure_boundary: Option<usize>) -> Vec<StartupModelEvent> {
5293        let mut events = Vec::new();
5294        let mut pool_open = false;
5295        let mut ownership_held = false;
5296        for (index, phase) in STARTUP_MODEL_PHASES.into_iter().enumerate() {
5297            events.push(StartupModelEvent::Attempt(phase));
5298            if failure_boundary == Some(index) {
5299                if ownership_held {
5300                    events.push(StartupModelEvent::ReleaseOwnership);
5301                }
5302                if pool_open {
5303                    events.push(StartupModelEvent::ClosePool);
5304                }
5305                return events;
5306            }
5307            events.push(StartupModelEvent::Complete(phase));
5308            pool_open |= phase == StartupModelPhase::Pool;
5309            ownership_held |= phase == StartupModelPhase::Ownership;
5310        }
5311        events.push(StartupModelEvent::ReturnHandle);
5312        events
5313    }
5314
5315    #[derive(Clone, Copy, Debug, PartialEq, Eq)]
5316    enum ShutdownModelEvent {
5317        CloseAdmission,
5318        CancelTasks,
5319        CallsComplete,
5320        DrainComplete,
5321        TasksComplete,
5322        JoinComplete,
5323        FinishTelemetry,
5324        AttemptShardRelease,
5325        CloseOwnershipAdmission,
5326        AttemptOwnershipRelease,
5327        AttemptPoolClose,
5328        ReturnToHost,
5329    }
5330
5331    fn shutdown_model(calls_pending: bool, tasks_pending: bool) -> Vec<ShutdownModelEvent> {
5332        let mut events = vec![
5333            ShutdownModelEvent::CloseAdmission,
5334            ShutdownModelEvent::CancelTasks,
5335        ];
5336        if calls_pending {
5337            events.push(ShutdownModelEvent::CallsComplete);
5338        }
5339        events.push(ShutdownModelEvent::DrainComplete);
5340        if tasks_pending {
5341            events.push(ShutdownModelEvent::TasksComplete);
5342        }
5343        events.extend([
5344            ShutdownModelEvent::JoinComplete,
5345            ShutdownModelEvent::FinishTelemetry,
5346            ShutdownModelEvent::AttemptShardRelease,
5347            ShutdownModelEvent::CloseOwnershipAdmission,
5348            ShutdownModelEvent::AttemptOwnershipRelease,
5349            ShutdownModelEvent::AttemptPoolClose,
5350            ShutdownModelEvent::ReturnToHost,
5351        ]);
5352        events
5353    }
5354
5355    #[derive(Clone, Copy, Debug)]
5356    enum HostInstrumentationModel {
5357        None,
5358        LocalRecorder,
5359        LocalSubscriber,
5360        RecorderAndSubscriber,
5361    }
5362
5363    fn host_instrumentation_strategy() -> impl Strategy<Value = HostInstrumentationModel> {
5364        prop_oneof![
5365            Just(HostInstrumentationModel::None),
5366            Just(HostInstrumentationModel::LocalRecorder),
5367            Just(HostInstrumentationModel::LocalSubscriber),
5368            Just(HostInstrumentationModel::RecorderAndSubscriber),
5369        ]
5370    }
5371
5372    #[derive(Clone, Copy, Debug)]
5373    enum EmbeddedStorageModeModel {
5374        InMemory,
5375        ManagedDsql,
5376        ExistingDsql,
5377    }
5378
5379    fn storage_mode_strategy() -> impl Strategy<Value = EmbeddedStorageModeModel> {
5380        prop_oneof![
5381            Just(EmbeddedStorageModeModel::InMemory),
5382            Just(EmbeddedStorageModeModel::ManagedDsql),
5383            Just(EmbeddedStorageModeModel::ExistingDsql),
5384        ]
5385    }
5386
5387    #[derive(Clone, Copy, Debug, PartialEq, Eq)]
5388    struct EmbeddedConstructionEffects {
5389        listener_attempts: usize,
5390        global_install_attempts: usize,
5391        signal_handler_attempts: usize,
5392        local_emission_attempts: usize,
5393    }
5394
5395    fn embedded_construction_effects(
5396        _host: HostInstrumentationModel,
5397        _storage: EmbeddedStorageModeModel,
5398    ) -> EmbeddedConstructionEffects {
5399        // Every storage choice crosses the same library boundary. Durable
5400        // modes add AWS/SQL I/O but never acquire host process facilities.
5401        EmbeddedConstructionEffects {
5402            listener_attempts: 0,
5403            global_install_attempts: 0,
5404            signal_handler_attempts: 0,
5405            local_emission_attempts: 1,
5406        }
5407    }
5408
5409    #[derive(Clone, Copy, Debug)]
5410    enum TelemetryFixture {
5411        NoSubscriber,
5412        LocalRecorder,
5413        FailingExporter,
5414    }
5415
5416    fn telemetry_fixture_strategy() -> impl Strategy<Value = TelemetryFixture> {
5417        prop_oneof![
5418            Just(TelemetryFixture::NoSubscriber),
5419            Just(TelemetryFixture::LocalRecorder),
5420            Just(TelemetryFixture::FailingExporter),
5421        ]
5422    }
5423
5424    #[derive(Clone, Debug)]
5425    struct FailingSpanExporter;
5426
5427    impl SpanExporter for FailingSpanExporter {
5428        async fn export(&self, _batch: Vec<SpanData>) -> OTelSdkResult {
5429            Err(OTelSdkError::InternalFailure(
5430                "fixture exporter rejected the batch".to_owned(),
5431            ))
5432        }
5433    }
5434
5435    fn apply_observed_sequence(ops: &[i16]) -> Vec<u8> {
5436        let mut state = 0_i64;
5437        let mut committed = Vec::with_capacity(ops.len() * std::mem::size_of::<i64>());
5438        let span = tracing::info_span!(
5439            "embedded.observational_fixture",
5440            tokeira.operation = "transition_sequence"
5441        );
5442        let _entered = span.enter();
5443        for op in ops {
5444            record_embedded_lifecycle(
5445                EmbeddedStorageModeLabel::InMemory,
5446                ClusterStatusLabel::NotApplicable,
5447                SchemaOutcomeLabel::NotApplicable,
5448                OwnershipOutcomeLabel::NotApplicable,
5449                DbClassLabel::Commit,
5450                EmbeddedOperationLabel::Startup,
5451                ErrorClassLabel::None,
5452            );
5453            state = state.wrapping_add(i64::from(*op));
5454            committed.extend_from_slice(&state.to_le_bytes());
5455        }
5456        committed
5457    }
5458
5459    fn observed_sequence_bytes(ops: &[i16], fixture: TelemetryFixture) -> Vec<u8> {
5460        match fixture {
5461            TelemetryFixture::NoSubscriber => apply_observed_sequence(ops),
5462            TelemetryFixture::LocalRecorder => {
5463                let recorder = DebuggingRecorder::new();
5464                with_local_recorder(&recorder, || apply_observed_sequence(ops))
5465            }
5466            TelemetryFixture::FailingExporter => {
5467                let provider = SdkTracerProvider::builder()
5468                    .with_simple_exporter(FailingSpanExporter)
5469                    .build();
5470                let tracer = provider.tracer("observational-property");
5471                let subscriber = tracing_subscriber::registry()
5472                    .with(tracing_opentelemetry::layer().with_tracer(tracer));
5473                let dispatch = tracing::Dispatch::new(subscriber);
5474                let bytes =
5475                    tracing::dispatcher::with_default(&dispatch, || apply_observed_sequence(ops));
5476                let _ = provider.force_flush();
5477                bytes
5478            }
5479        }
5480    }
5481
5482    proptest! {
5483        #![proptest_config(ProptestConfig::with_cases(128))]
5484
5485        // Feature: managed-embedded-dsql, Property 12: startup is prefix-safe and failure-atomic
5486        #[test]
5487        fn startup_is_prefix_safe_and_failure_atomic(
5488            failure_boundary in prop::option::of(0usize..STARTUP_MODEL_PHASES.len()),
5489        ) {
5490            let events = startup_model(failure_boundary);
5491            let attempted = events.iter().filter_map(|event| match event {
5492                StartupModelEvent::Attempt(phase) => Some(*phase),
5493                _ => None,
5494            }).collect::<Vec<_>>();
5495            let completed = events.iter().filter_map(|event| match event {
5496                StartupModelEvent::Complete(phase) => Some(*phase),
5497                _ => None,
5498            }).collect::<Vec<_>>();
5499
5500            prop_assert_eq!(attempted.as_slice(), &STARTUP_MODEL_PHASES[..attempted.len()]);
5501            prop_assert_eq!(completed.as_slice(), &STARTUP_MODEL_PHASES[..completed.len()]);
5502            if let Some(boundary) = failure_boundary {
5503                prop_assert!(!events.contains(&StartupModelEvent::ReturnHandle));
5504                let rollback = events.iter().filter(|event| matches!(
5505                    event,
5506                    StartupModelEvent::ReleaseOwnership | StartupModelEvent::ClosePool
5507                )).copied().collect::<Vec<_>>();
5508                let expected = match boundary {
5509                    0 | 1 => Vec::new(),
5510                    2 | 3 => vec![StartupModelEvent::ClosePool],
5511                    _ => vec![
5512                        StartupModelEvent::ReleaseOwnership,
5513                        StartupModelEvent::ClosePool,
5514                    ],
5515                };
5516                prop_assert_eq!(rollback, expected);
5517            } else {
5518                prop_assert_eq!(completed.len(), STARTUP_MODEL_PHASES.len());
5519                prop_assert_eq!(events.last(), Some(&StartupModelEvent::ReturnHandle));
5520            }
5521        }
5522
5523        // Feature: managed-embedded-dsql, Property 19: shutdown establishes the host flush boundary
5524        #[test]
5525        fn shutdown_establishes_the_host_flush_boundary(
5526            calls_pending in any::<bool>(),
5527            tasks_pending in any::<bool>(),
5528            independent_failure_mask in 0u8..8,
5529        ) {
5530            let events = shutdown_model(calls_pending, tasks_pending);
5531            prop_assert_eq!(events.first(), Some(&ShutdownModelEvent::CloseAdmission));
5532            prop_assert_eq!(events.get(1), Some(&ShutdownModelEvent::CancelTasks));
5533            prop_assert_eq!(events.last(), Some(&ShutdownModelEvent::ReturnToHost));
5534            for cleanup in [
5535                ShutdownModelEvent::AttemptShardRelease,
5536                ShutdownModelEvent::AttemptOwnershipRelease,
5537                ShutdownModelEvent::AttemptPoolClose,
5538            ] {
5539                prop_assert!(events.contains(&cleanup));
5540            }
5541            prop_assert!(
5542                events.iter().position(|event| *event == ShutdownModelEvent::AttemptShardRelease)
5543                    < events.iter().position(|event| *event == ShutdownModelEvent::CloseOwnershipAdmission)
5544            );
5545            prop_assert!(
5546                events.iter().position(|event| *event == ShutdownModelEvent::CloseOwnershipAdmission)
5547                    < events.iter().position(|event| *event == ShutdownModelEvent::AttemptOwnershipRelease)
5548            );
5549            let failures = (0..3)
5550                .filter(|bit| independent_failure_mask & (1 << bit) != 0)
5551                .count();
5552            prop_assert_eq!(failures, independent_failure_mask.count_ones() as usize);
5553            prop_assert!(
5554                events.iter().position(|event| *event == ShutdownModelEvent::FinishTelemetry)
5555                    < events.iter().position(|event| *event == ShutdownModelEvent::ReturnToHost)
5556            );
5557        }
5558
5559        // Feature: managed-embedded-dsql, Property 14: embedded construction is transport- and global-state-neutral
5560        #[test]
5561        fn embedded_construction_is_transport_and_global_state_neutral(
5562            host in host_instrumentation_strategy(),
5563            storage in storage_mode_strategy(),
5564        ) {
5565            let effects = embedded_construction_effects(host, storage);
5566            prop_assert_eq!(effects.listener_attempts, 0);
5567            prop_assert_eq!(effects.global_install_attempts, 0);
5568            prop_assert_eq!(effects.signal_handler_attempts, 0);
5569            prop_assert_eq!(effects.local_emission_attempts, 1);
5570        }
5571
5572        // Feature: embedded-engine-listener, Property 6: startup stays zero-listener
5573        #[test]
5574        fn startup_stays_zero_listener(
5575            host in host_instrumentation_strategy(),
5576            storage in storage_mode_strategy(),
5577            explicit_listens in 0usize..4,
5578        ) {
5579            // Every start path contributes no bind; only explicit `listen`
5580            // calls do, one socket each.
5581            let startup = embedded_construction_effects(host, storage);
5582            prop_assert_eq!(startup.listener_attempts, 0);
5583            prop_assert_eq!(
5584                startup.listener_attempts + explicit_listens,
5585                explicit_listens
5586            );
5587        }
5588
5589        // Feature: managed-embedded-dsql, Property 20: telemetry is observational only
5590        #[test]
5591        fn telemetry_is_observational_only(
5592            operations in prop::collection::vec(any::<i16>(), 0..64),
5593            first in telemetry_fixture_strategy(),
5594            second in telemetry_fixture_strategy(),
5595        ) {
5596            let expected = observed_sequence_bytes(&operations, TelemetryFixture::NoSubscriber);
5597            prop_assert_eq!(observed_sequence_bytes(&operations, first), expected.clone());
5598            prop_assert_eq!(observed_sequence_bytes(&operations, second), expected);
5599        }
5600    }
5601
5602    #[tokio::test]
5603    async fn listen_after_shutdown_began_is_refused_without_binding() {
5604        let engine = Engine::start().await.expect("engine starts");
5605        // The only way to observe a shutting-down engine from `listen` is the
5606        // engine's own cancellation token: `shutdown` consumes the engine, so a
5607        // host cannot race it, but internal cancellation must still be refused.
5608        engine.background_cancel.cancel();
5609        let error = engine
5610            .listen("127.0.0.1:0".parse().expect("static loopback address"))
5611            .await
5612            .expect_err("a cancelled engine must refuse to bind");
5613        assert!(matches!(error, EngineListenError::ShutDown));
5614        assert_eq!(engine.listeners.attached(), 0);
5615    }
5616
5617    #[test]
5618    fn every_startup_failure_boundary_rolls_back_without_returning_a_handle() {
5619        for boundary in 0..STARTUP_MODEL_PHASES.len() {
5620            let events = startup_model(Some(boundary));
5621            assert!(!events.contains(&StartupModelEvent::ReturnHandle));
5622        }
5623    }
5624
5625    #[test]
5626    fn durable_startup_report_exposes_only_approved_evidence() {
5627        let report = EngineStartupReport {
5628            storage_mode: EmbeddedStorageMode::ManagedDsql,
5629            cluster: Some(ClusterStartupReport {
5630                region: "eu-west-1".to_owned(),
5631                cluster_id: "cluster-1".to_owned(),
5632                cluster_arn: "arn:aws:dsql:eu-west-1:123456789012:cluster/cluster-1".to_owned(),
5633                endpoint: "cluster-1.dsql.eu-west-1.on.aws".to_owned(),
5634                action: ClusterAction::Recovered,
5635            }),
5636            schema: Some(SchemaStartupReport {
5637                observed_version: 1,
5638                minimum_supported_version: 1,
5639                target_version: 1,
5640                maximum_readable_version: 1,
5641                migration_set_digest: "release-digest".to_owned(),
5642                outcome: SchemaStartupOutcome::Compatible,
5643            }),
5644            ownership: Some(OwnershipStartupReport {
5645                outcome: ControlLeaseAcquireOutcome::Clean,
5646                fence_token: 7,
5647            }),
5648        };
5649        let diagnostic = format!("{report:?}");
5650        assert!(!diagnostic.contains("descriptor-path-secret"));
5651        assert!(!diagnostic.contains("create-token-secret"));
5652        assert!(!diagnostic.contains("credential-secret"));
5653    }
5654
5655    #[tokio::test]
5656    async fn explicit_in_memory_start_preserves_legacy_mode_and_shuts_down_cleanly() {
5657        let engine = Engine::start_with_embedded_config(EmbeddedEngineConfig::default())
5658            .await
5659            .expect("explicit in-memory startup succeeds");
5660        assert_eq!(engine.startup_report(), &EngineStartupReport::in_memory());
5661        engine.shutdown().await.expect("tracked shutdown succeeds");
5662    }
5663
5664    #[test]
5665    fn embedded_lifecycle_composes_with_a_host_local_recorder() {
5666        let recorder = DebuggingRecorder::new();
5667        let runtime = tokio::runtime::Builder::new_current_thread()
5668            .enable_all()
5669            .build()
5670            .expect("embedded fixture runtime");
5671
5672        with_local_recorder(&recorder, || {
5673            runtime.block_on(async {
5674                // Starting twice in one process also proves the library did not
5675                // consume the one-shot process observability reservation.
5676                for _ in 0..2 {
5677                    let engine =
5678                        Engine::start_with_embedded_config(EmbeddedEngineConfig::default())
5679                            .await
5680                            .expect("embedded startup");
5681                    engine.shutdown().await.expect("embedded shutdown");
5682                }
5683            });
5684        });
5685
5686        let lifecycle_observations = recorder
5687            .snapshotter()
5688            .snapshot()
5689            .into_vec()
5690            .into_iter()
5691            .filter(|(key, _, _, _)| {
5692                key.key().name() == tokeira_observability::EMBEDDED_LIFECYCLE_OPERATIONS_TOTAL
5693            })
5694            .count();
5695        assert_eq!(lifecycle_observations, 2);
5696    }
5697
5698    #[tokio::test]
5699    async fn invalid_explicit_config_fails_before_startup() {
5700        let config = EmbeddedEngineConfig {
5701            startup_timeout_ms: 0,
5702            ..EmbeddedEngineConfig::default()
5703        };
5704        assert!(matches!(
5705            Engine::start_with_embedded_config(config).await,
5706            Err(EmbeddedEngineStartError::InvalidConfiguration(_))
5707        ));
5708    }
5709
5710    #[tokio::test]
5711    async fn unequal_home_counts_fail_before_descriptor_or_listener_work() {
5712        let descriptor_path = std::env::temp_dir().join(format!(
5713            "tokeira-home-count-preflight-{}.json",
5714            IncarnationId::new()
5715        ));
5716        let mut config = EmbeddedEngineConfig::default();
5717        config.server.infrastructure.placement.shard_count = 8;
5718        config.server.infrastructure.placement.bundle_count = 4;
5719        config.storage =
5720            EmbeddedStorageConfig::ManagedDsql(tokeira_config::ManagedEmbeddedDsqlConfig {
5721                intent: tokeira_config::ManagedClusterIntent::CreateOrRecover,
5722                descriptor_path: descriptor_path.clone(),
5723                region: "test-region".to_owned(),
5724                migration_policy: None,
5725                limits: tokeira_config::EmbeddedDsqlLimits::default(),
5726                tags: std::collections::BTreeMap::new(),
5727            });
5728        let error = Engine::start_with_embedded_config(config.clone())
5729            .await
5730            .expect_err("unequal counts must fail before any AWS work");
5731        assert!(matches!(
5732            error,
5733            EmbeddedEngineStartError::InvalidConfiguration(_)
5734        ));
5735        assert!(
5736            error
5737                .to_string()
5738                .contains("shard_count (8) must equal bundle_count (4)")
5739        );
5740        assert!(!descriptor_path.exists());
5741
5742        let address = "127.0.0.1:0".parse().expect("test address");
5743        let error = TokeiradHandle::start_in_memory_with_config(address, config.server)
5744            .await
5745            .expect_err("unequal counts must fail before binding a listener");
5746        assert!(format!("{error:#}").contains("shard_count (8) must equal bundle_count (4)"));
5747    }
5748
5749    #[tokio::test]
5750    async fn durable_snapshot_conflict_fails_before_descriptor_or_aws_work() {
5751        let descriptor_path = std::env::temp_dir().join(format!(
5752            "tokeira-managed-dsql-preflight-{}.json",
5753            IncarnationId::new()
5754        ));
5755        let mut config = EmbeddedEngineConfig::default();
5756        config.server.policy.snapshot = Some(SnapshotPolicyConfig {
5757            location: descriptor_path.with_extension("snapshot"),
5758            interval_ms: 1_000,
5759        });
5760        config.storage =
5761            EmbeddedStorageConfig::ManagedDsql(tokeira_config::ManagedEmbeddedDsqlConfig {
5762                intent: tokeira_config::ManagedClusterIntent::CreateOrRecover,
5763                descriptor_path: descriptor_path.clone(),
5764                region: "eu-west-2".to_owned(),
5765                migration_policy: None,
5766                limits: tokeira_config::EmbeddedDsqlLimits::default(),
5767                tags: std::collections::BTreeMap::new(),
5768            });
5769
5770        assert!(matches!(
5771            Engine::start_with_embedded_config(config).await,
5772            Err(EmbeddedEngineStartError::Phase {
5773                phase: EmbeddedStartupPhase::Configuration
5774            })
5775        ));
5776        assert!(!descriptor_path.exists());
5777    }
5778
5779    #[tokio::test]
5780    async fn drop_synchronously_closes_endpoint_admission() {
5781        let engine = Engine::start().await.expect("engine starts");
5782        let endpoint = engine.endpoint();
5783        drop(engine);
5784
5785        let error = endpoint
5786            .call(InProcessGrpcRequest {
5787                service: "temporal.api.workflowservice.v1.WorkflowService".to_owned(),
5788                rpc: "GetSystemInfo".to_owned(),
5789                headers: http::HeaderMap::new(),
5790                proto: Bytes::new(),
5791            })
5792            .await
5793            .expect_err("dropped engine rejects endpoint clones");
5794        assert_eq!(error.code(), tonic::Code::Unavailable);
5795    }
5796
5797    #[tokio::test]
5798    async fn snapshot_policy_over_dsql_storage_is_rejected() {
5799        let mut config = TokeiraConfig::default();
5800        config.infrastructure.storage = ConfigStorageKind::Dsql;
5801        config.policy.snapshot = Some(SnapshotPolicyConfig {
5802            location: std::path::PathBuf::from("unused.snapshot"),
5803            interval_ms: 60_000,
5804        });
5805        let error = build_and_serve(
5806            "127.0.0.1:0".parse().expect("loopback addr"),
5807            Arc::new(config),
5808        )
5809        .await
5810        .expect_err("snapshot policy over DSQL must be rejected before any I/O");
5811        let message = format!("{error:#}");
5812        assert!(
5813            message.contains("in-memory store's persistence mechanism"),
5814            "the rejection must name the mechanism: {message}"
5815        );
5816    }
5817
5818    // Feature: distributed-schema-startup-check, Property 3: The policy
5819    // setting selects the check
5820    #[tokio::test]
5821    async fn property_the_policy_setting_selects_the_check() {
5822        for policy in [
5823            None,
5824            Some(DsqlMigrationPolicy::ValidateOnly),
5825            Some(DsqlMigrationPolicy::Automatic),
5826        ] {
5827            let mut config = TokeiraConfig::default();
5828            config.infrastructure.storage = ConfigStorageKind::Dsql;
5829            config.infrastructure.dsql.migration_policy = policy;
5830            let error = build_and_serve(
5831                "127.0.0.1:0".parse().expect("loopback addr"),
5832                Arc::new(config),
5833            )
5834            .await
5835            .expect_err("no endpoint is configured");
5836            let message = format!("{error:#}");
5837            // A start the policy lets through goes on to configure its DSQL
5838            // connection, which needs the endpoint this configuration lacks.
5839            let refused_by_policy = message.contains("migration_policy");
5840            assert_eq!(
5841                refused_by_policy,
5842                policy == Some(DsqlMigrationPolicy::Automatic),
5843                "{policy:?}: {message}"
5844            );
5845            assert!(
5846                refused_by_policy || message.contains("infrastructure.dsql.endpoint must be set"),
5847                "{policy:?}: {message}"
5848            );
5849        }
5850    }
5851
5852    #[test]
5853    fn each_schema_refusal_says_what_was_found_and_what_to_do() {
5854        let refusal = |decision| {
5855            format!(
5856                "{:#}",
5857                distributed_schema_admission(decision).expect_err("the node refuses")
5858            )
5859        };
5860        let reject = |category| {
5861            SchemaDecision::Reject(tokeira_storage::dsql::SchemaIncompatibility {
5862                observed_version: Some(83),
5863                minimum_supported_version: 1,
5864                target_version: 82,
5865                maximum_readable_version: 82,
5866                category,
5867            })
5868        };
5869        assert_eq!(
5870            distributed_schema_admission(SchemaDecision::Compatible {
5871                current: 82,
5872                legacy_backfill: false,
5873            })
5874            .expect("a compatible schema starts"),
5875            82
5876        );
5877        let message = refusal(SchemaDecision::MigrationRequired {
5878            current: 0,
5879            target: 82,
5880        });
5881        assert!(
5882            message.contains("no Tokeira schema")
5883                && message.contains("V82")
5884                && message.contains("tkr schema setup"),
5885            "{message}"
5886        );
5887        let message = refusal(SchemaDecision::MigrationRequired {
5888            current: 81,
5889            target: 82,
5890        });
5891        assert!(
5892            message.contains("V81")
5893                && message.contains("V82")
5894                && message.contains("tkr schema setup"),
5895            "{message}"
5896        );
5897        let message = refusal(reject(SchemaIncompatibilityCategory::FutureSchema));
5898        assert!(
5899            message.contains("a later release migrated it") && message.contains("FutureSchema"),
5900            "{message}"
5901        );
5902        let message = refusal(reject(SchemaIncompatibilityCategory::BelowMinimum));
5903        assert!(
5904            message.contains("older than this release migrates from"),
5905            "{message}"
5906        );
5907        let message = refusal(reject(SchemaIncompatibilityCategory::ChecksumMismatch {
5908            version: 7,
5909            expected: "expected".to_owned(),
5910            observed: "observed".to_owned(),
5911        }));
5912        assert!(
5913            message.contains("migration ledger doesn't match")
5914                && message.contains("ChecksumMismatch"),
5915            "{message}"
5916        );
5917        for decision in [
5918            SchemaDecision::Initialize { target: 82 },
5919            SchemaDecision::Migrate { from: 81, to: 82 },
5920        ] {
5921            let message = refusal(decision);
5922            assert!(
5923                message.contains("validate-only schema assessment decided"),
5924                "{message}"
5925            );
5926        }
5927    }
5928
5929    proptest! {
5930        #![proptest_config(ProptestConfig::with_cases(256))]
5931
5932        // Feature: distributed-schema-startup-check, Property 1: A node starts
5933        // only on a schema its release can run on
5934        #[test]
5935        fn property_a_node_starts_only_on_a_schema_its_release_runs_on(
5936            known in 2_u32..30,
5937            maximum_back in 0_u32..3,
5938            target_back in 0_u32..3,
5939            minimum_seed in 1_u32..30,
5940            current in 0_u32..34,
5941            tampered in proptest::option::of(0_u32..30),
5942        ) {
5943            use tokeira_storage::{
5944                dsql::{
5945                    AppliedMigration, SchemaCompatibilityContract, SchemaObservation,
5946                    assess_schema_compatibility,
5947                },
5948                schema_contract::{MigrationIdentity, migration_set_digest, sha256_hex},
5949            };
5950
5951            let maximum = known.saturating_sub(maximum_back).max(1);
5952            let target = maximum.saturating_sub(target_back).max(1);
5953            let minimum = minimum_seed.clamp(1, target);
5954            let recognized: Vec<MigrationIdentity> = (1..=known)
5955                .map(|version| MigrationIdentity {
5956                    version,
5957                    name: format!("migration_{version}"),
5958                    checksum: sha256_hex(format!("sql-{version}").as_bytes()),
5959                })
5960                .collect();
5961            let contract = SchemaCompatibilityContract {
5962                tokeira_release: "this".to_owned(),
5963                minimum_supported_version: minimum,
5964                target_version: target,
5965                maximum_readable_version: maximum,
5966                migration_set_digest: migration_set_digest(&recognized, maximum)
5967                    .expect("the readable prefix exists"),
5968            };
5969            // The ledger holds versions 1 to `current`; those past `known` are
5970            // a later release's, and `tampered` alters one of the release's.
5971            let tampered = tampered.filter(|version| *version >= 1 && *version <= current.min(known));
5972            let applied_migrations = (1..=current)
5973                .map(|version| match recognized.get(version as usize - 1) {
5974                    Some(identity) => AppliedMigration {
5975                        version,
5976                        name: identity.name.clone(),
5977                        checksum: if tampered == Some(version) {
5978                            sha256_hex(b"changed")
5979                        } else {
5980                            identity.checksum.clone()
5981                        },
5982                    },
5983                    None => AppliedMigration {
5984                        version,
5985                        name: "later_release".to_owned(),
5986                        checksum: sha256_hex(format!("later-{version}").as_bytes()),
5987                    },
5988                })
5989                .collect();
5990            let decision = assess_schema_compatibility(
5991                &contract,
5992                &recognized,
5993                &SchemaObservation { applied_migrations, compatibility: None },
5994                SchemaMigrationPolicy::ValidateOnly,
5995            );
5996
5997            let runs = tampered.is_none() && target <= current && current <= maximum;
5998            match distributed_schema_admission(decision) {
5999                Ok(version) => {
6000                    prop_assert!(runs, "admitted V{current}");
6001                    prop_assert_eq!(version, current);
6002                }
6003                Err(error) => {
6004                    prop_assert!(!runs, "refused V{current}");
6005                    let message = format!("{error:#}");
6006                    if tampered.is_none() && current < target && (current == 0 || current >= minimum) {
6007                        prop_assert!(
6008                            message.contains(&format!("V{target}")) && message.contains("tkr schema setup"),
6009                            "{message}"
6010                        );
6011                    } else {
6012                        prop_assert!(message.contains("schema is incompatible"), "{message}");
6013                    }
6014                }
6015            }
6016        }
6017    }
6018
6019    /// A distributed node never migrates the schema it shares, so `automatic`
6020    /// is refused before any I/O; nothing here names an endpoint, so a startup
6021    /// that went on to connect would fail on that instead.
6022    #[tokio::test]
6023    async fn automatic_migration_policy_over_distributed_dsql_is_refused() {
6024        let mut config = TokeiraConfig::default();
6025        config.infrastructure.storage = ConfigStorageKind::Dsql;
6026        config.infrastructure.dsql.migration_policy = Some(DsqlMigrationPolicy::Automatic);
6027        let error = build_and_serve(
6028            "127.0.0.1:0".parse().expect("loopback addr"),
6029            Arc::new(config),
6030        )
6031        .await
6032        .expect_err("automatic must be refused before any I/O");
6033        let message = format!("{error:#}");
6034        assert!(
6035            message.contains("\"automatic\"") && message.contains("tkr schema setup"),
6036            "the refusal must name the policy and the way to migrate: {message}"
6037        );
6038    }
6039
6040    /// The daemon drain regression: a real SIGTERM to the serving process
6041    /// finishes in-flight work, persists the final snapshot, and the next
6042    /// boot restores from it. Runs in-process (nextest isolates one process
6043    /// per test, so signalling ourselves is safe); the restart half uses the
6044    /// listener facade, which shares the same build path minus the
6045    /// signal/observability install that must stay daemon-only.
6046    #[tokio::test(flavor = "multi_thread")]
6047    async fn sigterm_drains_persists_final_snapshot_and_next_boot_restores() {
6048        use tokeira_proto::workflowservice::{
6049            DescribeWorkflowExecutionRequest, StartWorkflowExecutionRequest,
6050            workflow_service_client::WorkflowServiceClient,
6051        };
6052
6053        let scratch =
6054            std::env::temp_dir().join(format!("tokeira-engine-sigterm-{}", std::process::id()));
6055        std::fs::create_dir_all(&scratch).expect("create scratch dir");
6056        let snapshot_location = scratch.join("tokeirad.snapshot");
6057
6058        // Reserve a loopback port for the daemon config. The tiny window
6059        // between drop and rebind is acceptable in a single-purpose test
6060        // process.
6061        let reserved = std::net::TcpListener::bind("127.0.0.1:0").expect("reserve port");
6062        let addr = reserved.local_addr().expect("reserved addr");
6063        drop(reserved);
6064
6065        let mut config = TokeiraConfig::default();
6066        config.infrastructure.network.grpc_addr = addr.to_string();
6067        // Loopback-ephemeral Nexus completion listener, exactly as the
6068        // in-memory facade configures it, so parallel tests never collide.
6069        config.policy.nexus_completion.http_addr = "127.0.0.1:0".to_string();
6070        config.policy.nexus_completion.system_callback_url = "http://127.0.0.1:0".to_string();
6071        config.policy.snapshot = Some(SnapshotPolicyConfig {
6072            location: snapshot_location.clone(),
6073            // Effectively periodic-off: this regression is about the FINAL
6074            // cut on the signal path.
6075            interval_ms: 3_600_000,
6076        });
6077        let config_path = scratch.join("tokeirad.toml");
6078        std::fs::write(&config_path, config.to_toml().expect("render config"))
6079            .expect("write config");
6080
6081        let mut daemon = tokio::spawn(run_from_cli(Cli {
6082            command: None,
6083            config: Some(config_path.display().to_string()),
6084            dump_config: false,
6085            version: false,
6086            verbose: false,
6087            json: false,
6088        }));
6089
6090        // Connect-poll until the daemon serves. The retry interval is a
6091        // polling cadence against an external TCP bind (the one boundary a
6092        // channel cannot observe), bounded by the deadline; a daemon that
6093        // exits early fails the wait immediately with its own error.
6094        let target = format!("http://{addr}");
6095        let deadline = tokio::time::Instant::now() + std::time::Duration::from_secs(20);
6096        let mut client = loop {
6097            assert!(
6098                tokio::time::Instant::now() < deadline,
6099                "daemon must start serving"
6100            );
6101            tokio::select! {
6102                result = &mut daemon => {
6103                    panic!("daemon exited before serving: {result:?}");
6104                }
6105                connected = WorkflowServiceClient::connect(target.clone()) => {
6106                    match connected {
6107                        Ok(client) => break client,
6108                        Err(_) => {
6109                            tokio::time::sleep(std::time::Duration::from_millis(50)).await;
6110                        }
6111                    }
6112                }
6113            }
6114        };
6115        client
6116            .start_workflow_execution(StartWorkflowExecutionRequest {
6117                namespace: "default".to_owned(),
6118                workflow_id: "sigterm-canary".to_owned(),
6119                workflow_type: Some(tokeira_proto::common::WorkflowType {
6120                    name: "sigterm-workflow".to_owned(),
6121                }),
6122                task_queue: Some(tokeira_proto::taskqueue::TaskQueue {
6123                    name: "sigterm-queue".to_owned(),
6124                    ..Default::default()
6125                }),
6126                request_id: "sigterm-start".to_owned(),
6127                ..Default::default()
6128            })
6129            .await
6130            .expect("start canary workflow");
6131
6132        // The real signal, sent to this very process; the drain must treat it
6133        // exactly as an operator's `kill`.
6134        let delivered = std::process::Command::new("kill")
6135            .args(["-TERM", &std::process::id().to_string()])
6136            .status()
6137            .expect("send SIGTERM");
6138        assert!(delivered.success(), "kill -TERM must be deliverable");
6139
6140        daemon
6141            .await
6142            .expect("daemon task must not panic")
6143            .expect("SIGTERM drain must exit cleanly");
6144        assert!(
6145            snapshot_location.is_file(),
6146            "the drain must persist the final snapshot"
6147        );
6148
6149        // Next boot restores the canary from the snapshot.
6150        let restored =
6151            TokeiradHandle::start_in_memory_with_config("127.0.0.1:0".parse().expect("addr"), {
6152                let mut config = TokeiraConfig::default();
6153                config.policy.snapshot = Some(SnapshotPolicyConfig {
6154                    location: snapshot_location.clone(),
6155                    interval_ms: 3_600_000,
6156                });
6157                config
6158            })
6159            .await
6160            .expect("restart over the final snapshot");
6161        let mut client =
6162            WorkflowServiceClient::connect(format!("http://{}", restored.bound_addr()))
6163                .await
6164                .expect("connect to restarted server");
6165        let described = client
6166            .describe_workflow_execution(DescribeWorkflowExecutionRequest {
6167                namespace: "default".to_owned(),
6168                execution: Some(tokeira_proto::common::WorkflowExecution {
6169                    workflow_id: "sigterm-canary".to_owned(),
6170                    run_id: String::new(),
6171                }),
6172            })
6173            .await
6174            .expect("restored workflow must be describable")
6175            .into_inner();
6176        assert!(
6177            described.workflow_execution_info.is_some(),
6178            "state must survive the SIGTERM drain via snapshot restore"
6179        );
6180        restored.shutdown().await.expect("clean restart shutdown");
6181        let _ = std::fs::remove_dir_all(&scratch);
6182    }
6183
6184    #[test]
6185    fn embedded_config_removes_daemon_storage_and_placement() {
6186        let mut config = TokeiraConfig::default();
6187        config.infrastructure.storage = ConfigStorageKind::Dsql;
6188        config.infrastructure.placement.controller_endpoint =
6189            Some("http://controller.internal:8080".to_owned());
6190
6191        let config = embedded_config(config).expect("embedded defaults are valid");
6192
6193        assert_eq!(config.infrastructure.storage, ConfigStorageKind::InMemory);
6194        assert!(
6195            config
6196                .infrastructure
6197                .placement
6198                .controller_endpoint
6199                .is_none()
6200        );
6201    }
6202
6203    #[tokio::test]
6204    async fn embedded_system_nexus_callback_never_falls_through_to_http() {
6205        let client = InProcessNexusCompletionClient::new();
6206        let target = system_callback_post_url(EMBEDDED_NEXUS_CALLBACK_BASE);
6207        let outcome = client
6208            .complete_operation(
6209                &target,
6210                "not-yet-attached",
6211                "operation-token",
6212                NexusCompletion::Succeeded(tokeira_types::Payloads::default()),
6213                &[],
6214            )
6215            .await
6216            .expect("the in-process preflight outcome is classifiable");
6217
6218        assert!(matches!(
6219            outcome,
6220            CompletionDeliveryOutcome::RetryableError { detail }
6221                if detail == "embedded Nexus completion runtime is unavailable"
6222        ));
6223    }
6224
6225    // Security F1 (apps scope): the network-exposed `/nexus/callback` listener must cap
6226    // its request body before buffering, so an unauthenticated caller cannot exhaust
6227    // memory and abort the `panic = "abort"` process.
6228    #[tokio::test]
6229    async fn bounded_body_caps_oversized_payload() {
6230        // Within the cap: fully collected.
6231        match collect_bounded_body(Full::new(Bytes::from(vec![0u8; 8])), 16).await {
6232            BoundedBody::Collected(bytes) => assert_eq!(bytes.len(), 8),
6233            other => panic!("expected Collected, got {other:?}"),
6234        }
6235        // Over the cap: rejected before the whole body is buffered.
6236        assert!(matches!(
6237            collect_bounded_body(Full::new(Bytes::from(vec![0u8; 17])), 16).await,
6238            BoundedBody::TooLarge
6239        ));
6240    }
6241
6242    #[test]
6243    fn with_loopback_port_rewrites_or_appends_port() {
6244        // A base URL with an explicit port has it replaced with the bound port.
6245        assert_eq!(
6246            with_loopback_port("http://127.0.0.1:7253", 51_000),
6247            "http://127.0.0.1:51000"
6248        );
6249        // A trailing slash is trimmed before rewriting.
6250        assert_eq!(
6251            with_loopback_port("http://127.0.0.1:7253/", 51_000),
6252            "http://127.0.0.1:51000"
6253        );
6254        // A port-less base URL gets the port appended (the scheme `:` is not mistaken for
6255        // a port because the suffix is not all-digits).
6256        assert_eq!(
6257            with_loopback_port("http://localhost", 51_000),
6258            "http://localhost:51000"
6259        );
6260    }
6261
6262    #[test]
6263    fn version_renderer_is_deterministic() {
6264        let short = render_build_info(false, false);
6265        let verbose = render_build_info(true, false);
6266        let json = render_build_info(false, true);
6267
6268        assert_eq!(short, render_build_info(false, false));
6269        assert_eq!(verbose, render_build_info(true, false));
6270        assert_eq!(json, render_build_info(false, true));
6271        assert!(short.contains(tokeira_build_info::SERVER_VERSION));
6272        assert!(short.contains(tokeira_build_info::TEMPORAL_PROTO_VERSION));
6273        assert!(short.contains(tokeira_build_info::TEMPORAL_SERVER_COMPAT));
6274        assert!(json.contains("server_version"));
6275        assert!(json.contains("temporal_proto_version"));
6276    }
6277
6278    #[test]
6279    fn reported_problem_search_attribute_has_exact_v131_keyword_list() {
6280        let mut search_attributes = tokeira_types::SearchAttributes::default();
6281        apply_reported_problem_search_attribute(
6282            &mut search_attributes,
6283            Some(WorkflowTaskReportedProblem {
6284                attempts_since_last_success: 5,
6285                problem: tokeira_kernel::WorkflowTaskProblem::Failed(
6286                    tokeira_kernel::WorkflowTaskFailedCause::WorkflowWorkerUnhandledFailure,
6287                ),
6288            }),
6289        );
6290
6291        assert_eq!(
6292            search_attributes.0.get("TemporalReportedProblems"),
6293            Some(&SearchAttrValue::KeywordList(vec![
6294                "category=WorkflowTaskFailed".to_string(),
6295                "cause=WorkflowTaskFailedCauseWorkflowWorkerUnhandledFailure".to_string(),
6296            ]))
6297        );
6298    }
6299
6300    #[test]
6301    fn reported_problem_search_attribute_renders_timeout_category() {
6302        // A timeout-first sequence reports the TimedOut pair — v1.31.0 renders
6303        // `TimeoutType.String()` for the only stored type
6304        // (`mutable_state_impl.go:6486-6491 @ v1.31.0`).
6305        let mut search_attributes = tokeira_types::SearchAttributes::default();
6306        apply_reported_problem_search_attribute(
6307            &mut search_attributes,
6308            Some(WorkflowTaskReportedProblem {
6309                attempts_since_last_success: 5,
6310                problem: tokeira_kernel::WorkflowTaskProblem::TimedOutStartToClose,
6311            }),
6312        );
6313
6314        assert_eq!(
6315            search_attributes.0.get("TemporalReportedProblems"),
6316            Some(&SearchAttrValue::KeywordList(vec![
6317                "category=WorkflowTaskTimedOut".to_string(),
6318                "cause=WorkflowTaskTimedOutCauseStartToClose".to_string(),
6319            ]))
6320        );
6321    }
6322
6323    fn self_assigning_runtime(
6324        store: Arc<InMemoryStore>,
6325        shard_count: u32,
6326    ) -> Arc<TokeiraRuntime<InMemoryStore>> {
6327        Arc::new(TokeiraRuntime::new_with_nexus_and_shards_and_endpoint(
6328            store,
6329            2,
6330            tokeira_runtime::LaneConfig::default(),
6331            tokeira_runtime::TimerScannerConfig::default(),
6332            tokeira_runtime::WorkflowTimeoutScannerConfig::default(),
6333            tokeira_runtime::ActivityTimeoutScannerConfig::default(),
6334            tokeira_runtime::NexusTimeoutScannerConfig::default(),
6335            NexusEndpointRegistry::default(),
6336            Arc::new(tokeira_runtime::NoopNexusHttpClient),
6337            NexusCompletionDeps::default(),
6338            shard_count,
6339            IncarnationId::new().to_string(),
6340            "127.0.0.1:7233".to_owned(),
6341            false,
6342            None,
6343        ))
6344    }
6345
6346    fn boot_start_request(
6347        namespace_id: tokeira_types::NamespaceId,
6348        index: usize,
6349    ) -> tokeira_kernel::StartRequest {
6350        let run_id = tokeira_types::RunId::new();
6351        let workflow_id = tokeira_types::WorkflowId(format!("boot-{index}"));
6352        let now = time::OffsetDateTime::now_utc();
6353        tokeira_kernel::StartRequest {
6354            advice_policy: tokeira_kernel::ContinueAsNewAdvicePolicy::V1_31_0,
6355            initiator: None,
6356            run_key: tokeira_types::RunKey::derive(namespace_id, &workflow_id, run_id),
6357            namespace_id,
6358            workflow_id,
6359            run_id,
6360            workflow_type: tokeira_types::WorkflowType("boot".to_owned()),
6361            task_queue: tokeira_types::TaskQueueName("boot-queue".to_owned()),
6362            input: tokeira_types::Payloads::default(),
6363            header: None,
6364            memo: tokeira_types::Memo::default(),
6365            search_attributes: tokeira_types::SearchAttributes::default(),
6366            workflow_execution_timeout: None,
6367            workflow_run_timeout: None,
6368            workflow_task_timeout: time::Duration::seconds(10),
6369            retry_policy: None,
6370            conflict_policy: tokeira_kernel::WorkflowIdConflictPolicy::Fail,
6371            reuse_policy: tokeira_kernel::WorkflowIdReusePolicy::AllowDuplicate,
6372            deployment: None,
6373            build_id: None,
6374            versioning_override: None,
6375            workflow_start_delay: None,
6376            completion_callbacks: Vec::new(),
6377            user_metadata: None,
6378            links: Vec::new(),
6379            on_conflict_options: None,
6380            priority: None,
6381            attempt: 1,
6382            continued_execution_run_id: None,
6383            first_execution_run_id: None,
6384            parent_run_key: None,
6385            parent_workflow_id: None,
6386            parent_run_id: None,
6387            parent_namespace_id: None,
6388            parent_namespace_name: None,
6389            parent_initiated_event_id: 0,
6390            root_workflow_id: None,
6391            root_run_id: None,
6392            original_execution_run_id: Some(run_id),
6393            continued_failure: None,
6394            last_completion_result: None,
6395            first_run_started_at: None,
6396            request: tokeira_types::RequestContext {
6397                request_id: tokeira_types::RequestId(format!("boot-start-{index}")),
6398                caller_identity: None,
6399                principal: None,
6400                received_at: now,
6401            },
6402            now,
6403            client_cron_schedule: None,
6404            cron_schedule: None,
6405            eager_execution_accepted: false,
6406            reserved_poller_identity: None,
6407            inherited_versioning_info: None,
6408        }
6409    }
6410
6411    #[tokio::test]
6412    async fn self_assigned_dsql_boot_recovers_every_shard_before_admission() -> Result<()> {
6413        // A single node without a controller takes every shard's lease at
6414        // boot. It must also sweep each shard: the previous process's offered
6415        // workflow tasks ended with it (runtime-sweeper-recovery
6416        // Requirement 11.6).
6417        let shard_count = 2;
6418        let store = Arc::new(InMemoryStore::with_shard_count(shard_count));
6419        let endpoint = NodeEndpoint {
6420            host: "127.0.0.1".to_owned(),
6421            port: 7233,
6422        };
6423        let namespace_id = tokeira_types::NamespaceId::new();
6424
6425        let previous = self_assigning_runtime(Arc::clone(&store), shard_count);
6426        self_assign_dsql_shards(
6427            previous.as_ref(),
6428            store.as_ref(),
6429            shard_count,
6430            &IncarnationId::new(),
6431            &endpoint,
6432            true,
6433        )
6434        .await?;
6435        let mut started = Vec::new();
6436        let mut shards = std::collections::BTreeSet::new();
6437        for index in 0..8 {
6438            let request = boot_start_request(namespace_id, index);
6439            shards.insert(tokeira_types::execution_home_bundle(
6440                namespace_id.0.as_bytes(),
6441                request.workflow_id.0.as_bytes(),
6442                shard_count,
6443            ));
6444            started.push(request.run_key);
6445            let result = previous.start_workflow(request).await?;
6446            assert!(matches!(
6447                result,
6448                tokeira_storage::CommitResult::Applied { .. }
6449            ));
6450        }
6451        assert_eq!(shards.len(), 2, "the seeded runs span both shards");
6452
6453        let restarted = self_assigning_runtime(Arc::clone(&store), shard_count);
6454        let leases = self_assign_dsql_shards(
6455            restarted.as_ref(),
6456            store.as_ref(),
6457            shard_count,
6458            &IncarnationId::new(),
6459            &endpoint,
6460            true,
6461        )
6462        .await?;
6463        assert_eq!(leases.len(), 2);
6464        let mut active = restarted.active_shards();
6465        active.sort();
6466        assert_eq!(active, vec![ShardId(0), ShardId(1)]);
6467
6468        let queue = tokeira_types::QueueKey {
6469            namespace_id,
6470            task_queue: tokeira_types::TaskQueueName("boot-queue".to_owned()),
6471            task_kind: tokeira_types::TaskKind::Workflow,
6472            deployment: None,
6473            build_id: None,
6474        };
6475        let mut offered = Vec::new();
6476        while offered.len() < started.len() {
6477            let task = restarted
6478                .poll_workflow_task(
6479                    queue.clone(),
6480                    tokeira_types::WorkerIdentity("worker".to_owned()),
6481                    tokio::time::Duration::from_secs(1),
6482                )
6483                .await?
6484                .expect("every pending first workflow task is offered again");
6485            offered.push(task.run_key);
6486        }
6487        offered.sort();
6488        started.sort();
6489        assert_eq!(offered, started);
6490        Ok(())
6491    }
6492
6493    #[tokio::test]
6494    async fn build_nexus_endpoint_store_resolves_worker_namespace_names() {
6495        let cache = InMemoryNamespaceCache::new();
6496        let namespace = ResolvedNamespace::active("payments");
6497        let namespace_id = namespace_id_for("payments");
6498        cache.insert(namespace).await.expect("namespace insert");
6499
6500        let store = build_nexus_endpoint_store(
6501            &cache,
6502            HashMap::from([(
6503                "payments-endpoint".to_string(),
6504                BootstrapNexusEndpointConfig {
6505                    target: BootstrapNexusEndpointTarget::Worker {
6506                        namespace_name: "payments".to_string(),
6507                        task_queue: "nexus-q".to_string(),
6508                    },
6509                },
6510            )]),
6511        )
6512        .await
6513        .expect("store should build");
6514
6515        let registry = NexusEndpointRegistry::new(store);
6516        let config = registry.resolve("payments-endpoint").expect("endpoint");
6517        assert_eq!(
6518            config,
6519            NexusEndpointConfig {
6520                target: EndpointTarget::Worker {
6521                    namespace_id,
6522                    task_queue: tokeira_types::TaskQueueName("nexus-q".to_string()),
6523                },
6524            }
6525        );
6526    }
6527
6528    #[tokio::test]
6529    async fn build_nexus_endpoint_store_rejects_unknown_worker_namespace() {
6530        let cache = InMemoryNamespaceCache::new();
6531
6532        let result = build_nexus_endpoint_store(
6533            &cache,
6534            HashMap::from([(
6535                "payments-endpoint".to_string(),
6536                BootstrapNexusEndpointConfig {
6537                    target: BootstrapNexusEndpointTarget::Worker {
6538                        namespace_name: "missing".to_string(),
6539                        task_queue: "nexus-q".to_string(),
6540                    },
6541                },
6542            )]),
6543        )
6544        .await;
6545
6546        assert!(result.is_err(), "missing namespace should fail");
6547        let error = result.expect_err("error");
6548
6549        assert!(error.to_string().contains("namespace `missing` not found"));
6550    }
6551
6552    #[test]
6553    fn placement_helpers_build_runtime_and_edge_config_from_server_config() {
6554        let mut config = TokeiraConfig::default();
6555        config.infrastructure.placement.controller_endpoint =
6556            Some("http://127.0.0.1:7240".to_string());
6557        config.infrastructure.placement.node_host = "10.0.0.9".to_string();
6558        config.infrastructure.placement.node_port = Some(8123);
6559        config.infrastructure.placement.shard_count = 8;
6560        config.infrastructure.placement.bundle_count = 4;
6561        config.infrastructure.placement.partition_count = 64;
6562        config.infrastructure.placement.hash_version = 2;
6563        let listen_addr = "127.0.0.1:7233".parse().unwrap();
6564        let node_id = IncarnationId::new();
6565        let endpoint = configured_node_endpoint(&config, listen_addr);
6566        let membership = membership_config(
6567            &config,
6568            node_id,
6569            endpoint.clone(),
6570            "http://127.0.0.1:7240".into(),
6571        );
6572
6573        assert_eq!(endpoint.as_authority(), "10.0.0.9:8123");
6574        assert_eq!(membership.node_id, node_id);
6575        assert_eq!(membership.node_endpoint, endpoint);
6576        assert_eq!(
6577            placement_config(&config),
6578            PlacementConfig {
6579                shard_count: 8,
6580                bundle_count: 4,
6581                partition_count: 64,
6582                hash_version: 2,
6583            }
6584        );
6585    }
6586
6587    #[test]
6588    fn dsql_pool_config_uses_effective_placement_values() {
6589        let mut config = TokeiraConfig::default();
6590        config.infrastructure.placement.shard_count = 32;
6591        config.infrastructure.placement.partition_count = 4;
6592
6593        let (pool_config, _ddb_client) =
6594            dsql_pool_config_with_client(&config, tokeira_storage::dsql::offline_ddb_client());
6595
6596        assert_eq!(pool_config.shard_count, 32);
6597        assert_eq!(pool_config.projection_partition_count, 4);
6598    }
6599
6600    #[test]
6601    fn wire_coverage_out_path_prefers_env_override_then_falls_back_to_default() {
6602        // Exercised in one test because all cases mutate the same process-global env var;
6603        // splitting them risks cross-test interference under the parallel test runner.
6604
6605        // Unset: the documented default path is used.
6606        // SAFETY: all mutation of this env var is confined to this single test.
6607        unsafe { std::env::remove_var(WIRE_COVERAGE_OUT_ENV) };
6608        assert_eq!(
6609            wire_coverage_out_path(),
6610            PathBuf::from(WIRE_COVERAGE_DEFAULT_OUT)
6611        );
6612
6613        // A real path is honoured verbatim (after trimming).
6614        // SAFETY: all mutation of this env var is confined to this single test.
6615        unsafe { std::env::set_var(WIRE_COVERAGE_OUT_ENV, "  /tmp/cov.json  ") };
6616        assert_eq!(wire_coverage_out_path(), PathBuf::from("/tmp/cov.json"));
6617
6618        // An empty/whitespace override is treated as unset, not as a blank path.
6619        // SAFETY: all mutation of this env var is confined to this single test.
6620        unsafe { std::env::set_var(WIRE_COVERAGE_OUT_ENV, "   ") };
6621        assert_eq!(
6622            wire_coverage_out_path(),
6623            PathBuf::from(WIRE_COVERAGE_DEFAULT_OUT)
6624        );
6625
6626        // SAFETY: all mutation of this env var is confined to this single test.
6627        unsafe { std::env::remove_var(WIRE_COVERAGE_OUT_ENV) };
6628    }
6629
6630    #[test]
6631    fn export_wire_coverage_writes_pretty_json_snapshot() {
6632        let recorder = WireCoverageRecorder::new();
6633        recorder.record(
6634            "/temporal.api.workflowservice.v1.WorkflowService/StartWorkflowExecution",
6635            0,
6636        );
6637
6638        let dir = std::env::temp_dir().join(format!("tokeirad-cov-{}", std::process::id()));
6639        std::fs::create_dir_all(&dir).expect("temp dir");
6640        let path = dir.join("wire-coverage.json");
6641
6642        export_wire_coverage(&recorder, &path).expect("export succeeds");
6643
6644        let written = std::fs::read_to_string(&path).expect("evidence file readable");
6645        assert!(written.contains("StartWorkflowExecution"));
6646        // Pretty JSON is multi-line; a compact encoding would be a single line.
6647        assert!(written.contains('\n'));
6648
6649        std::fs::remove_dir_all(&dir).ok();
6650    }
6651}