1#![deny(rust_2018_idioms)]
15#![allow(clippy::too_many_arguments)]
17
18use tokeira_kernel::deadline_after;
19
20use std::{
21 collections::HashMap,
22 convert::Infallible,
23 future::Future,
24 net::SocketAddr,
25 path::{Path, PathBuf},
26 pin::Pin,
27 sync::{
28 Arc, RwLock, Weak,
29 atomic::{AtomicU64, Ordering},
30 },
31 time::{Duration as StdDuration, Instant},
32};
33
34use anyhow::{Context, Result, anyhow};
35use async_trait::async_trait;
36use clap::Parser;
37use http_body_util::{BodyExt, Full, LengthLimitError, Limited};
38use hyper::{
39 Method, Request, Response, StatusCode,
40 body::{Bytes, Incoming},
41 service::service_fn,
42};
43use hyper_util::rt::TokioIo;
44use tokio::{
45 fs::{self, OpenOptions},
46 io::AsyncWriteExt as _,
47 net::TcpListener,
48 sync::{broadcast, mpsc, oneshot},
49 task::JoinHandle,
50};
51use tokio_stream::wrappers::TcpListenerStream;
52use tokio_util::sync::CancellationToken;
53use tonic::transport::Server;
54use tonic_web::GrpcWebLayer;
55use tower_http::cors::CorsLayer;
56use tracing::{error, info};
57
58#[cfg(feature = "chasm-extensions")]
59pub mod chasm;
60mod chasm_extensions;
61#[cfg(feature = "chasm-extensions")]
62pub use chasm_extensions::EngineBuilder;
63use chasm_extensions::{
64 ChasmExtensions, check_rebuilt_outboxes, check_registered_archetypes,
65 declared_search_attributes, embedded_stack_error, registry_start_error,
66 seed_declared_search_attributes, seed_start_error,
67};
68
69pub mod correlation_format;
70#[cfg(all(test, feature = "dsql-integration"))]
71mod distributed_schema_live_tests;
72#[cfg(test)]
73mod embedded_dsql_tests;
74#[doc(hidden)]
75pub mod harness;
76mod http_api_transport;
77mod listener;
78mod nexus_http_transport;
79mod node_claim;
80pub mod observability;
81mod peer;
82#[cfg(all(test, feature = "dsql-integration"))]
83mod single_node_claim_live_tests;
84mod storage_maintenance;
85
86pub use listener::{EngineListenError, EngineListener, EngineListenerShutdownError};
87
88use http_api_transport::HttpApiLayer;
89use nexus_http_transport::NexusHttpLayer;
90use tokeira_auth::{
91 Authorizer, ClaimMapper, DefaultAuthorizer, GrantRule, GrantRules, JwksKeyProvider,
92 JwtAuthenticator, JwtIssuerProfile, MultiSourceClaimMapper, StsAuthenticator, WorkerScope,
93 WorkerScopeRule, WorkerScopeRules,
94};
95use tokeira_chasm::Library as _;
96use tokeira_config::{Cli, CliCommand, ConfigStorageKind, PeerCredentialCommand};
97pub use tokeira_config::{
98 DsqlMigrationPolicy, EmbeddedEngineConfig, EmbeddedStorageConfig, SnapshotPolicyConfig,
99 TokeiraConfig,
100};
101use tokeira_edge::{
102 Authenticator, CacheBackedRouter, CallbackResponse, EdgeInterceptors, EdgeRoutingConfig,
103 HistoryNotifyingRepository, HistoryWaitRegistry, HttpApiCatalog, HttpApiPolicy,
104 InMemoryNamespaceCache, InMemoryOperatorApi, InProcessGrpcService, LocalOnlyRouter,
105 LongPollConfig, LongPollGate, NamespaceCache, OperatorService, PendingQueryStore,
106 PollerRegistry, ResolvedNamespace, RoutingCache, ScopedWorkerSessionRegistry,
107 WorkerComputeNamespaceCatalogAdapter, WorkflowExecutionDescription, WorkflowService,
108 conformance::{WireCoverageLayer, WireCoverageRecorder},
109 grpc::{
110 admin_service::AdminServiceGrpc, operator_service::OperatorServiceGrpc,
111 runtime_adapter::RuntimeAdapter, workflow_service::WorkflowServiceGrpc,
112 },
113 handle_nexus_callback,
114 nexus_http::MAX_NEXUS_PAYLOAD_BYTES,
115 operator_service::{ClusterInfo, OperatorApi, SearchAttributeDefinition},
116 run_routing_subscription,
117 translate::to_internal::namespace_id_for,
118 workflow_service::{ExecutionResolver, WorkflowRuntimeApi},
119};
120pub use tokeira_edge::{InProcessGrpcRequest, InProcessGrpcResponse};
121
122struct AuthorizationStack {
123 grpc: Arc<dyn Authenticator>,
124 nexus: Arc<dyn tokeira_edge::nexus_http::NexusHttpAuthorizer>,
125 principal_attribution: bool,
126}
127
128async fn build_authorization_stack(config: &TokeiraConfig) -> Result<AuthorizationStack> {
129 let Some(authorization) = config
130 .policy
131 .authorization
132 .as_ref()
133 .filter(|authorization| authorization.has_identity_source())
134 else {
135 let grpc = harness::installed()
136 .and_then(|hooks| hooks.fallback_grpc_authenticator.clone())
137 .unwrap_or_else(|| Arc::new(tokeira_edge::AllowAllAuthenticator));
138 let stack = AuthorizationStack {
139 grpc,
140 nexus: Arc::new(tokeira_edge::nexus_http::PermissiveNexusHttpAuthorizer),
141 principal_attribution: false,
142 };
143 return Ok(stack);
144 };
145
146 let mut profiles = Vec::with_capacity(authorization.jwt.issuers.len());
147 for issuer in &authorization.jwt.issuers {
148 let grants = GrantRules::new(
149 issuer
150 .grants
151 .iter()
152 .map(|rule| {
153 GrantRule::new(
154 rule.match_sub.clone(),
155 rule.grant.iter().map(String::as_str),
156 )
157 .with_context(|| format!("invalid JWT grants for issuer {}", issuer.name))
158 })
159 .collect::<Result<Vec<_>>>()?,
160 );
161 let worker_scopes = WorkerScopeRules::new(
162 issuer
163 .worker_scopes
164 .iter()
165 .map(|rule| {
166 let scope = WorkerScope::try_new(
167 rule.namespace.clone(),
168 rule.task_queues.clone(),
169 rule.deployment_name.clone(),
170 rule.build_id.clone(),
171 )
172 .with_context(|| {
173 format!("invalid JWT Worker scope for issuer {}", issuer.name)
174 })?;
175 WorkerScopeRule::new(rule.match_sub.clone(), scope).with_context(|| {
176 format!(
177 "invalid JWT Worker-scope subject for issuer {}",
178 issuer.name
179 )
180 })
181 })
182 .collect::<Result<Vec<_>>>()?,
183 );
184 let keys =
185 JwksKeyProvider::start(issuer.jwks_uri.clone(), issuer.refresh_interval_duration())
186 .await;
187 profiles.push(
188 JwtIssuerProfile::new(
189 issuer.name.clone(),
190 issuer.issuer.clone(),
191 issuer.audience.clone(),
192 issuer.permissions_claim.clone(),
193 grants,
194 keys,
195 )
196 .with_worker_scopes(worker_scopes),
197 );
198 }
199 let jwt = (!profiles.is_empty()).then(|| JwtAuthenticator::new(profiles));
200 let sts = authorization
201 .aws_iam
202 .as_ref()
203 .map(|aws_iam| {
204 let grants = aws_iam
205 .grants
206 .iter()
207 .map(|rule| {
208 GrantRule::new(
209 rule.match_arn.clone(),
210 rule.grant.iter().map(String::as_str),
211 )
212 .context("invalid AWS IAM grants")
213 })
214 .collect::<Result<Vec<_>>>()
215 .map(GrantRules::new)?;
216 let worker_scopes = WorkerScopeRules::new(
217 aws_iam
218 .worker_scopes
219 .iter()
220 .map(|rule| {
221 let scope = WorkerScope::try_new(
222 rule.namespace.clone(),
223 rule.task_queues.clone(),
224 rule.deployment_name.clone(),
225 rule.build_id.clone(),
226 )
227 .context("invalid AWS IAM Worker scope")?;
228 WorkerScopeRule::new(rule.match_arn.clone(), scope)
229 .context("invalid AWS IAM Worker-scope ARN")
230 })
231 .collect::<Result<Vec<_>>>()?,
232 );
233 Ok::<_, anyhow::Error>(StsAuthenticator::new(grants).with_worker_scopes(worker_scopes))
234 })
235 .transpose()?;
236 let mapper: Arc<dyn ClaimMapper> = Arc::new(MultiSourceClaimMapper::new(jwt, sts));
237 let authorizer: Arc<dyn Authorizer> = Arc::new(DefaultAuthorizer);
238 Ok(AuthorizationStack {
239 grpc: Arc::new(tokeira_edge::PolicyAuthenticator::new(
240 mapper.clone(),
241 authorizer.clone(),
242 authorization.expose_authorizer_errors,
243 )),
244 nexus: Arc::new(tokeira_edge::nexus_http::PolicyNexusHttpAuthorizer::new(
245 mapper,
246 authorizer,
247 authorization.expose_authorizer_errors,
248 )),
249 principal_attribution: authorization.principal_attribution,
250 })
251}
252use tokeira_kernel::{Link, LoadedRun};
253use tokeira_managed_dsql::{
254 AwsDsqlControlPlane, CanonicalClusterIdentity, ClusterAction, CreateOrRecoverRequest,
255 DsqlControlPlane, LifecycleEnvironment, LocalClusterDescriptorStore, ManagedDsqlLifecycle,
256 Readiness, ResolvedCluster, StartupDeadline, SystemLifecycleEnvironment,
257};
258use tokeira_observability::{
259 ClusterStatusLabel, DbClassLabel, EmbeddedOperationLabel, EmbeddedStorageModeLabel,
260 ErrorClassLabel, OwnershipOutcomeLabel, SchemaOutcomeLabel, record_embedded_lifecycle,
261};
262use tokeira_projection::{
263 DsqlVisibilityStore, InMemoryVisibilityStore, ProjectionSink, ProjectorHome,
264 ProjectorSupervisor, SearchAttrType, VisibilityQueryService, VisibilitySink, VisibilityStore,
265};
266use tokeira_runtime::{
267 ChannelDemandObservationSink, ChannelWorkerComputeReconcileSink,
268 CompletionCallbackScannerConfig, CompletionDeliveryOutcome, ConnectionBudgetApplier,
269 HttpNexusClient, HttpNexusCompletionClient, InMemoryNexusEndpointStore, MembershipConfig,
270 NEXUS_CALLBACK_PATH, NEXUS_OPERATION_STATE_HEADER, NEXUS_OPERATION_TOKEN_HEADER,
271 NexusCompletion, NexusCompletionClient, NexusCompletionDeps, NexusCompletionRuntimeConfig,
272 NexusEndpointRegistry, NexusEndpointSpec, NexusEndpointSpecTarget, NexusEndpointStore,
273 NexusNamespaceResolver, NexusWorkerComputeProvider, OBSERVATION_CHANNEL_CAPACITY,
274 RECONCILE_CHANNEL_CAPACITY, RepositoryBackedTaskQueueConfigStore, RuntimeConfig,
275 RuntimeShutdownHandle, ScheduleEngineConfig, ScheduleStore, SweepResult,
276 SystemWorkerComputeClock, TEMPORAL_CALLBACK_TOKEN_HEADER, TokeiraRuntime,
277 WorkerComputeControllerService, WorkerComputeOutbox, WorkerComputeReconciler,
278 WorkflowTaskReportedProblem, nexus_payload_to_body, reported_problem_from_state,
279 run_schedule_engine, system_callback_post_url,
280};
281use tokeira_storage::{
282 ConnectionDirector, DbClass, InMemoryStore, InMemoryWorkerComputeRepository, LeaseOutcome,
283 LeaseRepository, ProjectionLog, RunRepository, TaskQueueConfigRepository,
284 WorkerComputeRepository, WorkerDeploymentRepository, WorkerTaskProvenanceStore,
285 dsql::{
286 ConnectionControlLeaseRepository, ControlLeaseAcquireOutcome, ControlLeaseAcquireRequest,
287 ControlLeaseClusterIdentity, ControlLeaseError, ControlLeaseGuard, DsqlAuthConfig,
288 DsqlConnectionDirector, DsqlCoordinationConfig, DsqlPoolConfig, DsqlStore,
289 EmbeddedDsqlPoolConfig, MigrationRunner, OWNER_ADMISSION_MARGIN, OWNER_LEASE_DURATION,
290 OWNER_RENEW_INTERVAL, OwnershipAdmissionGate, OwnershipAdmissionState,
291 SCHEMA_MIGRATION_ADMISSION_MARGIN, SCHEMA_MIGRATION_LEASE_DURATION,
292 SchemaCompatibilityError, SchemaDecision, SchemaIncompatibilityCategory,
293 SchemaMigrationPolicy, WarmupDeadline,
294 },
295};
296use tokeira_types::{
297 ExecutionRef, IncarnationId, NamespaceId, NodeEndpoint, PlacementConfig, SearchAttrValue,
298 ShardEpoch, ShardId, WorkflowId,
299};
300
301#[allow(dead_code)]
305#[derive(Clone, Debug)]
306pub enum BootstrapNexusEndpointTarget {
307 External {
308 address: String,
309 },
310 Worker {
311 namespace_name: String,
312 task_queue: String,
313 },
314}
315
316#[derive(Debug)]
317struct VisibilityRegistryOperatorApi<V> {
318 inner: InMemoryOperatorApi,
319 visibility_store: V,
320 declared_search_attributes: Vec<(String, SearchAttrType)>,
321}
322
323impl<V> VisibilityRegistryOperatorApi<V> {
324 fn new(inner: InMemoryOperatorApi, visibility_store: V) -> Self {
325 Self {
326 inner,
327 visibility_store,
328 declared_search_attributes: Vec::new(),
329 }
330 }
331
332 fn with_declared_search_attributes(mut self, declared: Vec<(String, SearchAttrType)>) -> Self {
333 self.declared_search_attributes = declared;
334 self
335 }
336}
337
338#[async_trait]
339impl<V> OperatorApi for VisibilityRegistryOperatorApi<V>
340where
341 V: VisibilityStore + Clone + 'static,
342{
343 async fn cluster_info(&self) -> Result<ClusterInfo> {
344 self.inner.cluster_info().await
345 }
346
347 async fn list_search_attributes(
348 &self,
349 namespace: Option<&str>,
350 ) -> Result<Vec<SearchAttributeDefinition>> {
351 self.inner.list_search_attributes(namespace).await
352 }
353
354 async fn upsert_search_attribute(
355 &self,
356 namespace: &str,
357 attr: SearchAttributeDefinition,
358 ) -> Result<()> {
359 let attr_type = visibility_search_attr_type(&attr.attr_type)?;
360 self.visibility_store
365 .register_attr(namespace_id_for(namespace), attr.name.clone(), attr_type)
366 .await?;
367 self.inner.upsert_search_attribute(namespace, attr).await
368 }
369
370 async fn remove_search_attribute(&self, namespace: &str, attr_name: &str) -> Result<()> {
371 self.inner
372 .remove_search_attribute(namespace, attr_name)
373 .await
374 }
375
376 async fn seed_predefined_search_attributes(&self, namespace: &str) -> Result<()> {
382 tokeira_projection::seed_predefined_search_attributes(
383 &self.visibility_store,
384 namespace_id_for(namespace),
385 )
386 .await?;
387 seed_declared_search_attributes(
391 &self.visibility_store,
392 namespace_id_for(namespace),
393 &self.declared_search_attributes,
394 )
395 .await?;
396 Ok(())
397 }
398}
399
400struct CacheNexusNamespaceResolver {
411 namespaces: Arc<dyn NamespaceCache>,
412}
413
414#[async_trait]
415impl NexusNamespaceResolver for CacheNexusNamespaceResolver {
416 async fn name_for_id(&self, namespace_id: NamespaceId) -> Result<Option<String>> {
417 Ok(self
418 .namespaces
419 .list_all()
420 .await?
421 .into_iter()
422 .map(|namespace| namespace.name)
423 .find(|name| namespace_id_for(name) == namespace_id))
424 }
425}
426
427fn visibility_search_attr_type(value: &str) -> Result<SearchAttrType> {
428 match value {
429 "keyword" => Ok(SearchAttrType::Keyword),
430 "keyword_list" => Ok(SearchAttrType::KeywordList),
431 "int" => Ok(SearchAttrType::Int),
432 "bool" => Ok(SearchAttrType::Bool),
433 "double" => Ok(SearchAttrType::Double),
434 "datetime" => Ok(SearchAttrType::Datetime),
435 "text" => Ok(SearchAttrType::Text),
436 other => Err(anyhow!("unsupported search attribute type `{other}`")),
437 }
438}
439
440#[allow(dead_code)]
443#[derive(Clone, Debug)]
444pub struct BootstrapNexusEndpointConfig {
445 pub target: BootstrapNexusEndpointTarget,
446}
447
448#[derive(Debug)]
459pub struct Engine {
460 #[cfg_attr(not(feature = "chasm-extensions"), allow(dead_code))]
462 component_visibility_store: chasm_extensions::VisibilityQueryStore,
463 #[cfg_attr(not(feature = "chasm-extensions"), allow(dead_code))]
465 chasm_engine: Arc<tokeira_runtime::chasm::ChasmEngine>,
466 #[cfg_attr(not(feature = "chasm-extensions"), allow(dead_code))]
467 registry: Arc<tokeira_chasm::Registry>,
468 endpoint: TemporalEndpoint,
469 background_cancel: CancellationToken,
470 log_broadcast: broadcast::Sender<LogEvent>,
471 snapshot_policy: Option<EngineSnapshotPolicy>,
472 recovery_task: Option<JoinHandle<Result<()>>>,
473 startup_report: EngineStartupReport,
474 shutdown_coordinator: Option<EmbeddedShutdownCoordinator>,
475 listeners: listener::ListenerRegistry,
476}
477
478#[derive(Clone, Copy, Debug, PartialEq, Eq)]
480pub enum EmbeddedStorageMode {
481 InMemory,
483 ManagedDsql,
485 ExistingDsql,
487}
488
489#[derive(Clone, Debug, PartialEq, Eq)]
491pub struct EngineStartupReport {
492 pub storage_mode: EmbeddedStorageMode,
494 pub cluster: Option<ClusterStartupReport>,
496 pub schema: Option<SchemaStartupReport>,
498 pub ownership: Option<OwnershipStartupReport>,
500}
501
502impl EngineStartupReport {
503 fn in_memory() -> Self {
504 Self {
505 storage_mode: EmbeddedStorageMode::InMemory,
506 cluster: None,
507 schema: None,
508 ownership: None,
509 }
510 }
511}
512
513#[derive(Clone, Debug, PartialEq, Eq)]
515pub struct ClusterStartupReport {
516 pub region: String,
518 pub cluster_id: String,
520 pub cluster_arn: String,
522 pub endpoint: String,
528 pub action: ClusterAction,
530}
531
532#[derive(Clone, Copy, Debug, PartialEq, Eq)]
534pub enum SchemaStartupOutcome {
535 Compatible,
537 Initialized,
539 Migrated,
541 MetadataBackfilled,
543}
544
545#[derive(Clone, Debug, PartialEq, Eq)]
547pub struct SchemaStartupReport {
548 pub observed_version: u32,
550 pub minimum_supported_version: u32,
552 pub target_version: u32,
554 pub maximum_readable_version: u32,
556 pub migration_set_digest: String,
558 pub outcome: SchemaStartupOutcome,
560}
561
562#[derive(Clone, Copy, Debug, PartialEq, Eq)]
564pub struct OwnershipStartupReport {
565 pub outcome: ControlLeaseAcquireOutcome,
567 pub fence_token: i64,
569}
570
571#[derive(Clone, Copy, Debug, PartialEq, Eq)]
573pub enum EmbeddedStartupPhase {
574 Configuration,
576 ClusterResolution,
578 ConnectionWarmup,
580 Schema,
582 Ownership,
584 RuntimeRestore,
586}
587
588impl std::fmt::Display for EmbeddedStartupPhase {
589 fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
590 formatter.write_str(match self {
591 Self::Configuration => "configuration validation",
592 Self::ClusterResolution => "cluster resolution",
593 Self::ConnectionWarmup => "DSQL connection warmup",
594 Self::Schema => "schema compatibility",
595 Self::Ownership => "embedded ownership",
596 Self::RuntimeRestore => "runtime restoration",
597 })
598 }
599}
600
601#[derive(Debug)]
603pub enum EmbeddedEngineStartError {
604 Registry(tokeira_chasm::ChasmError),
606 UnregisteredArchetype {
608 archetype_id: u32,
610 executions: u64,
612 },
613 SearchAttributeType {
615 namespace: NamespaceId,
617 key: String,
619 declared: SearchAttrType,
621 existing: SearchAttrType,
623 },
624 UnserviceableOutbox {
626 archetype_id: u32,
628 cause: tokeira_runtime::chasm::RebuildFailure,
630 executions: usize,
632 },
633 InvalidConfiguration(tokeira_config::EmbeddedConfigError),
635 Phase {
637 phase: EmbeddedStartupPhase,
639 },
640 DeadlineExceeded {
642 phase: EmbeddedStartupPhase,
644 },
645}
646
647impl std::fmt::Display for EmbeddedEngineStartError {
648 fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
649 match self {
650 Self::Registry(error) => write!(
651 formatter,
652 "CHASM registration failed: {error}; correct the conflicting library or executor before restarting"
653 ),
654 Self::UnregisteredArchetype {
655 archetype_id,
656 executions,
657 } => write!(
658 formatter,
659 "CHASM archetype {archetype_id} is not registered but has {executions} stored executions; load its library before restarting"
660 ),
661 Self::SearchAttributeType {
662 namespace,
663 key,
664 declared,
665 existing,
666 } => write!(
667 formatter,
668 "namespace {} search attribute `{key}` declares {declared:?} but storage has {existing:?}; restore the matching declaration or correct the registered type before restarting",
669 namespace.0
670 ),
671 Self::UnserviceableOutbox {
672 archetype_id,
673 cause,
674 executions,
675 } => write!(
676 formatter,
677 "CHASM rebuild cannot serve {executions} executions; first failure in archetype {archetype_id}: {cause}; restore the matching library or repair its persisted data before restarting"
678 ),
679 Self::InvalidConfiguration(error) => {
680 write!(formatter, "invalid embedded engine configuration: {error}")
681 }
682 Self::Phase { phase } => {
683 write!(formatter, "embedded engine startup failed during {phase}")
684 }
685 Self::DeadlineExceeded { phase } => {
686 write!(
687 formatter,
688 "embedded engine startup deadline exceeded during {phase}"
689 )
690 }
691 }
692 }
693}
694
695impl std::error::Error for EmbeddedEngineStartError {}
696
697impl From<tokeira_config::EmbeddedConfigError> for EmbeddedEngineStartError {
698 fn from(error: tokeira_config::EmbeddedConfigError) -> Self {
699 Self::InvalidConfiguration(error)
700 }
701}
702
703#[derive(Clone, Copy, Debug, PartialEq, Eq)]
705pub enum EmbeddedShutdownFailure {
706 RpcDrain,
708 RuntimeTasks,
710 ShardRelease,
712 OwnershipRelease,
714 Storage,
716 Snapshot,
718 ListenerDrain,
720}
721
722#[derive(Debug)]
724pub struct EmbeddedEngineShutdownError {
725 failures: Vec<EmbeddedShutdownFailure>,
726}
727
728impl std::fmt::Display for EmbeddedEngineShutdownError {
729 fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
730 write!(
731 formatter,
732 "embedded engine shutdown completed with {} cleanup failure(s)",
733 self.failures.len()
734 )
735 }
736}
737
738impl std::error::Error for EmbeddedEngineShutdownError {}
739
740impl EmbeddedEngineShutdownError {
741 pub fn failures(&self) -> &[EmbeddedShutdownFailure] {
743 &self.failures
744 }
745}
746
747type CleanupFuture = Pin<Box<dyn Future<Output = Result<()>> + Send>>;
748type CleanupAction = Box<dyn FnOnce(Instant) -> CleanupFuture + Send>;
749
750#[derive(Clone)]
751struct EmbeddedOwnership {
752 repository: ConnectionControlLeaseRepository,
753 guard: Arc<tokio::sync::Mutex<Option<ControlLeaseGuard>>>,
754 gate: OwnershipAdmissionGate,
755}
756
757impl std::fmt::Debug for EmbeddedOwnership {
758 fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
759 formatter
760 .debug_struct("EmbeddedOwnership")
761 .field("gate", &self.gate)
762 .finish_non_exhaustive()
763 }
764}
765
766struct EmbeddedShutdownCoordinator {
767 service: InProcessGrpcService,
768 runtime_tasks: RuntimeShutdownHandle,
769 engine_tasks: RuntimeShutdownHandle,
770 shard_cleanup: Option<CleanupAction>,
771 director: Option<Arc<DsqlConnectionDirector>>,
772 ownership: Option<EmbeddedOwnership>,
773}
774
775impl std::fmt::Debug for EmbeddedShutdownCoordinator {
776 fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
777 formatter
778 .debug_struct("EmbeddedShutdownCoordinator")
779 .field("service", &self.service)
780 .field("runtime_tasks", &self.runtime_tasks)
781 .field("engine_tasks", &self.engine_tasks)
782 .field("has_shard_cleanup", &self.shard_cleanup.is_some())
783 .field("has_director", &self.director.is_some())
784 .field("ownership", &self.ownership)
785 .finish()
786 }
787}
788
789#[derive(Debug)]
790struct EngineSnapshotPolicy {
791 store: InMemoryStore,
792 location: PathBuf,
793 periodic_task: Option<JoinHandle<()>>,
794}
795
796static SNAPSHOT_TEMP_SEQUENCE: AtomicU64 = AtomicU64::new(0);
797
798impl EngineSnapshotPolicy {
799 fn start(
800 store: InMemoryStore,
801 config: SnapshotPolicyConfig,
802 cancel: CancellationToken,
803 ) -> Self {
804 let interval_duration = std::time::Duration::from_millis(config.interval_ms);
805 let location = config.location;
806 let periodic_store = store.clone();
807 let periodic_location = location.clone();
808 let first_tick = tokio::time::Instant::now() + interval_duration;
812 let periodic_task = tokio::spawn(async move {
813 let mut interval = tokio::time::interval_at(first_tick, interval_duration);
818 interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
819 loop {
820 tokio::select! {
821 biased;
822 _ = cancel.cancelled() => break,
823 _ = interval.tick() => {
824 if let Err(error) = persist_snapshot(&periodic_store, &periodic_location).await {
825 error!(
826 snapshot_path = %periodic_location.display(),
827 error = %error,
828 "failed to persist periodic engine snapshot"
829 );
830 }
831 }
832 }
833 }
834 });
835 Self {
836 store,
837 location,
838 periodic_task: Some(periodic_task),
839 }
840 }
841}
842
843#[derive(Clone, Debug)]
850pub struct TemporalEndpoint {
851 service: InProcessGrpcService,
852 shutdown: CancellationToken,
853}
854
855impl Engine {
856 #[cfg(feature = "chasm-extensions")]
862 pub fn chasm_visibility<C>(
863 &self,
864 namespace_id: tokeira_types::NamespaceId,
865 ) -> Result<tokeira_projection::ComponentVisibility, tokeira_chasm::ChasmError>
866 where
867 C: tokeira_chasm::EngineComponent
868 + tokeira_chasm::RootComponent
869 + tokeira_chasm::SearchAttributeProvider
870 + tokeira_chasm::VisibilityContributor,
871 {
872 let archetype_id = self.registry.archetype_id(C::FQN).ok_or_else(|| {
873 tokeira_chasm::ChasmError::Internal(format!("archetype `{}` is not registered", C::FQN))
874 })?;
875 Ok(tokeira_projection::ComponentVisibility::new(
876 self.component_visibility_store.0.clone(),
877 namespace_id,
878 tokeira_types::ArchetypeId(archetype_id),
879 ))
880 }
881 #[cfg(feature = "chasm-extensions")]
883 pub fn builder(config: EmbeddedEngineConfig) -> EngineBuilder {
884 EngineBuilder {
885 config,
886 extensions: ChasmExtensions::default(),
887 }
888 }
889
890 #[cfg(feature = "chasm-extensions")]
893 pub fn chasm<C>(
894 &self,
895 ) -> Result<tokeira_runtime::chasm::TypedEngine<C>, tokeira_chasm::ChasmError>
896 where
897 C: tokeira_chasm::EngineComponent
898 + tokeira_chasm::RootComponent
899 + tokeira_chasm::SearchAttributeProvider
900 + tokeira_chasm::VisibilityContributor,
901 {
902 if self.registry.archetype_id(C::FQN).is_none() {
903 return Err(tokeira_chasm::ChasmError::Internal(format!(
904 "archetype `{}` is not registered",
905 C::FQN
906 )));
907 }
908 Ok(tokeira_runtime::chasm::TypedEngine::new(
909 self.chasm_engine.clone(),
910 ))
911 }
912
913 pub async fn start() -> Result<Self> {
915 Self::start_with_config(TokeiraConfig::default()).await
916 }
917
918 pub async fn embedded() -> Result<Self> {
920 Self::start().await
921 }
922
923 pub async fn start_with_config(config: TokeiraConfig) -> Result<Self> {
929 Self::start_in_memory_with_extensions(config, ChasmExtensions::default()).await
930 }
931
932 async fn start_in_memory_with_extensions(
933 config: TokeiraConfig,
934 extensions: ChasmExtensions,
935 ) -> Result<Self> {
936 let config = embedded_config(config)?;
937 let snapshot_config = config.policy.snapshot.clone();
938 let (store, restored) = restore_snapshot_store(&config).await?;
939 let stack = build_embedded(Arc::new(config), store.clone(), restored, extensions).await?;
940 let snapshot_policy = snapshot_config.map(|config| {
941 EngineSnapshotPolicy::start(store, config, stack.background_cancel.clone())
942 });
943 stack.engine_tasks.close_registration();
944 let shutdown_coordinator = EmbeddedShutdownCoordinator {
945 service: stack.service.clone(),
946 runtime_tasks: stack.runtime_tasks,
947 engine_tasks: stack.engine_tasks,
948 shard_cleanup: stack.shard_cleanup,
949 director: None,
950 ownership: None,
951 };
952 let engine = Self {
953 component_visibility_store: stack.component_visibility_store,
954 chasm_engine: stack.chasm_engine,
955 registry: stack.registry,
956 endpoint: TemporalEndpoint {
957 service: stack.service,
958 shutdown: stack.background_cancel.clone(),
959 },
960 background_cancel: stack.background_cancel,
961 log_broadcast: stack.log_broadcast,
962 snapshot_policy,
963 recovery_task: stack.recovery_task,
964 startup_report: EngineStartupReport::in_memory(),
965 shutdown_coordinator: Some(shutdown_coordinator),
966 listeners: listener::ListenerRegistry::default(),
967 };
968 record_embedded_startup(&engine.startup_report);
969 Ok(engine)
970 }
971
972 pub async fn start_with_embedded_config(
978 config: EmbeddedEngineConfig,
979 ) -> Result<Self, EmbeddedEngineStartError> {
980 Self::start_with_extensions(config, ChasmExtensions::default()).await
981 }
982
983 async fn start_with_extensions(
984 config: EmbeddedEngineConfig,
985 extensions: ChasmExtensions,
986 ) -> Result<Self, EmbeddedEngineStartError> {
987 if !matches!(config.storage, EmbeddedStorageConfig::InMemory)
988 && config.server.policy.snapshot.is_some()
989 {
990 return Err(EmbeddedEngineStartError::Phase {
991 phase: EmbeddedStartupPhase::Configuration,
992 });
993 }
994 config.validate()?;
995 if matches!(config.storage, EmbeddedStorageConfig::InMemory) {
996 return Self::start_in_memory_with_extensions(config.server, extensions)
997 .await
998 .map_err(embedded_stack_error);
999 }
1000 start_embedded_dsql(config, extensions).await
1001 }
1002
1003 pub fn startup_report(&self) -> &EngineStartupReport {
1005 &self.startup_report
1006 }
1007
1008 pub fn endpoint(&self) -> TemporalEndpoint {
1010 self.endpoint.clone()
1011 }
1012
1013 pub fn log_sink(&self) -> broadcast::Receiver<LogEvent> {
1015 self.log_broadcast.subscribe()
1016 }
1017
1018 #[cfg(feature = "temporalio-client")]
1023 pub fn service_override(&self) -> temporalio_client::callback_based::CallbackBasedGrpcService {
1024 self.endpoint.service_override()
1025 }
1026
1027 pub async fn shutdown(mut self) -> Result<(), EmbeddedEngineShutdownError> {
1041 let deadline = Instant::now() + StdDuration::from_secs(30);
1042 let mut failures = Vec::new();
1043 if let Some(mut coordinator) = self.shutdown_coordinator.take() {
1044 coordinator.begin_shutdown();
1045 self.listeners.stop_all(deadline, &mut failures).await;
1046 coordinator.shutdown(deadline, &mut failures).await;
1047 } else {
1048 self.listeners.stop_all(deadline, &mut failures).await;
1049 self.background_cancel.cancel();
1050 }
1051 if let Some(task) = self.recovery_task.take()
1052 && task.await.is_err()
1053 {
1054 failures.push(EmbeddedShutdownFailure::ShardRelease);
1055 }
1056 if let Some(policy) = self.snapshot_policy.take()
1057 && finish_snapshot_policy(policy).await.is_err()
1058 {
1059 failures.push(EmbeddedShutdownFailure::Snapshot);
1060 }
1061 record_embedded_shutdown(&self.startup_report, &failures);
1062 if failures.is_empty() {
1063 Ok(())
1064 } else {
1065 Err(EmbeddedEngineShutdownError { failures })
1066 }
1067 }
1068}
1069
1070impl EmbeddedShutdownCoordinator {
1071 fn begin_shutdown(&self) {
1072 self.service.begin_shutdown();
1073 self.engine_tasks.begin_shutdown();
1074 self.runtime_tasks.begin_shutdown();
1075 }
1076
1077 async fn shutdown(&mut self, deadline: Instant, failures: &mut Vec<EmbeddedShutdownFailure>) {
1078 if self.service.drain(deadline).await.is_err() {
1079 failures.push(EmbeddedShutdownFailure::RpcDrain);
1080 }
1081 if self.engine_tasks.wait(deadline).await.is_err() {
1082 failures.push(EmbeddedShutdownFailure::RuntimeTasks);
1083 }
1084 if self.runtime_tasks.wait(deadline).await.is_err() {
1085 failures.push(EmbeddedShutdownFailure::RuntimeTasks);
1086 }
1087 if let Some(cleanup) = self.shard_cleanup.take()
1088 && cleanup(deadline).await.is_err()
1089 {
1090 failures.push(EmbeddedShutdownFailure::ShardRelease);
1091 }
1092 if let Some(ownership) = &self.ownership {
1098 ownership.gate.begin_closing();
1099 }
1100 if let (Some(ownership), Some(director)) = (&self.ownership, &self.director) {
1101 let now = Instant::now();
1102 let ownership_drained = now < deadline
1103 && tokio::time::timeout(
1104 deadline.saturating_duration_since(now),
1105 ownership.gate.wait_for_drain(),
1106 )
1107 .await
1108 .is_ok();
1109 if ownership_drained {
1110 let now = Instant::now();
1111 let release_result = if now < deadline {
1112 tokio::time::timeout(
1113 deadline.saturating_duration_since(now),
1114 release_embedded_ownership(ownership, director),
1115 )
1116 .await
1117 .map_err(|_| anyhow!("embedded ownership release deadline elapsed"))
1118 .and_then(|result| result)
1119 } else {
1120 Err(anyhow!("embedded ownership release deadline elapsed"))
1121 };
1122 if release_result.is_err() {
1123 failures.push(EmbeddedShutdownFailure::OwnershipRelease);
1124 }
1125 } else {
1126 failures.push(EmbeddedShutdownFailure::OwnershipRelease);
1130 }
1131 }
1132 if let Some(director) = &self.director
1133 && director.shutdown_with_deadline(deadline).await.is_err()
1134 {
1135 failures.push(EmbeddedShutdownFailure::Storage);
1136 }
1137 }
1138}
1139
1140async fn start_embedded_dsql(
1141 config: EmbeddedEngineConfig,
1142 extensions: ChasmExtensions,
1143) -> Result<Engine, EmbeddedEngineStartError> {
1144 let deadline = Instant::now() + StdDuration::from_millis(config.startup_timeout_ms);
1145 let storage_mode = match &config.storage {
1146 EmbeddedStorageConfig::ManagedDsql(_) => EmbeddedStorageMode::ManagedDsql,
1147 EmbeddedStorageConfig::ExistingDsql(_) => EmbeddedStorageMode::ExistingDsql,
1148 EmbeddedStorageConfig::InMemory => unreachable!("in-memory handled by caller"),
1149 };
1150 let region = match &config.storage {
1151 EmbeddedStorageConfig::ManagedDsql(managed) => managed.region.clone(),
1152 EmbeddedStorageConfig::ExistingDsql(existing) => existing.region.clone(),
1153 EmbeddedStorageConfig::InMemory => unreachable!("in-memory handled by caller"),
1154 };
1155 let control = startup_infallible_phase(
1156 deadline,
1157 EmbeddedStartupPhase::ClusterResolution,
1158 AwsDsqlControlPlane::from_region(region.clone()),
1159 )
1160 .await?;
1161 let migration_policy = config
1162 .effective_migration_policy()
1163 .expect("durable embedded mode always has a migration policy");
1164 let storage = config.storage.clone();
1165 let ConnectedEmbeddedDsql {
1166 server,
1167 cluster,
1168 connection_endpoint,
1169 store: dsql_store,
1170 wake_required,
1171 } = connect_embedded_dsql(
1172 config,
1173 control.clone(),
1174 SystemLifecycleEnvironment,
1175 deadline,
1176 DsqlStore::connect_embedded,
1177 )
1178 .await?;
1179 let lifecycle_deadline = StartupDeadline::at(deadline);
1180 let director = dsql_store.connection_director_arc();
1181
1182 let cluster = if wake_required {
1183 match refresh_cluster_after_wake(
1184 &storage,
1185 control,
1186 SystemLifecycleEnvironment,
1187 cluster,
1188 lifecycle_deadline,
1189 deadline,
1190 )
1191 .await
1192 {
1193 Ok(cluster) => cluster,
1194 Err(error) => {
1195 let _ = director.shutdown_with_deadline(deadline).await;
1196 return Err(error);
1197 }
1198 }
1199 } else {
1200 cluster
1201 };
1202
1203 let schema = match startup_phase(
1204 deadline,
1205 EmbeddedStartupPhase::Schema,
1206 apply_embedded_schema(
1207 director.clone(),
1208 &cluster,
1209 schema_migration_policy(migration_policy),
1210 deadline,
1211 ),
1212 )
1213 .await
1214 {
1215 Ok(report) => report,
1216 Err(error) => {
1217 let _ = director.shutdown_with_deadline(deadline).await;
1218 return Err(error);
1219 }
1220 };
1221
1222 let (ownership, ownership_report) = match startup_phase(
1223 deadline,
1224 EmbeddedStartupPhase::Ownership,
1225 acquire_embedded_ownership(director.clone(), &cluster, deadline),
1226 )
1227 .await
1228 {
1229 Ok(ownership) => ownership,
1230 Err(error) => {
1231 let _ = director.shutdown_with_deadline(deadline).await;
1232 return Err(error);
1233 }
1234 };
1235 if director
1236 .install_ownership_gate(ownership.gate.clone())
1237 .is_err()
1238 {
1239 rollback_embedded_dsql(&ownership, &director, deadline).await;
1240 return Err(EmbeddedEngineStartError::Phase {
1241 phase: EmbeddedStartupPhase::Ownership,
1242 });
1243 }
1244
1245 let stack =
1246 match startup_infallible_phase(deadline, EmbeddedStartupPhase::RuntimeRestore, async {
1247 build_dsql_stack(
1248 StackTransport::Embedded,
1249 Arc::new(server),
1250 dsql_store,
1251 connection_endpoint.clone(),
1252 extensions,
1253 None,
1254 )
1255 .await
1256 .map_err(embedded_stack_error)
1257 })
1258 .await
1259 .and_then(|result| result)
1260 {
1261 Ok(ConstructedStack::Embedded(stack)) => stack,
1262 Ok(ConstructedStack::Network(_)) => {
1263 rollback_embedded_dsql(&ownership, &director, deadline).await;
1264 return Err(EmbeddedEngineStartError::Phase {
1265 phase: EmbeddedStartupPhase::RuntimeRestore,
1266 });
1267 }
1268 Err(error) => {
1269 rollback_embedded_dsql(&ownership, &director, deadline).await;
1270 return Err(error);
1271 }
1272 };
1273
1274 spawn_ownership_renewal(
1275 &stack.engine_tasks,
1276 director.clone(),
1277 ownership.clone(),
1278 stack.service.clone(),
1279 );
1280 stack.engine_tasks.close_registration();
1281 let report = EngineStartupReport {
1282 storage_mode,
1283 cluster: Some(cluster_startup_report(&cluster, &connection_endpoint)),
1284 schema: Some(schema),
1285 ownership: Some(ownership_report),
1286 };
1287 let coordinator = EmbeddedShutdownCoordinator {
1288 service: stack.service.clone(),
1289 runtime_tasks: stack.runtime_tasks,
1290 engine_tasks: stack.engine_tasks,
1291 shard_cleanup: stack.shard_cleanup,
1292 director: Some(director),
1293 ownership: Some(ownership),
1294 };
1295 let engine = Engine {
1296 component_visibility_store: stack.component_visibility_store,
1297 chasm_engine: stack.chasm_engine,
1298 registry: stack.registry,
1299 endpoint: TemporalEndpoint {
1300 service: stack.service,
1301 shutdown: stack.background_cancel.clone(),
1302 },
1303 background_cancel: stack.background_cancel,
1304 log_broadcast: stack.log_broadcast,
1305 snapshot_policy: None,
1306 recovery_task: stack.recovery_task,
1307 startup_report: report,
1308 shutdown_coordinator: Some(coordinator),
1309 listeners: listener::ListenerRegistry::default(),
1310 };
1311 record_embedded_startup(&engine.startup_report);
1312 Ok(engine)
1313}
1314
1315#[derive(Debug)]
1319struct ConnectedEmbeddedDsql<S> {
1320 server: TokeiraConfig,
1321 cluster: ResolvedCluster,
1322 connection_endpoint: String,
1323 store: S,
1324 wake_required: bool,
1325}
1326
1327async fn connect_embedded_dsql<C, T, F, Fut, S>(
1330 config: EmbeddedEngineConfig,
1331 control: C,
1332 time: T,
1333 deadline: Instant,
1334 connect: F,
1335) -> Result<ConnectedEmbeddedDsql<S>, EmbeddedEngineStartError>
1336where
1337 C: DsqlControlPlane + Clone,
1338 T: LifecycleEnvironment + Clone,
1339 F: FnOnce(DsqlAuthConfig, EmbeddedDsqlPoolConfig, WarmupDeadline) -> Fut,
1340 Fut: Future<Output = Result<S>>,
1341{
1342 let lifecycle_deadline = StartupDeadline::at(deadline);
1343 let resolved = match &config.storage {
1344 EmbeddedStorageConfig::ManagedDsql(managed) => {
1345 let lifecycle = ManagedDsqlLifecycle::new(
1346 control.clone(),
1347 LocalClusterDescriptorStore::new(&managed.descriptor_path),
1348 time.clone(),
1349 );
1350 startup_phase(
1351 deadline,
1352 EmbeddedStartupPhase::ClusterResolution,
1353 lifecycle.create_or_recover(
1354 CreateOrRecoverRequest {
1355 region: managed.region.clone(),
1356 tags: managed.tags.clone(),
1357 },
1358 lifecycle_deadline,
1359 ),
1360 )
1361 .await?
1362 }
1363 EmbeddedStorageConfig::ExistingDsql(existing) => {
1364 let unused_descriptor = LocalClusterDescriptorStore::new(PathBuf::new());
1368 let lifecycle =
1369 ManagedDsqlLifecycle::new(control.clone(), unused_descriptor, time.clone());
1370 let identity = CanonicalClusterIdentity::new(
1371 &existing.region,
1372 &existing.cluster_id,
1373 &existing.cluster_arn,
1374 )
1375 .map_err(|_| EmbeddedEngineStartError::Phase {
1376 phase: EmbeddedStartupPhase::ClusterResolution,
1377 })?;
1378 startup_phase(
1379 deadline,
1380 EmbeddedStartupPhase::ClusterResolution,
1381 lifecycle.resolve_existing(identity, lifecycle_deadline),
1382 )
1383 .await?
1384 }
1385 EmbeddedStorageConfig::InMemory => unreachable!("in-memory handled by caller"),
1386 };
1387 let readiness = refresh_cluster_until_storage_handoff(
1388 &config.storage,
1389 control.clone(),
1390 time,
1391 resolved,
1392 lifecycle_deadline,
1393 deadline,
1394 )
1395 .await?;
1396 let cluster = match &readiness {
1397 Readiness::Active(usable) => usable.resolved().clone(),
1398 Readiness::WakeRequired(cluster) => cluster.clone(),
1399 };
1400
1401 let connection_endpoint = match &config.storage {
1408 EmbeddedStorageConfig::ExistingDsql(existing) => existing.endpoint.clone(),
1409 EmbeddedStorageConfig::ManagedDsql(_) => cluster.endpoint.clone(),
1410 EmbeddedStorageConfig::InMemory => unreachable!("in-memory handled by caller"),
1411 };
1412 let mut server = config.server;
1413 server.infrastructure.storage = ConfigStorageKind::Dsql;
1414 server.infrastructure.placement.controller_endpoint = None;
1415 server.infrastructure.dsql.endpoint = Some(connection_endpoint.clone());
1416 server.infrastructure.dsql.region = Some(cluster.identity.region.clone());
1417 server
1418 .validate()
1419 .map_err(|_| EmbeddedEngineStartError::Phase {
1420 phase: EmbeddedStartupPhase::Configuration,
1421 })?;
1422
1423 let limits = match &config.storage {
1424 EmbeddedStorageConfig::ManagedDsql(managed) => &managed.limits,
1425 EmbeddedStorageConfig::ExistingDsql(existing) => &existing.limits,
1426 EmbeddedStorageConfig::InMemory => unreachable!("in-memory handled by caller"),
1427 };
1428 let mut pool_config = EmbeddedDsqlPoolConfig::with_limits(
1429 limits.max_connections,
1430 limits.concurrent_connection_creations,
1431 limits.connection_rate_per_second,
1432 limits.connection_burst,
1433 );
1434 pool_config.shard_count = server.infrastructure.placement.shard_count;
1435 pool_config.projection_partition_count = server.infrastructure.placement.partition_count;
1436 let auth = DsqlAuthConfig {
1437 endpoint: connection_endpoint.clone(),
1438 region: Some(cluster.identity.region.clone()),
1439 admin_role_arn: server.infrastructure.dsql.admin_role_arn.clone(),
1440 runtime_role_arn: server.infrastructure.dsql.runtime_role_arn.clone(),
1441 readonly_role_arn: server.infrastructure.dsql.readonly_role_arn.clone(),
1442 };
1443 let store = startup_phase(
1444 deadline,
1445 EmbeddedStartupPhase::ConnectionWarmup,
1446 connect(auth, pool_config, WarmupDeadline::new(deadline)),
1447 )
1448 .await?;
1449 Ok(ConnectedEmbeddedDsql {
1450 server,
1451 cluster,
1452 connection_endpoint,
1453 store,
1454 wake_required: matches!(readiness, Readiness::WakeRequired(_)),
1455 })
1456}
1457
1458fn cluster_startup_report(
1459 cluster: &ResolvedCluster,
1460 connection_endpoint: &str,
1461) -> ClusterStartupReport {
1462 ClusterStartupReport {
1463 region: cluster.identity.region.clone(),
1464 cluster_id: cluster.identity.cluster_id.clone(),
1465 cluster_arn: cluster.identity.cluster_arn.clone(),
1466 endpoint: connection_endpoint.to_owned(),
1467 action: cluster.action,
1468 }
1469}
1470
1471fn storage_mode_label(mode: EmbeddedStorageMode) -> EmbeddedStorageModeLabel {
1472 match mode {
1473 EmbeddedStorageMode::InMemory => EmbeddedStorageModeLabel::InMemory,
1474 EmbeddedStorageMode::ManagedDsql => EmbeddedStorageModeLabel::ManagedDsql,
1475 EmbeddedStorageMode::ExistingDsql => EmbeddedStorageModeLabel::ExistingDsql,
1476 }
1477}
1478
1479fn schema_outcome_label(report: &EngineStartupReport) -> SchemaOutcomeLabel {
1480 match report.schema.as_ref().map(|schema| schema.outcome) {
1481 None => SchemaOutcomeLabel::NotApplicable,
1482 Some(SchemaStartupOutcome::Compatible) => SchemaOutcomeLabel::Compatible,
1483 Some(SchemaStartupOutcome::Initialized) => SchemaOutcomeLabel::Initialized,
1484 Some(SchemaStartupOutcome::Migrated) => SchemaOutcomeLabel::Migrated,
1485 Some(SchemaStartupOutcome::MetadataBackfilled) => SchemaOutcomeLabel::MetadataBackfilled,
1486 }
1487}
1488
1489fn ownership_outcome_label(report: &EngineStartupReport) -> OwnershipOutcomeLabel {
1490 match report.ownership.map(|ownership| ownership.outcome) {
1491 None => OwnershipOutcomeLabel::NotApplicable,
1492 Some(ControlLeaseAcquireOutcome::Clean) => OwnershipOutcomeLabel::AcquiredClean,
1493 Some(ControlLeaseAcquireOutcome::ExpiredTakeover) => OwnershipOutcomeLabel::AcquiredExpired,
1494 }
1495}
1496
1497fn record_embedded_startup(report: &EngineStartupReport) {
1503 record_embedded_lifecycle(
1504 storage_mode_label(report.storage_mode),
1505 if report.cluster.is_some() {
1506 ClusterStatusLabel::Active
1507 } else {
1508 ClusterStatusLabel::NotApplicable
1509 },
1510 schema_outcome_label(report),
1511 ownership_outcome_label(report),
1512 DbClassLabel::Control,
1513 EmbeddedOperationLabel::Startup,
1514 ErrorClassLabel::None,
1515 );
1516}
1517
1518fn record_embedded_shutdown(report: &EngineStartupReport, failures: &[EmbeddedShutdownFailure]) {
1519 let ownership = if report.ownership.is_some() && failures.is_empty() {
1520 OwnershipOutcomeLabel::Released
1521 } else {
1522 ownership_outcome_label(report)
1523 };
1524 record_embedded_lifecycle(
1525 storage_mode_label(report.storage_mode),
1526 if report.cluster.is_some() {
1527 ClusterStatusLabel::Active
1528 } else {
1529 ClusterStatusLabel::NotApplicable
1530 },
1531 schema_outcome_label(report),
1532 ownership,
1533 DbClassLabel::Control,
1534 EmbeddedOperationLabel::Shutdown,
1535 if failures.is_empty() {
1536 ErrorClassLabel::None
1537 } else {
1538 ErrorClassLabel::Internal
1539 },
1540 );
1541}
1542
1543async fn refresh_cluster_until_storage_handoff<C, T>(
1544 storage: &EmbeddedStorageConfig,
1545 control: C,
1546 time: T,
1547 cluster: ResolvedCluster,
1548 lifecycle_deadline: StartupDeadline,
1549 deadline: Instant,
1550) -> Result<Readiness, EmbeddedEngineStartError>
1551where
1552 C: DsqlControlPlane,
1553 T: LifecycleEnvironment,
1554{
1555 match storage {
1556 EmbeddedStorageConfig::ManagedDsql(managed) => {
1557 let lifecycle = ManagedDsqlLifecycle::new(
1558 control,
1559 LocalClusterDescriptorStore::new(&managed.descriptor_path),
1560 time,
1561 );
1562 startup_phase(
1563 deadline,
1564 EmbeddedStartupPhase::ClusterResolution,
1565 lifecycle.refresh_until_usable(cluster, lifecycle_deadline),
1566 )
1567 .await
1568 }
1569 EmbeddedStorageConfig::ExistingDsql(_) => {
1570 let lifecycle = ManagedDsqlLifecycle::new(
1571 control,
1572 LocalClusterDescriptorStore::new(PathBuf::new()),
1573 time,
1574 );
1575 startup_phase(
1576 deadline,
1577 EmbeddedStartupPhase::ClusterResolution,
1578 lifecycle.refresh_until_usable(cluster, lifecycle_deadline),
1579 )
1580 .await
1581 }
1582 EmbeddedStorageConfig::InMemory => unreachable!("in-memory handled by caller"),
1583 }
1584}
1585
1586async fn refresh_cluster_after_wake<C, T>(
1587 storage: &EmbeddedStorageConfig,
1588 control: C,
1589 time: T,
1590 cluster: ResolvedCluster,
1591 lifecycle_deadline: StartupDeadline,
1592 deadline: Instant,
1593) -> Result<ResolvedCluster, EmbeddedEngineStartError>
1594where
1595 C: DsqlControlPlane,
1596 T: LifecycleEnvironment,
1597{
1598 let usable = match storage {
1599 EmbeddedStorageConfig::ManagedDsql(managed) => {
1600 let lifecycle = ManagedDsqlLifecycle::new(
1601 control,
1602 LocalClusterDescriptorStore::new(&managed.descriptor_path),
1603 time,
1604 );
1605 startup_phase(
1606 deadline,
1607 EmbeddedStartupPhase::ConnectionWarmup,
1608 lifecycle.refresh_after_wake(cluster, lifecycle_deadline),
1609 )
1610 .await?
1611 }
1612 EmbeddedStorageConfig::ExistingDsql(_) => {
1613 let lifecycle = ManagedDsqlLifecycle::new(
1614 control,
1615 LocalClusterDescriptorStore::new(PathBuf::new()),
1616 time,
1617 );
1618 startup_phase(
1619 deadline,
1620 EmbeddedStartupPhase::ConnectionWarmup,
1621 lifecycle.refresh_after_wake(cluster, lifecycle_deadline),
1622 )
1623 .await?
1624 }
1625 EmbeddedStorageConfig::InMemory => unreachable!("in-memory handled by caller"),
1626 };
1627 Ok(usable.into_resolved())
1628}
1629
1630fn schema_migration_policy(policy: DsqlMigrationPolicy) -> SchemaMigrationPolicy {
1631 match policy {
1632 DsqlMigrationPolicy::Automatic => SchemaMigrationPolicy::Automatic,
1633 DsqlMigrationPolicy::ValidateOnly => SchemaMigrationPolicy::ValidateOnly,
1634 }
1635}
1636
1637async fn apply_embedded_schema(
1638 director: Arc<DsqlConnectionDirector>,
1639 cluster: &ResolvedCluster,
1640 policy: SchemaMigrationPolicy,
1641 deadline: Instant,
1642) -> Result<SchemaStartupReport> {
1643 let runner = MigrationRunner::embedded();
1644 let contract = MigrationRunner::compatibility_contract();
1645 let leases = ConnectionControlLeaseRepository::new();
1646 let mut permit = director.acquire(DbClass::Control).await?;
1647 let decision = runner
1648 .assess_connection(permit.connection()?, &contract, policy)
1649 .await?;
1650 let outcome = match &decision {
1651 SchemaDecision::Compatible {
1652 legacy_backfill: false,
1653 ..
1654 } => SchemaStartupOutcome::Compatible,
1655 SchemaDecision::Compatible {
1656 legacy_backfill: true,
1657 ..
1658 } => SchemaStartupOutcome::MetadataBackfilled,
1659 SchemaDecision::Initialize { .. } => SchemaStartupOutcome::Initialized,
1660 SchemaDecision::Migrate { .. } => SchemaStartupOutcome::Migrated,
1661 SchemaDecision::MigrationRequired { current, target } => {
1662 return Err(anyhow!(
1663 "schema migration required from V{current} to V{target}"
1664 ));
1665 }
1666 SchemaDecision::Reject(_) => return Err(anyhow!("schema compatibility rejected")),
1667 };
1668 if !matches!(outcome, SchemaStartupOutcome::Compatible) {
1669 runner
1670 .bootstrap_migration_coordination(permit.connection()?, &decision)
1671 .await?;
1672 leases.bootstrap(permit.connection()?).await?;
1673 let mut migration_guard = leases
1674 .acquire(
1675 permit.connection()?,
1676 &ControlLeaseAcquireRequest {
1677 claim_name: "schema-migration".to_owned(),
1678 cluster: control_lease_identity(cluster),
1679 owner_id: format!("schema-{}", IncarnationId::new()),
1680 lease_duration: SCHEMA_MIGRATION_LEASE_DURATION,
1681 admission_margin: SCHEMA_MIGRATION_ADMISSION_MARGIN,
1682 acquire_deadline: deadline,
1683 },
1684 )
1685 .await?;
1686 let migration_gate = OwnershipAdmissionGate::for_guard(&migration_guard);
1687 let apply_result = runner
1688 .apply_decision(
1689 permit.connection()?,
1690 &decision,
1691 &leases,
1692 &mut migration_guard,
1693 &migration_gate,
1694 )
1695 .await;
1696 let release_result = leases
1697 .release(permit.connection()?, &migration_guard, &migration_gate)
1698 .await;
1699 apply_result?;
1700 release_result?;
1701 }
1702 let final_decision = runner
1703 .assess_connection(
1704 permit.connection()?,
1705 &contract,
1706 SchemaMigrationPolicy::ValidateOnly,
1707 )
1708 .await?;
1709 let observed_version = match final_decision {
1710 SchemaDecision::Compatible { current, .. } => current,
1711 _ => return Err(anyhow!("schema did not converge to a readable version")),
1712 };
1713 Ok(SchemaStartupReport {
1714 observed_version,
1715 minimum_supported_version: contract.minimum_supported_version,
1716 target_version: contract.target_version,
1717 maximum_readable_version: contract.maximum_readable_version,
1718 migration_set_digest: contract.migration_set_digest,
1719 outcome,
1720 })
1721}
1722
1723async fn acquire_embedded_ownership(
1724 director: Arc<DsqlConnectionDirector>,
1725 cluster: &ResolvedCluster,
1726 deadline: Instant,
1727) -> Result<(EmbeddedOwnership, OwnershipStartupReport)> {
1728 let repository = ConnectionControlLeaseRepository::new();
1729 let mut permit = director.acquire(DbClass::Control).await?;
1730 repository.bootstrap(permit.connection()?).await?;
1731 let guard = repository
1732 .acquire(
1733 permit.connection()?,
1734 &ControlLeaseAcquireRequest {
1735 claim_name: "embedded-owner".to_owned(),
1736 cluster: control_lease_identity(cluster),
1737 owner_id: format!("engine-{}", IncarnationId::new()),
1738 lease_duration: OWNER_LEASE_DURATION,
1739 admission_margin: OWNER_ADMISSION_MARGIN,
1740 acquire_deadline: deadline,
1741 },
1742 )
1743 .await?;
1744 let gate = OwnershipAdmissionGate::for_guard(&guard);
1745 if let Some(quiescence_deadline) = guard.quiescence_deadline() {
1746 let now = Instant::now();
1747 if now >= deadline || quiescence_deadline > deadline {
1748 return Err(anyhow!("expired owner quiescence exceeds startup deadline"));
1749 }
1750 tokio::time::sleep_until(tokio::time::Instant::from_std(quiescence_deadline)).await;
1751 gate.finish_quiescence(&guard, Instant::now())?;
1752 }
1753 let report = OwnershipStartupReport {
1754 outcome: guard.outcome(),
1755 fence_token: guard.fence_token(),
1756 };
1757 Ok((
1758 EmbeddedOwnership {
1759 repository,
1760 guard: Arc::new(tokio::sync::Mutex::new(Some(guard))),
1761 gate,
1762 },
1763 report,
1764 ))
1765}
1766
1767fn spawn_ownership_renewal(
1768 tasks: &RuntimeShutdownHandle,
1769 director: Arc<DsqlConnectionDirector>,
1770 ownership: EmbeddedOwnership,
1771 service: InProcessGrpcService,
1772) {
1773 let cancel = tasks.cancellation_token();
1774 let _renewal = tasks.spawn(async move {
1775 let interval_duration =
1776 StdDuration::try_from(OWNER_RENEW_INTERVAL).expect("positive owner renewal interval");
1777 let mut interval = tokio::time::interval(interval_duration);
1778 interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
1779 interval.tick().await;
1780 loop {
1781 tokio::select! {
1782 _ = cancel.cancelled() => break,
1783 _ = interval.tick() => {
1784 let mut guard_slot = ownership.guard.lock().await;
1785 let Some(guard) = guard_slot.as_mut() else {
1786 break;
1787 };
1788 let result = match director.acquire(DbClass::Control).await {
1789 Ok(mut permit) => match permit.connection() {
1790 Ok(connection) => ownership.repository.renew(
1791 connection,
1792 guard,
1793 OWNER_LEASE_DURATION,
1794 OWNER_ADMISSION_MARGIN,
1795 &ownership.gate,
1796 )
1797 .await
1798 .map_err(anyhow::Error::from),
1799 Err(error) => Err(error),
1800 },
1801 Err(error) => Err(error),
1802 };
1803 if let Err(error) = result {
1804 if matches!(
1805 error.downcast_ref::<ControlLeaseError>(),
1806 Some(ControlLeaseError::Fenced)
1807 ) {
1808 ownership.gate.fence();
1809 } else {
1810 guard.enforce_admission_deadline(Instant::now(), &ownership.gate);
1811 }
1812 }
1813 if ownership.gate.state() != OwnershipAdmissionState::Open {
1814 service.begin_shutdown();
1815 break;
1816 }
1817 }
1818 }
1819 }
1820 });
1821}
1822
1823fn control_lease_identity(cluster: &ResolvedCluster) -> ControlLeaseClusterIdentity {
1824 ControlLeaseClusterIdentity {
1825 cluster_id: cluster.identity.cluster_id.clone(),
1826 cluster_arn: cluster.identity.cluster_arn.clone(),
1827 }
1828}
1829
1830async fn rollback_embedded_dsql(
1831 ownership: &EmbeddedOwnership,
1832 director: &Arc<DsqlConnectionDirector>,
1833 deadline: Instant,
1834) {
1835 ownership.gate.begin_closing();
1836 let now = Instant::now();
1837 if now < deadline {
1838 let _ = tokio::time::timeout(
1839 deadline.saturating_duration_since(now),
1840 release_embedded_ownership(ownership, director),
1841 )
1842 .await;
1843 }
1844 let _ = director.shutdown_with_deadline(deadline).await;
1845}
1846
1847async fn release_embedded_ownership(
1848 ownership: &EmbeddedOwnership,
1849 director: &Arc<DsqlConnectionDirector>,
1850) -> Result<()> {
1851 let mut permit = director.acquire_shutdown_control().await?;
1854 let mut guard = ownership.guard.lock().await;
1855 if let Some(guard) = guard.take() {
1856 ownership
1857 .repository
1858 .release(permit.connection()?, &guard, &ownership.gate)
1859 .await?;
1860 }
1861 Ok(())
1862}
1863
1864async fn startup_phase<T, E, F>(
1865 deadline: Instant,
1866 phase: EmbeddedStartupPhase,
1867 future: F,
1868) -> Result<T, EmbeddedEngineStartError>
1869where
1870 F: Future<Output = Result<T, E>>,
1871{
1872 let now = Instant::now();
1873 if now >= deadline {
1874 return Err(EmbeddedEngineStartError::DeadlineExceeded { phase });
1875 }
1876 match tokio::time::timeout(deadline.saturating_duration_since(now), future).await {
1877 Ok(Ok(value)) => Ok(value),
1878 Ok(Err(_)) => Err(EmbeddedEngineStartError::Phase { phase }),
1879 Err(_) => Err(EmbeddedEngineStartError::DeadlineExceeded { phase }),
1880 }
1881}
1882
1883async fn startup_infallible_phase<T, F>(
1884 deadline: Instant,
1885 phase: EmbeddedStartupPhase,
1886 future: F,
1887) -> Result<T, EmbeddedEngineStartError>
1888where
1889 F: Future<Output = T>,
1890{
1891 let now = Instant::now();
1892 if now >= deadline {
1893 return Err(EmbeddedEngineStartError::DeadlineExceeded { phase });
1894 }
1895 tokio::time::timeout(deadline.saturating_duration_since(now), future)
1896 .await
1897 .map_err(|_| EmbeddedEngineStartError::DeadlineExceeded { phase })
1898}
1899
1900async fn finish_snapshot_policy(mut policy: EngineSnapshotPolicy) -> Result<()> {
1907 if let Some(task) = policy.periodic_task.take() {
1908 task.await
1909 .context("engine snapshot task panicked during shutdown")?;
1910 }
1911 tokio::task::yield_now().await;
1916 persist_snapshot(&policy.store, &policy.location)
1917 .await
1918 .with_context(|| {
1919 format!(
1920 "failed to persist final engine snapshot to {}",
1921 policy.location.display()
1922 )
1923 })
1924}
1925
1926async fn restore_snapshot_store(config: &TokeiraConfig) -> Result<(InMemoryStore, bool)> {
1927 let Some(snapshot) = config.policy.snapshot.as_ref() else {
1928 return Ok((InMemoryStore::default(), false));
1929 };
1930 match fs::read(&snapshot.location).await {
1931 Ok(bytes) => {
1932 let store = InMemoryStore::from_snapshot(&bytes).with_context(|| {
1933 format!(
1934 "failed to restore engine snapshot from {}",
1935 snapshot.location.display()
1936 )
1937 })?;
1938 retire_snapshot_leases(&store).await.with_context(|| {
1939 format!(
1940 "failed to retire prior embedded-engine leases from {}",
1941 snapshot.location.display()
1942 )
1943 })?;
1944 info!(
1945 snapshot_path = %snapshot.location.display(),
1946 "restored engine snapshot"
1947 );
1948 Ok((store, true))
1949 }
1950 Err(error) if error.kind() == std::io::ErrorKind::NotFound => {
1951 Ok((InMemoryStore::default(), false))
1952 }
1953 Err(error) => Err(error).with_context(|| {
1954 format!(
1955 "failed to read engine snapshot from {}",
1956 snapshot.location.display()
1957 )
1958 }),
1959 }
1960}
1961
1962async fn retire_snapshot_leases(store: &InMemoryStore) -> Result<()> {
1963 for lease in store
1964 .list_bundle_leases()
1965 .await
1966 .context("failed to list engine snapshot leases")?
1967 {
1968 let Some(owner) = lease.owner_node_id else {
1969 continue;
1970 };
1971 match store
1972 .relinquish_bundle(lease.bundle_id, owner, lease.epoch)
1973 .await
1974 .context("failed to relinquish an engine snapshot lease")?
1975 {
1976 LeaseOutcome::Acquired { .. } => {}
1977 LeaseOutcome::Rejected {
1978 current_owner,
1979 current_epoch,
1980 } => {
1981 return Err(anyhow!(
1982 "engine snapshot lease retirement was fenced by owner {current_owner} at epoch {}",
1983 current_epoch.0
1984 ));
1985 }
1986 LeaseOutcome::Renewed { epoch } => {
1987 return Err(anyhow!(
1988 "engine snapshot lease retirement unexpectedly renewed epoch {}",
1989 epoch.0
1990 ));
1991 }
1992 }
1993 }
1994 Ok(())
1995}
1996
1997async fn persist_snapshot(store: &InMemoryStore, location: &Path) -> Result<()> {
1998 let bytes = store
1999 .snapshot()
2000 .await
2001 .context("failed to capture engine snapshot")?;
2002 write_snapshot_atomically(location, &bytes).await
2003}
2004
2005async fn write_snapshot_atomically(location: &Path, bytes: &[u8]) -> Result<()> {
2006 if let Some(parent) = location
2007 .parent()
2008 .filter(|parent| !parent.as_os_str().is_empty())
2009 {
2010 fs::create_dir_all(parent).await.with_context(|| {
2011 format!(
2012 "failed to create engine snapshot directory {}",
2013 parent.display()
2014 )
2015 })?;
2016 }
2017
2018 let sequence = SNAPSHOT_TEMP_SEQUENCE.fetch_add(1, Ordering::Relaxed);
2022 let mut temporary_name = location.as_os_str().to_os_string();
2023 temporary_name.push(format!(".tmp-{}-{sequence}", std::process::id()));
2024 let temporary = PathBuf::from(temporary_name);
2025
2026 let write_result: Result<()> = async {
2027 let mut file = OpenOptions::new()
2028 .create(true)
2029 .truncate(true)
2030 .write(true)
2031 .open(&temporary)
2032 .await
2033 .with_context(|| {
2034 format!(
2035 "failed to create temporary engine snapshot {}",
2036 temporary.display()
2037 )
2038 })?;
2039 file.write_all(bytes).await.with_context(|| {
2040 format!(
2041 "failed to write temporary engine snapshot {}",
2042 temporary.display()
2043 )
2044 })?;
2045 file.sync_all().await.with_context(|| {
2046 format!(
2047 "failed to sync temporary engine snapshot {}",
2048 temporary.display()
2049 )
2050 })?;
2051 drop(file);
2052 fs::rename(&temporary, location)
2053 .await
2054 .with_context(|| format!("failed to replace engine snapshot {}", location.display()))?;
2055 Ok(())
2056 }
2057 .await;
2058
2059 if write_result.is_err() {
2060 let _ = fs::remove_file(&temporary).await;
2063 }
2064 write_result
2065}
2066
2067fn embedded_config(mut config: TokeiraConfig) -> Result<TokeiraConfig> {
2068 config.infrastructure.storage = ConfigStorageKind::InMemory;
2069 config.infrastructure.placement.controller_endpoint = None;
2073 config
2074 .validate()
2075 .context("invalid embedded engine config")?;
2076 Ok(config)
2077}
2078
2079impl Drop for Engine {
2080 fn drop(&mut self) {
2081 if let Some(coordinator) = &self.shutdown_coordinator {
2082 coordinator.begin_shutdown();
2083 }
2084 self.background_cancel.cancel();
2085 }
2086}
2087
2088impl TemporalEndpoint {
2089 pub async fn call(
2091 &self,
2092 request: InProcessGrpcRequest,
2093 ) -> Result<InProcessGrpcResponse, tonic::Status> {
2094 tokio::select! {
2095 biased;
2096 _ = self.shutdown.cancelled() => {
2097 Err(tonic::Status::unavailable("embedded Tokeira engine is shut down"))
2098 }
2099 response = self.service.call(request) => response,
2100 }
2101 }
2102
2103 pub fn is_shutdown(&self) -> bool {
2105 self.shutdown.is_cancelled()
2106 }
2107
2108 #[cfg(feature = "temporalio-client")]
2110 pub fn service_override(&self) -> temporalio_client::callback_based::CallbackBasedGrpcService {
2111 let endpoint = self.clone();
2112 temporalio_client::callback_based::CallbackBasedGrpcService {
2113 callback: Arc::new(move |request| {
2114 let endpoint = endpoint.clone();
2115 Box::pin(async move {
2116 endpoint
2117 .call(InProcessGrpcRequest {
2118 service: request.service,
2119 rpc: request.rpc,
2120 headers: request.headers,
2121 proto: request.proto,
2122 })
2123 .await
2124 .map(
2128 |response| temporalio_client::callback_based::GrpcSuccessResponse {
2129 headers: response.headers,
2130 proto: response.proto,
2131 },
2132 )
2133 })
2134 }),
2135 }
2136 }
2137}
2138
2139const EMBEDDED_NEXUS_CALLBACK_BASE: &str = "http://tokeira-engine.invalid";
2140
2141struct InProcessNexusCompletionClient {
2144 runtime: RwLock<Option<Weak<dyn WorkflowRuntimeApi>>>,
2145 http: HttpNexusCompletionClient,
2146}
2147
2148impl std::fmt::Debug for InProcessNexusCompletionClient {
2149 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
2150 f.debug_struct("InProcessNexusCompletionClient")
2151 .finish_non_exhaustive()
2152 }
2153}
2154
2155impl InProcessNexusCompletionClient {
2156 fn new() -> Self {
2157 Self {
2158 runtime: RwLock::new(None),
2159 http: HttpNexusCompletionClient::new(),
2160 }
2161 }
2162
2163 fn attach(&self, runtime: &Arc<dyn WorkflowRuntimeApi>) -> Result<()> {
2164 let mut slot = self
2165 .runtime
2166 .write()
2167 .map_err(|_| anyhow!("embedded Nexus completion runtime lock poisoned"))?;
2168 *slot = Some(Arc::downgrade(runtime));
2169 Ok(())
2170 }
2171}
2172
2173#[async_trait]
2174impl NexusCompletionClient for InProcessNexusCompletionClient {
2175 async fn complete_operation(
2176 &self,
2177 url: &str,
2178 token: &str,
2179 operation_token: &str,
2180 completion: NexusCompletion,
2181 links: &[Link],
2182 ) -> Result<CompletionDeliveryOutcome> {
2183 if url != system_callback_post_url(EMBEDDED_NEXUS_CALLBACK_BASE) {
2184 return self
2185 .http
2186 .complete_operation(url, token, operation_token, completion, links)
2187 .await;
2188 }
2189
2190 let runtime = self
2191 .runtime
2192 .read()
2193 .ok()
2194 .and_then(|slot| slot.as_ref().and_then(Weak::upgrade));
2195 let Some(runtime) = runtime else {
2196 return Ok(CompletionDeliveryOutcome::RetryableError {
2197 detail: "embedded Nexus completion runtime is unavailable".to_owned(),
2198 });
2199 };
2200
2201 let state = completion.operation_state();
2202 let (body, content_type) = match completion {
2203 NexusCompletion::Succeeded(payloads) => nexus_payload_to_body(&payloads),
2204 NexusCompletion::Failed(body) | NexusCompletion::Canceled(body) => {
2205 (body, Some("application/json".to_owned()))
2206 }
2207 };
2208 let response = handle_nexus_callback(
2209 runtime.as_ref(),
2210 Some(token),
2211 Some(operation_token),
2212 Some(state),
2213 content_type.as_deref(),
2214 &body,
2215 )
2216 .await;
2217 let detail = response
2218 .body
2219 .map(|body| String::from_utf8_lossy(&body).into_owned())
2220 .unwrap_or_else(|| format!("Nexus callback returned status {}", response.status));
2221 Ok(match response.status {
2222 200..=299 => CompletionDeliveryOutcome::Delivered,
2223 400 | 401 | 403 | 404 | 409 | 501 => {
2224 CompletionDeliveryOutcome::NonRetryableError { detail }
2225 }
2226 _ => CompletionDeliveryOutcome::RetryableError { detail },
2227 })
2228 }
2229}
2230
2231#[derive(Debug)]
2244pub struct TokeiradHandle {
2245 bound_addr: SocketAddr,
2246 shutdown_tx: Option<oneshot::Sender<()>>,
2247 server_task: Option<JoinHandle<Result<()>>>,
2248 background_cancel: CancellationToken,
2254 log_broadcast: broadcast::Sender<LogEvent>,
2255 snapshot_policy: Option<EngineSnapshotPolicy>,
2259}
2260
2261#[derive(Clone, Debug)]
2268pub struct LogEvent {
2269 pub rpc: String,
2270 pub namespace: String,
2271 pub heartbeat_count: usize,
2272}
2273
2274impl TokeiradHandle {
2275 pub async fn start_in_memory(addr: SocketAddr) -> Result<Self> {
2282 Self::start_in_memory_with_config(addr, TokeiraConfig::default()).await
2283 }
2284
2285 pub async fn start_in_memory_with_config(
2292 addr: SocketAddr,
2293 mut config: TokeiraConfig,
2294 ) -> Result<Self> {
2295 config.infrastructure.storage = ConfigStorageKind::InMemory;
2296 config.policy.nexus_completion.http_addr = "127.0.0.1:0".to_string();
2300 config.policy.nexus_completion.system_callback_url = "http://127.0.0.1:0".to_string();
2301 config
2302 .validate()
2303 .context("invalid in-memory server config")?;
2304 let effective_config = Arc::new(config);
2305 let (
2306 (server_task, bound_addr, shutdown_tx, background_cancel, log_broadcast, _recorder),
2307 snapshot_policy,
2308 _node_claim,
2309 ) = build_and_serve(addr, effective_config).await?;
2310 Ok(Self {
2311 bound_addr,
2312 shutdown_tx: Some(shutdown_tx),
2313 server_task: Some(server_task),
2314 background_cancel,
2315 log_broadcast,
2316 snapshot_policy,
2317 })
2318 }
2319
2320 pub async fn start_in_memory_with_toml(addr: SocketAddr, config: &str) -> Result<Self> {
2326 let config = toml::from_str(config).context("parse in-memory server config")?;
2327 Self::start_in_memory_with_config(addr, config).await
2328 }
2329
2330 pub fn bound_addr(&self) -> SocketAddr {
2332 self.bound_addr
2333 }
2334
2335 pub fn log_sink(&self) -> broadcast::Receiver<LogEvent> {
2338 self.log_broadcast.subscribe()
2339 }
2340
2341 pub async fn shutdown(mut self) -> Result<()> {
2344 if let Some(tx) = self.shutdown_tx.take() {
2345 let _ = tx.send(());
2346 }
2347 self.background_cancel.cancel();
2348 if let Some(task) = self.server_task.take() {
2349 task.await
2350 .context("tokeirad server task panicked")?
2351 .context("tokeirad server task returned an error")?;
2352 }
2353 match self.snapshot_policy.take() {
2354 Some(policy) => finish_snapshot_policy(policy).await,
2355 None => Ok(()),
2356 }
2357 }
2358}
2359
2360impl Drop for TokeiradHandle {
2361 fn drop(&mut self) {
2362 if let Some(tx) = self.shutdown_tx.take() {
2363 let _ = tx.send(());
2364 }
2365 self.background_cancel.cancel();
2366 }
2369}
2370
2371fn dsql_auth_config(config: &TokeiraConfig) -> Result<DsqlAuthConfig> {
2372 let dsql = &config.infrastructure.dsql;
2373 let endpoint = dsql
2374 .endpoint
2375 .clone()
2376 .filter(|endpoint| !endpoint.trim().is_empty())
2377 .ok_or_else(|| {
2378 anyhow!("infrastructure.dsql.endpoint must be set when infrastructure.storage is dsql")
2379 })?;
2380 Ok(DsqlAuthConfig {
2381 endpoint,
2382 region: dsql.region.clone(),
2383 admin_role_arn: dsql.admin_role_arn.clone(),
2384 runtime_role_arn: dsql.runtime_role_arn.clone(),
2385 readonly_role_arn: dsql.readonly_role_arn.clone(),
2386 })
2387}
2388
2389fn dsql_pool_config_with_client(
2390 config: &TokeiraConfig,
2391 ddb_client: aws_sdk_dynamodb::Client,
2392) -> (DsqlPoolConfig, aws_sdk_dynamodb::Client) {
2393 let (rate_limiter_table, conn_lease_table) = dsql_coordination_table_names(config);
2394 let pool_config = DsqlPoolConfig {
2395 coordination: DsqlCoordinationConfig {
2396 rate_limiter_table,
2397 conn_lease_table,
2398 },
2399 shard_count: config.infrastructure.placement.shard_count,
2400 projection_partition_count: config.infrastructure.placement.partition_count,
2401 ..DsqlPoolConfig::default()
2402 };
2403 (pool_config, ddb_client)
2404}
2405
2406async fn dsql_pool_config(
2407 config: &TokeiraConfig,
2408 auth: &DsqlAuthConfig,
2409) -> Result<(DsqlPoolConfig, aws_sdk_dynamodb::Client)> {
2410 let region = auth
2411 .resolved_region()
2412 .ok_or_else(|| anyhow!("dsql region must be configured or derivable from endpoint"))?;
2413 let sdk_config = aws_config::defaults(aws_config::BehaviorVersion::latest())
2414 .region(aws_config::Region::new(region))
2415 .load()
2416 .await;
2417 Ok(dsql_pool_config_with_client(
2418 config,
2419 aws_sdk_dynamodb::Client::new(&sdk_config),
2420 ))
2421}
2422
2423fn dsql_coordination_table_names(config: &TokeiraConfig) -> (String, String) {
2424 let dsql = &config.infrastructure.dsql;
2425 let rate_limiter = dsql
2426 .rate_limiter_table
2427 .clone()
2428 .unwrap_or_else(|| format!("{}-dsql-rate-limiter", config.infrastructure.cluster_name));
2429 let conn_lease = dsql
2430 .conn_lease_table
2431 .clone()
2432 .unwrap_or_else(|| format!("{}-dsql-conn-lease", config.infrastructure.cluster_name));
2433 (rate_limiter, conn_lease)
2434}
2435
2436#[allow(clippy::print_stdout)]
2445pub async fn run_from_cli(cli: Cli) -> Result<()> {
2446 if let Some(command) = cli.command {
2447 return match command {
2448 CliCommand::PeerCredential {
2449 command: PeerCredentialCommand::Generate { out },
2450 } => {
2451 println!("{}", peer::generate(&out)?);
2452 Ok(())
2453 }
2454 CliCommand::Storage { command } => storage_maintenance::run(command).await,
2455 };
2456 }
2457 if cli.version {
2458 println!("{}", render_build_info(cli.verbose, cli.json));
2459 return Ok(());
2460 }
2461
2462 let (effective_config, config_source) = TokeiraConfig::resolve(cli.config.as_deref())?;
2463 if cli.dump_config {
2464 println!("# config source: {config_source}");
2466 println!("{}", effective_config.to_toml()?);
2467 return Ok(());
2468 }
2469 let observability = observability::ObservabilityConfig::from_tokeira_config(&effective_config)?;
2470 let effective_config = Arc::new(effective_config);
2471 let readiness = observability::TokeiradReadiness::new();
2472 let _observability_runtime = observability::install_process_observability(
2473 &observability,
2474 effective_config.clone(),
2475 readiness.registry.clone(),
2476 )
2477 .await?;
2478 log_build_info("tokeirad");
2479 for warning in effective_config.emergency_warnings() {
2480 tracing::warn!("{warning}");
2481 }
2482
2483 let addr: SocketAddr = effective_config
2484 .infrastructure
2485 .network
2486 .grpc_addr
2487 .parse()
2488 .with_context(|| {
2489 format!(
2490 "invalid infrastructure.network.grpc_addr value: {}",
2491 effective_config.infrastructure.network.grpc_addr
2492 )
2493 })?;
2494 info!(config_source = %config_source, "loaded tokeirad configuration");
2495
2496 let (
2497 (
2498 mut server_task,
2499 bound_addr,
2500 shutdown_tx,
2501 background_cancel,
2502 _log_broadcast,
2503 wire_recorder,
2504 ),
2505 snapshot_policy,
2506 node_claim,
2507 ) = build_and_serve(addr, effective_config).await?;
2508 readiness.mark_started();
2509 info!("tokeirad gRPC server listening on {bound_addr}");
2510
2511 let serve_result = {
2519 let drain_signal = async {
2520 #[cfg(unix)]
2521 {
2522 let mut sigterm =
2523 tokio::signal::unix::signal(tokio::signal::unix::SignalKind::terminate())
2524 .context("failed to install the SIGTERM handler")?;
2525 tokio::select! {
2526 result = tokio::signal::ctrl_c() => result
2527 .context("failed to listen for ctrl-c")
2528 .map(|()| "SIGINT"),
2529 _ = sigterm.recv() => Ok("SIGTERM"),
2530 }
2531 }
2532 #[cfg(not(unix))]
2533 {
2534 tokio::signal::ctrl_c()
2535 .await
2536 .context("failed to listen for ctrl-c")
2537 .map(|()| "ctrl-c")
2538 }
2539 };
2540 tokio::select! {
2541 result = &mut server_task => result
2542 .context("tokeirad server task panicked")?
2543 .context("tokeirad server task returned an error"),
2544 signal = drain_signal => {
2545 match signal {
2546 Ok(signal) => info!(signal, "shutdown signal received; draining tokeirad"),
2547 Err(error) => {
2548 tracing::warn!(?error, "signal handling failed; shutting down");
2551 }
2552 }
2553 let _ = shutdown_tx.send(());
2554 (&mut server_task)
2555 .await
2556 .context("tokeirad server task panicked")?
2557 .context("tokeirad server task returned an error")
2558 }
2559 }
2560 };
2561 background_cancel.cancel();
2562 let claim_result = match node_claim {
2567 Some(claim) => claim.shut_down().await,
2568 None => Ok(()),
2569 };
2570 let snapshot_result = match snapshot_policy {
2575 Some(policy) => finish_snapshot_policy(policy).await,
2576 None => Ok(()),
2577 };
2578
2579 if let Some(recorder) = wire_recorder {
2588 let out_path = wire_coverage_out_path();
2589 if let Err(error) = export_wire_coverage(recorder.as_ref(), &out_path) {
2590 tracing::warn!(
2591 path = %out_path.display(),
2592 ?error,
2593 "failed to export conformance wire-coverage evidence"
2594 );
2595 }
2596 }
2597
2598 serve_result?;
2599 claim_result?;
2600 snapshot_result?;
2601 Ok(())
2602}
2603
2604fn export_wire_coverage(recorder: &WireCoverageRecorder, path: &Path) -> Result<()> {
2615 let record = recorder.snapshot();
2616 let row_count = record.rows.len();
2617 let json = serde_json::to_string_pretty(&record)
2618 .context("failed to serialize wire-coverage record to JSON")?;
2619 std::fs::write(path, json).with_context(|| {
2620 format!(
2621 "failed to write wire-coverage evidence to {}",
2622 path.display()
2623 )
2624 })?;
2625 info!(
2626 path = %path.display(),
2627 rows = row_count,
2628 "wrote conformance wire-coverage evidence"
2629 );
2630 Ok(())
2631}
2632
2633fn log_build_info(process: &'static str) {
2634 let info = tokeira_build_info::summary();
2635 tracing::info!(
2636 process,
2637 server_version = tokeira_build_info::SERVER_VERSION,
2638 tokeira_version = info.tokeira_version,
2639 tokeira_git_sha = info.tokeira_git_sha,
2640 tokeira_source_revision = info.tokeira_source_revision,
2641 temporal_proto_version = info.temporal_proto_version,
2642 temporal_server_compat = info.temporal_server_compat,
2643 rust_toolchain = info.rust_toolchain,
2644 source_tree_hash = info.source_tree_hash,
2645 feature_matrix_digest = info.feature_matrix_digest,
2646 sdk_matrix_digest = info.sdk_matrix_digest,
2647 build_mode = info.build_mode,
2648 "tokeira build provenance"
2649 );
2650}
2651
2652fn render_build_info(verbose: bool, json: bool) -> String {
2653 let info = tokeira_build_info::summary();
2654 if json {
2655 return serde_json::json!({
2656 "server_version": tokeira_build_info::SERVER_VERSION,
2657 "tokeira_version": info.tokeira_version,
2658 "tokeira_git_sha": info.tokeira_git_sha,
2659 "source_revision": info.tokeira_source_revision,
2660 "temporal_proto_version": info.temporal_proto_version,
2661 "temporal_server_compat": info.temporal_server_compat,
2662 "rust_toolchain": info.rust_toolchain,
2663 "source_tree_hash": info.source_tree_hash,
2664 "feature_matrix_digest": info.feature_matrix_digest,
2665 "sdk_matrix_digest": info.sdk_matrix_digest,
2666 "build_mode": info.build_mode,
2667 })
2668 .to_string();
2669 }
2670
2671 if verbose {
2672 return [
2673 format!("server_version: {}", tokeira_build_info::SERVER_VERSION),
2674 format!("tokeira_version: {}", info.tokeira_version),
2675 format!("tokeira_git_sha: {}", info.tokeira_git_sha),
2676 format!("tokeira_source_revision: {}", info.tokeira_source_revision),
2677 format!("temporal_proto_version: {}", info.temporal_proto_version),
2678 format!("temporal_server_compat: {}", info.temporal_server_compat),
2679 format!("rust_toolchain: {}", info.rust_toolchain),
2680 format!("source_tree_hash: {}", info.source_tree_hash),
2681 format!("feature_matrix_digest: {}", info.feature_matrix_digest),
2682 format!("sdk_matrix_digest: {}", info.sdk_matrix_digest),
2683 format!("build_mode: {}", info.build_mode),
2684 ]
2685 .join("\n");
2686 }
2687
2688 format!(
2689 "tokeira {}\ngit {}\ntemporal_proto {}\ntemporal_server {}",
2690 tokeira_build_info::SERVER_VERSION,
2691 info.tokeira_git_sha,
2692 info.temporal_proto_version,
2693 info.temporal_server_compat
2694 )
2695}
2696
2697type ServerStack = (
2707 JoinHandle<Result<()>>,
2708 SocketAddr,
2709 oneshot::Sender<()>,
2710 CancellationToken,
2711 broadcast::Sender<LogEvent>,
2712 Option<Arc<WireCoverageRecorder>>,
2713);
2714
2715#[derive(Clone, Copy, Debug)]
2716enum StackTransport {
2717 Network(SocketAddr),
2718 Embedded,
2719}
2720
2721#[derive(Debug)]
2722enum ConstructedStack {
2723 Network(ServerStack),
2724 Embedded(EmbeddedStack),
2725}
2726
2727struct EmbeddedStack {
2728 component_visibility_store: chasm_extensions::VisibilityQueryStore,
2729 chasm_engine: Arc<tokeira_runtime::chasm::ChasmEngine>,
2730 registry: Arc<tokeira_chasm::Registry>,
2731 service: InProcessGrpcService,
2732 background_cancel: CancellationToken,
2733 log_broadcast: broadcast::Sender<LogEvent>,
2734 recovery_task: Option<JoinHandle<Result<()>>>,
2735 runtime_tasks: RuntimeShutdownHandle,
2736 engine_tasks: RuntimeShutdownHandle,
2737 shard_cleanup: Option<CleanupAction>,
2738}
2739
2740struct StackStartupGuard {
2741 runtime_tasks: RuntimeShutdownHandle,
2742 engine_tasks: RuntimeShutdownHandle,
2743 armed: bool,
2744}
2745
2746impl StackStartupGuard {
2747 fn new(runtime_tasks: RuntimeShutdownHandle, engine_tasks: RuntimeShutdownHandle) -> Self {
2748 Self {
2749 runtime_tasks,
2750 engine_tasks,
2751 armed: true,
2752 }
2753 }
2754
2755 fn disarm(&mut self) {
2756 self.armed = false;
2757 }
2758}
2759
2760impl Drop for StackStartupGuard {
2761 fn drop(&mut self) {
2762 if self.armed {
2763 self.engine_tasks.begin_shutdown();
2767 self.runtime_tasks.begin_shutdown();
2768 }
2769 }
2770}
2771
2772#[derive(Clone, Debug)]
2773struct SelfAssignedShardLease {
2774 shard_id: ShardId,
2775 owner: String,
2776 epoch: ShardEpoch,
2777}
2778
2779impl std::fmt::Debug for EmbeddedStack {
2780 fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
2781 formatter
2782 .debug_struct("EmbeddedStack")
2783 .field("service", &self.service)
2784 .field("background_cancel", &self.background_cancel)
2785 .field("runtime_tasks", &self.runtime_tasks)
2786 .field("engine_tasks", &self.engine_tasks)
2787 .field("has_shard_cleanup", &self.shard_cleanup.is_some())
2788 .finish_non_exhaustive()
2789 }
2790}
2791
2792const DEFAULT_MAX_ID_LENGTH: usize = 1000;
2796
2797fn refuse_distributed_migration_policy(config: &TokeiraConfig) -> Result<()> {
2806 match config.infrastructure.dsql.migration_policy {
2807 None | Some(DsqlMigrationPolicy::ValidateOnly) => Ok(()),
2808 Some(DsqlMigrationPolicy::Automatic) => Err(anyhow!(
2809 "infrastructure.dsql.migration_policy = \"automatic\" is not supported by \
2810 listener-backed distributed DSQL: a distributed tokeirad never migrates the \
2811 schema it shares with other nodes. Apply migrations with `tkr schema setup`, \
2812 then set the policy to \"validate_only\" or remove it"
2813 )),
2814 }
2815}
2816
2817async fn check_distributed_schema(dsql_store: &DsqlStore) -> Result<()> {
2828 let contract = MigrationRunner::compatibility_contract();
2829 let decision = {
2830 let mut permit = dsql_store
2831 .connection_director()
2832 .acquire(DbClass::Control)
2833 .await?;
2834 MigrationRunner::embedded()
2835 .assess_connection(
2836 permit.connection()?,
2837 &contract,
2838 SchemaMigrationPolicy::ValidateOnly,
2839 )
2840 .await
2841 .context("failed to read the DSQL schema's migration ledger")?
2842 };
2843 let current = distributed_schema_admission(decision)?;
2844 info!(
2845 schema_version = current,
2846 target_version = contract.target_version,
2847 maximum_readable_version = contract.maximum_readable_version,
2848 "DSQL schema check passed"
2849 );
2850 Ok(())
2851}
2852
2853fn distributed_schema_admission(decision: SchemaDecision) -> Result<u32> {
2858 match decision {
2859 SchemaDecision::Compatible { current, .. } => Ok(current),
2860 SchemaDecision::MigrationRequired { current: 0, target } => Err(anyhow!(
2861 "tokeirad refuses to start: the DSQL database has no Tokeira schema. Apply this \
2862 release's migrations, through V{target}, with `tkr schema setup`, then start \
2863 tokeirad"
2864 )),
2865 SchemaDecision::MigrationRequired { current, target } => Err(anyhow!(
2866 "tokeirad refuses to start: the DSQL schema is at V{current}, behind this \
2867 release's target V{target}. Apply the missing migrations with `tkr schema \
2868 setup`, then start tokeirad"
2869 )),
2870 SchemaDecision::Reject(incompatibility) => {
2871 let advice = match incompatibility.category {
2872 SchemaIncompatibilityCategory::FutureSchema => {
2873 "a later release migrated it; run that release or a later one"
2874 }
2875 SchemaIncompatibilityCategory::BelowMinimum => {
2876 "it is older than this release migrates from; upgrade it through an \
2877 earlier release first"
2878 }
2879 _ => {
2880 "its migration ledger doesn't match this release's migrations; find out \
2881 what changed it before any node runs on it"
2882 }
2883 };
2884 Err(
2885 anyhow::Error::new(SchemaCompatibilityError::Incompatible(incompatibility))
2886 .context(format!(
2887 "tokeirad refuses to start: this release can't run on the DSQL \
2888 schema, because {advice}"
2889 )),
2890 )
2891 }
2892 decision @ (SchemaDecision::Initialize { .. } | SchemaDecision::Migrate { .. }) => {
2895 Err(anyhow!(
2896 "tokeirad refuses to start: a validate-only schema assessment decided \
2897 {decision:?}"
2898 ))
2899 }
2900 }
2901}
2902
2903async fn build_and_serve(
2916 addr: SocketAddr,
2917 effective_config: Arc<TokeiraConfig>,
2918) -> Result<(
2919 ServerStack,
2920 Option<EngineSnapshotPolicy>,
2921 Option<node_claim::NodeClaim>,
2922)> {
2923 let no_controller = effective_config
2926 .infrastructure
2927 .placement
2928 .controller_endpoint
2929 .is_none();
2930 let peer = if no_controller {
2931 None
2932 } else {
2933 Some(peer::bootstrap(
2934 effective_config.infrastructure.peer.as_ref(),
2935 )?)
2936 };
2937 let mut snapshot_inputs = None;
2938 let mut node_claim = None;
2939 let stack = match effective_config.infrastructure.storage {
2940 ConfigStorageKind::InMemory => {
2941 let (store, restored) = restore_snapshot_store(&effective_config).await?;
2946 snapshot_inputs = effective_config
2947 .policy
2948 .snapshot
2949 .clone()
2950 .map(|config| (store.clone(), config));
2951 build_in_memory_stack(
2952 StackTransport::Network(addr),
2953 effective_config,
2954 store,
2955 restored,
2956 ChasmExtensions::default(),
2957 peer,
2958 )
2959 .await
2960 }
2961 ConfigStorageKind::Dsql => {
2962 if effective_config.policy.snapshot.is_some() {
2963 return Err(anyhow!(
2964 "policy.snapshot is configured but infrastructure.storage is \"dsql\": \
2965 snapshots are the in-memory store's persistence mechanism, and DSQL \
2966 storage is already durable — remove policy.snapshot or switch \
2967 infrastructure.storage to \"memory\""
2968 ));
2969 }
2970 refuse_distributed_migration_policy(&effective_config)?;
2971 let auth = dsql_auth_config(&effective_config)?;
2972 let endpoint = auth.endpoint.clone();
2973 let (pool_config, ddb_client) = dsql_pool_config(&effective_config, &auth).await?;
2974 let dsql_store = DsqlStore::connect(auth, pool_config, ddb_client)
2975 .await
2976 .context("failed to connect DSQL storage backend")?;
2977 if let Err(refusal) = check_distributed_schema(&dsql_store).await {
2982 if let Err(error) = dsql_store.shutdown().await {
2983 tracing::warn!(
2984 error = %format!("{error:#}"),
2985 "failed to shut DSQL storage down after refusing its schema"
2986 );
2987 }
2988 return Err(refusal);
2989 }
2990 if no_controller {
2995 match take_node_claim(&dsql_store).await {
2996 Ok(claim) => node_claim = Some(claim),
2997 Err(refusal) => {
2998 if let Err(error) = dsql_store.shutdown().await {
2999 tracing::warn!(
3000 error = %format!("{error:#}"),
3001 "failed to shut DSQL storage down after refusing to start"
3002 );
3003 }
3004 return Err(refusal);
3005 }
3006 }
3007 }
3008 build_dsql_stack(
3009 StackTransport::Network(addr),
3010 effective_config,
3011 dsql_store,
3012 endpoint,
3013 ChasmExtensions::default(),
3014 peer,
3015 )
3016 .await
3017 }
3018 };
3019 let stack = match stack {
3020 Ok(ConstructedStack::Network(stack)) => stack,
3021 Ok(ConstructedStack::Embedded(_)) => {
3022 release_node_claim_after_failed_start(node_claim).await;
3023 return Err(anyhow!(
3024 "network service construction returned an embedded stack"
3025 ));
3026 }
3027 Err(error) => {
3028 release_node_claim_after_failed_start(node_claim).await;
3029 return Err(error);
3030 }
3031 };
3032 if let Some(claim) = &node_claim {
3035 claim.drain_on_loss(stack.3.clone());
3036 }
3037 let snapshot_policy = snapshot_inputs
3041 .map(|(store, config)| EngineSnapshotPolicy::start(store, config, stack.3.clone()));
3042 Ok((stack, snapshot_policy, node_claim))
3043}
3044
3045async fn take_node_claim(dsql_store: &DsqlStore) -> Result<node_claim::NodeClaim> {
3051 let director = dsql_store.connection_director_arc();
3052 let acquired = node_claim::acquire(&director).await?;
3053 let gate = acquired.gate();
3054 let mut claim = acquired.start(director.clone());
3055 if let Err(error) = director.install_ownership_gate(gate) {
3056 release_node_claim_after_failed_start(Some(claim)).await;
3057 return Err(error.context("failed to install the single-node claim's admission gate"));
3058 }
3059 if let Err(error) = claim.wait_until_serving().await {
3060 release_node_claim_after_failed_start(Some(claim)).await;
3061 return Err(error);
3062 }
3063 Ok(claim)
3064}
3065
3066async fn release_node_claim_after_failed_start(claim: Option<node_claim::NodeClaim>) {
3070 if let Some(claim) = claim
3071 && let Err(error) = claim.shut_down().await
3072 {
3073 tracing::warn!(
3074 error = %format!("{error:#}"),
3075 "the single-node claim was lost while the node failed to start"
3076 );
3077 }
3078}
3079
3080async fn build_dsql_stack(
3081 transport: StackTransport,
3082 effective_config: Arc<TokeiraConfig>,
3083 dsql_store: DsqlStore,
3084 endpoint: String,
3085 extensions: ChasmExtensions,
3086 peer: Option<Arc<tokeira_edge::grpc::workflow_offer::PeerOfferEndpoint>>,
3087) -> Result<ConstructedStack> {
3088 let (director, run_repository, projection_log, worker_deployment_repository, _migration_runner) =
3089 dsql_store.into_parts();
3090 let budget_applier: Arc<dyn ConnectionBudgetApplier> = Arc::new(DsqlConnectionBudgetApplier {
3094 director: director.clone(),
3095 });
3096 let chasm_node_repo = Arc::new(tokeira_storage::dsql::DsqlChasmNodeRepository::new(
3097 director.clone(),
3098 ));
3099 let task_queue_config_repository: Arc<dyn TaskQueueConfigRepository> = Arc::new(
3100 tokeira_storage::dsql::DsqlTaskQueueConfigRepository::new(director.clone()),
3101 );
3102 let worker_compute_repository = effective_config.policy.worker_compute.enabled.then(|| {
3103 Arc::new(tokeira_storage::dsql::DsqlWorkerComputeRepository::new(
3104 director.clone(),
3105 )) as Arc<dyn WorkerComputeRepository>
3106 });
3107 let worker_task_provenance: Arc<dyn WorkerTaskProvenanceStore> = Arc::new(
3108 tokeira_storage::dsql::DsqlWorkerTaskProvenanceStore::new(director.clone()),
3109 );
3110 let visibility_store = DsqlVisibilityStore::new(director);
3111 visibility_store
3117 .copy_legacy_attributes()
3118 .await
3119 .context("failed to copy the visibility attribute index")?;
3120 let worker_deployment_repository: Arc<dyn WorkerDeploymentRepository> =
3121 Arc::new(worker_deployment_repository);
3122 build_service_stack_with_storage(
3123 transport,
3124 effective_config,
3125 Arc::new(run_repository),
3126 worker_deployment_repository,
3127 worker_compute_repository,
3128 task_queue_config_repository,
3129 worker_task_provenance,
3130 projection_log,
3131 visibility_store.clone(),
3132 {
3133 let visibility_store = visibility_store.clone();
3134 move || visibility_store.clone()
3135 },
3136 Some(endpoint),
3137 chasm_node_repo,
3138 false,
3139 budget_applier,
3140 extensions,
3141 peer,
3142 )
3143 .await
3144}
3145
3146async fn build_embedded(
3147 effective_config: Arc<TokeiraConfig>,
3148 store: InMemoryStore,
3149 recover_self_assigned_shard: bool,
3150 extensions: ChasmExtensions,
3151) -> Result<EmbeddedStack> {
3152 match build_in_memory_stack(
3153 StackTransport::Embedded,
3154 effective_config,
3155 store,
3156 recover_self_assigned_shard,
3157 extensions,
3158 None,
3159 )
3160 .await?
3161 {
3162 ConstructedStack::Embedded(stack) => Ok(stack),
3163 ConstructedStack::Network(_) => Err(anyhow!(
3164 "embedded service construction returned a network stack"
3165 )),
3166 }
3167}
3168
3169async fn build_in_memory_stack(
3170 transport: StackTransport,
3171 effective_config: Arc<TokeiraConfig>,
3172 store: InMemoryStore,
3173 recover_self_assigned_shard: bool,
3174 extensions: ChasmExtensions,
3175 peer: Option<Arc<tokeira_edge::grpc::workflow_offer::PeerOfferEndpoint>>,
3176) -> Result<ConstructedStack> {
3177 let visibility_store = InMemoryVisibilityStore::default();
3178 let chasm_node_repo: Arc<dyn tokeira_storage::ChasmNodeRepository> =
3179 Arc::new(tokeira_storage::InMemoryChasmNodeStore::new());
3180 #[cfg(test)]
3181 let (chasm_node_repo, visibility_store) = extensions
3182 .test_storage
3183 .clone()
3184 .unwrap_or((chasm_node_repo, visibility_store));
3185 let worker_deployment_repository: Arc<dyn WorkerDeploymentRepository> = Arc::new(store.clone());
3186 let worker_task_provenance: Arc<dyn WorkerTaskProvenanceStore> = Arc::new(store.clone());
3187 let worker_compute_repository = effective_config.policy.worker_compute.enabled.then(|| {
3188 Arc::new(InMemoryWorkerComputeRepository::default()) as Arc<dyn WorkerComputeRepository>
3189 });
3190 build_service_stack_with_storage(
3191 transport,
3192 effective_config,
3193 Arc::new(store.clone()),
3194 worker_deployment_repository,
3195 worker_compute_repository,
3196 Arc::new(store.clone()),
3197 worker_task_provenance,
3198 store,
3199 visibility_store.clone(),
3200 {
3201 let visibility_store = visibility_store.clone();
3202 move || VisibilitySink::new(visibility_store.clone())
3203 },
3204 None,
3205 chasm_node_repo,
3206 recover_self_assigned_shard,
3207 Arc::new(NoopConnectionBudgetApplier),
3209 extensions,
3210 peer,
3211 )
3212 .await
3213}
3214
3215const VISIBILITY_REPAIR_INTERVAL: std::time::Duration = std::time::Duration::from_secs(30);
3218
3219const CHASM_REBUILD_INTERVAL: std::time::Duration = std::time::Duration::from_secs(30);
3221
3222const WORKER_TASK_PROVENANCE_CLEANUP_INTERVAL: std::time::Duration =
3226 std::time::Duration::from_secs(30);
3227const WORKER_TASK_PROVENANCE_CLEANUP_BATCH: usize = 256;
3228
3229fn spawn_worker_task_provenance_cleanup(
3230 tasks: &RuntimeShutdownHandle,
3231 store: Arc<dyn WorkerTaskProvenanceStore>,
3232 cancel: CancellationToken,
3233) -> JoinHandle<()> {
3234 tasks.spawn(async move {
3235 let mut interval = tokio::time::interval(WORKER_TASK_PROVENANCE_CLEANUP_INTERVAL);
3236 interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
3237 loop {
3238 tokio::select! {
3239 _ = cancel.cancelled() => break,
3240 _ = interval.tick() => {
3241 if let Err(error) = store
3242 .delete_expired(
3243 time::OffsetDateTime::now_utc(),
3244 WORKER_TASK_PROVENANCE_CLEANUP_BATCH,
3245 )
3246 .await
3247 {
3248 tracing::warn!(
3249 error_kind = worker_task_provenance_cleanup_error_kind(&error),
3250 "Worker task authorization-evidence cleanup pass failed"
3251 );
3252 }
3253 }
3254 }
3255 }
3256 })
3257}
3258
3259fn worker_task_provenance_cleanup_error_kind(
3260 error: &tokeira_storage::WorkerTaskProvenanceError,
3261) -> &'static str {
3262 match error {
3263 tokeira_storage::WorkerTaskProvenanceError::Unavailable { .. } => "unavailable",
3264 tokeira_storage::WorkerTaskProvenanceError::DigestConflict => "conflict",
3265 tokeira_storage::WorkerTaskProvenanceError::Corrupt { .. } => "corrupt",
3266 }
3267}
3268
3269fn spawn_visibility_repair(
3275 tasks: &RuntimeShutdownHandle,
3276 nodes: Arc<dyn tokeira_storage::ChasmNodeRepository>,
3277 sink: Arc<dyn tokeira_projection::ProjectionSink>,
3278 registry: Arc<tokeira_chasm::Registry>,
3279 partition_count: u32,
3280 cancel: CancellationToken,
3281) {
3282 let rebuild: tokeira_runtime::chasm::SnapshotRebuilder =
3283 Arc::new(move |archetype_id, bytes| {
3284 match registry.visibility_snapshot(archetype_id, bytes) {
3285 Ok(snapshot) => snapshot,
3286 Err(error) => {
3287 tracing::warn!(
3288 ?error,
3289 archetype_id,
3290 "CHASM visibility rebuild decode failed"
3291 );
3292 None
3293 }
3294 }
3295 });
3296 let scanner =
3297 tokeira_runtime::chasm::VisibilityRepairScanner::new(nodes, sink, rebuild, partition_count);
3298 let _repair = tasks.spawn(async move {
3299 let mut interval = tokio::time::interval(VISIBILITY_REPAIR_INTERVAL);
3300 interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
3301 loop {
3302 tokio::select! {
3303 _ = cancel.cancelled() => break,
3304 _ = interval.tick() => {
3305 if let Err(error) = scanner.repair_once().await {
3306 tracing::warn!(?error, "visibility repair pass failed");
3307 }
3308 }
3309 }
3310 }
3311 });
3312}
3313
3314fn spawn_outbox_rebuild(
3317 tasks: &RuntimeShutdownHandle,
3318 scanner: tokeira_runtime::chasm::OutboxRebuildScanner,
3319 cancel: CancellationToken,
3320) {
3321 let _rebuild = tasks.spawn(async move {
3322 let mut interval = tokio::time::interval(CHASM_REBUILD_INTERVAL);
3323 interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
3324 interval.tick().await;
3326 loop {
3327 tokio::select! {
3328 _ = cancel.cancelled() => break,
3329 _ = interval.tick() => {
3330 tokio::select! {
3331 _ = cancel.cancelled() => break,
3332 result = scanner.rebuild_once() => {
3333 match result {
3334 Ok(stats) if stats.unserviceable > 0 => tracing::error!(unserviceable = stats.unserviceable, first = ?stats.first_unserviceable, "CHASM rebuild isolated executions; healthy executions were rebuilt"),
3335 Ok(_) => {},
3336 Err(error) => tracing::warn!(?error, "CHASM outbox rebuild pass failed"),
3337 }
3338 }
3339 }
3340 }
3341 }
3342 }
3343 });
3344}
3345
3346const CHASM_TIMER_SWEEP_INTERVAL: std::time::Duration = std::time::Duration::from_millis(200);
3351
3352fn spawn_chasm_timer_sweeper(
3361 tasks: &RuntimeShutdownHandle,
3362 engine: Arc<tokeira_runtime::chasm::ChasmEngine>,
3363 evaluator: Arc<dyn tokeira_runtime::chasm::TimeoutEvaluator>,
3364 archetype_id: u32,
3365 cancel: CancellationToken,
3366) {
3367 let sweeper = tokeira_runtime::chasm::ChasmTimerSweeper::new(engine)
3368 .with_evaluator(archetype_id, evaluator);
3369 let _sweeper = tasks.spawn(async move {
3370 let mut interval = tokio::time::interval(CHASM_TIMER_SWEEP_INTERVAL);
3371 interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
3372 loop {
3373 tokio::select! {
3374 _ = cancel.cancelled() => break,
3375 _ = interval.tick() => {
3376 sweeper.sweep_once().await;
3377 }
3378 }
3379 }
3380 });
3381}
3382
3383async fn build_service_stack_with_storage<R, L, S, V, F>(
3384 transport: StackTransport,
3385 effective_config: Arc<TokeiraConfig>,
3386 run_repository: Arc<R>,
3387 worker_deployment_repository: Arc<dyn WorkerDeploymentRepository>,
3388 worker_compute_repository: Option<Arc<dyn WorkerComputeRepository>>,
3389 task_queue_config_repository: Arc<dyn TaskQueueConfigRepository>,
3390 worker_task_provenance: Arc<dyn WorkerTaskProvenanceStore>,
3391 projection_log: L,
3392 visibility_query_store: V,
3393 projection_sink: F,
3394 dsql_endpoint: Option<String>,
3395 chasm_node_repo: Arc<dyn tokeira_storage::ChasmNodeRepository>,
3396 recover_self_assigned_shard: bool,
3397 budget_applier: Arc<dyn ConnectionBudgetApplier>,
3398 extensions: ChasmExtensions,
3399 peer: Option<Arc<tokeira_edge::grpc::workflow_offer::PeerOfferEndpoint>>,
3400) -> Result<ConstructedStack>
3401where
3402 R: LeaseRepository + RunRepository + 'static,
3403 L: ProjectionLog + Clone + 'static,
3404 S: ProjectionSink + VisibilityStore + 'static,
3405 V: VisibilityStore + Clone + 'static,
3406 F: Fn() -> S + Clone + Send + Sync + 'static,
3407{
3408 tokeira_storage::prepare_execution_placement(run_repository.as_ref()).await?;
3411 let has_extensions = !extensions.libraries.is_empty();
3415 let mut registry_builder = tokeira_chasm::Registry::builder();
3416 tokeira_chasm_activity::ActivityLibrary::register(&mut registry_builder)
3417 .map_err(registry_start_error)?;
3418 registry_builder.seal_built_ins();
3419 for library in extensions.libraries {
3420 library(&mut registry_builder).map_err(registry_start_error)?;
3421 }
3422 let registry = Arc::new(registry_builder.build());
3423 let declared = declared_search_attributes(®istry);
3424 let node_id = IncarnationId::new();
3427 let advertised_addr = match transport {
3428 StackTransport::Network(addr) => addr,
3429 StackTransport::Embedded => SocketAddr::from(([127, 0, 0, 1], 0)),
3430 };
3431 let node_endpoint = configured_node_endpoint(&effective_config, advertised_addr);
3432 let history_waits = HistoryWaitRegistry::default();
3433 let repo = Arc::new(HistoryNotifyingRepository::new(
3434 run_repository,
3435 history_waits.clone(),
3436 ));
3437 let default_namespace = ResolvedNamespace::active("default");
3438 let default_namespace_id = namespace_id_for("default");
3439
3440 let namespaces = Arc::new(InMemoryNamespaceCache::new());
3442 namespaces.insert(default_namespace).await?;
3443 let nexus_store = build_nexus_endpoint_store(
3447 namespaces.as_ref(),
3448 HashMap::<String, BootstrapNexusEndpointConfig>::new(),
3449 )
3450 .await?;
3451 let nexus_registry = NexusEndpointRegistry::new(nexus_store.clone());
3452 let nexus_http_client: Arc<dyn tokeira_runtime::NexusHttpClient> =
3453 Arc::new(HttpNexusClient::new());
3454
3455 let schedule_store = Arc::new(ScheduleStore::default());
3458 let task_queue_config_store = Arc::new(RepositoryBackedTaskQueueConfigStore::new(
3459 task_queue_config_repository,
3460 ));
3461 task_queue_config_store
3464 .hydrate()
3465 .await
3466 .context("failed to hydrate task-queue policy")?;
3467 let initial_routing_cache = if let Some(endpoint) = &effective_config
3471 .infrastructure
3472 .placement
3473 .controller_endpoint
3474 {
3475 let cache = Arc::new(RoutingCache::new(placement_config(&effective_config)));
3476 tokio::time::timeout(
3477 std::time::Duration::from_secs(10),
3478 tokeira_edge::subscribe_routing_once(cache.clone(), endpoint.clone()),
3479 )
3480 .await
3481 .context("timed out waiting for controller placement snapshot")??;
3482 Some(cache)
3483 } else {
3484 None
3485 };
3486 let mut runtime_config = RuntimeConfig::default();
3487 runtime_config.lane.controller_managed_placement = effective_config
3488 .infrastructure
3489 .placement
3490 .controller_endpoint
3491 .is_some();
3492 let seed_default_shard = !recover_self_assigned_shard
3493 && dsql_endpoint.is_none()
3494 && effective_config
3495 .infrastructure
3496 .placement
3497 .controller_endpoint
3498 .is_none();
3499
3500 let nexus_completion_cfg = effective_config.policy.nexus_completion.clone();
3501 let (
3502 nexus_callback_listener,
3503 in_process_nexus_client,
3504 nexus_completion_client,
3505 resolved_system_callback_url,
3506 ): (
3507 Option<TcpListener>,
3508 Option<Arc<InProcessNexusCompletionClient>>,
3509 Arc<dyn NexusCompletionClient>,
3510 String,
3511 ) = match transport {
3512 StackTransport::Network(_) => {
3513 let listener = TcpListener::bind(&nexus_completion_cfg.http_addr)
3516 .await
3517 .with_context(|| {
3518 format!(
3519 "failed to bind nexus completion listener on {}",
3520 nexus_completion_cfg.http_addr
3521 )
3522 })?;
3523 let callback_addr = listener
3524 .local_addr()
3525 .context("failed to resolve bound nexus completion listener address")?;
3526 let callback_url = with_loopback_port(
3527 &nexus_completion_cfg.system_callback_url,
3528 callback_addr.port(),
3529 );
3530 info!(
3531 bind = %callback_addr,
3532 loopback = %callback_url,
3533 "nexus completion callback listener bound"
3534 );
3535 (
3536 Some(listener),
3537 None,
3538 Arc::new(HttpNexusCompletionClient::new()),
3539 callback_url,
3540 )
3541 }
3542 StackTransport::Embedded => {
3543 let client = Arc::new(InProcessNexusCompletionClient::new());
3546 (
3547 None,
3548 Some(client.clone()),
3549 client,
3550 EMBEDDED_NEXUS_CALLBACK_BASE.to_owned(),
3551 )
3552 }
3553 };
3554 let nexus_completion_deps = NexusCompletionDeps {
3555 client: nexus_completion_client,
3556 config: NexusCompletionRuntimeConfig {
3557 system_callback_url: resolved_system_callback_url,
3558 retry_initial_interval: time::Duration::milliseconds(
3559 nexus_completion_cfg.retry_initial_interval_ms as i64,
3560 ),
3561 retry_max_interval: time::Duration::milliseconds(
3562 nexus_completion_cfg.retry_max_interval_ms as i64,
3563 ),
3564 retry_backoff_coefficient: nexus_completion_cfg.retry_backoff_coefficient,
3565 retry_max_attempts: nexus_completion_cfg.retry_max_attempts,
3566 },
3567 scanner: CompletionCallbackScannerConfig::default(),
3568 };
3569
3570 let chasm_nexus_client = nexus_completion_deps.client.clone();
3571 let chasm_nexus_config = nexus_completion_deps.config.clone();
3572
3573 let worker_compute_deployment_repository = worker_deployment_repository.clone();
3574 let runtime = TokeiraRuntime::new_with_nexus_and_shards_and_endpoint(
3575 repo.clone(),
3576 runtime_config.lane_count,
3577 runtime_config.lane,
3578 runtime_config.timer_scanner,
3579 runtime_config.workflow_timeout_scanner,
3580 runtime_config.activity_timeout_scanner,
3581 runtime_config.nexus_timeout_scanner,
3582 nexus_registry.clone(),
3583 nexus_http_client.clone(),
3584 nexus_completion_deps,
3587 effective_config.infrastructure.placement.shard_count,
3588 node_id.to_string(),
3589 node_endpoint.as_authority(),
3590 seed_default_shard,
3591 Some(Arc::new(CacheNexusNamespaceResolver {
3595 namespaces: namespaces.clone(),
3596 }) as Arc<dyn NexusNamespaceResolver>),
3597 )
3598 .with_queue_placement(
3602 effective_config.infrastructure.placement.partition_count,
3603 effective_config.infrastructure.placement.bundle_count,
3604 )
3605 .with_worker_deployment_repository(worker_deployment_repository)
3606 .with_task_queue_config_store(task_queue_config_store.clone())
3607 .with_delivery_mode_provider(Arc::new(
3608 tokeira_runtime::ConfiguredDeliveryModeProvider::new(
3609 tokeira_runtime::StaticDeliveryPolicy {
3610 enable_fairness: effective_config.policy.task_queues.enable_fairness,
3611 },
3612 ),
3613 ));
3614 runtime.ready().await?;
3615 let runtime = Arc::new(runtime);
3616 let runtime_tasks = runtime.shutdown_handle();
3617 let engine_tasks = RuntimeShutdownHandle::new();
3618 let mut startup_guard = StackStartupGuard::new(runtime_tasks.clone(), engine_tasks.clone());
3619 let background_cancel = engine_tasks.cancellation_token();
3620
3621 let recovered_lease = if recover_self_assigned_shard {
3622 let shard_id = ShardId(0);
3623 let epoch = runtime
3624 .acquire_shard(shard_id)
3625 .await
3626 .context("failed to recover embedded snapshot shard")?;
3627 Some((shard_id, node_id.to_string(), epoch))
3628 } else {
3629 None
3630 };
3631
3632 if matches!(transport, StackTransport::Network(_))
3633 && dsql_endpoint.is_some()
3634 && effective_config
3635 .infrastructure
3636 .placement
3637 .controller_endpoint
3638 .is_none()
3639 {
3640 let _leases = self_assign_dsql_shards(
3641 runtime.as_ref(),
3642 repo.as_ref(),
3643 effective_config.infrastructure.placement.shard_count,
3644 &node_id,
3645 &node_endpoint,
3646 false,
3647 )
3648 .await?;
3649 }
3650
3651 let recovery_task = recovered_lease.map(|(shard_id, owner, epoch)| {
3652 let runtime = runtime.clone();
3653 let repo = repo.clone();
3654 let cancel = background_cancel.clone();
3655 tokio::spawn(async move {
3656 cancel.cancelled().await;
3657 runtime.relinquish_shard(shard_id).await;
3662 match repo
3663 .relinquish_bundle(shard_id, owner, epoch)
3664 .await
3665 .context("failed to release embedded snapshot recovery lease")?
3666 {
3667 LeaseOutcome::Acquired { .. } => Ok(()),
3668 LeaseOutcome::Rejected {
3669 current_owner,
3670 current_epoch,
3671 } => Err(anyhow!(
3672 "embedded snapshot recovery lease was fenced by owner {current_owner} at epoch {}",
3673 current_epoch.0
3674 )),
3675 LeaseOutcome::Renewed { epoch } => Err(anyhow!(
3676 "embedded snapshot recovery lease unexpectedly renewed at epoch {}",
3677 epoch.0
3678 )),
3679 }
3680 })
3681 });
3682 let task_queue_config_refresh = task_queue_config_store
3683 .clone()
3684 .spawn_refresh(background_cancel.clone());
3685 let _task_queue_config_refresh = engine_tasks.track_join(task_queue_config_refresh);
3686 let _worker_task_provenance_cleanup = spawn_worker_task_provenance_cleanup(
3687 &engine_tasks,
3688 worker_task_provenance.clone(),
3689 background_cancel.clone(),
3690 );
3691
3692 let _worker_compute_service = if effective_config.policy.worker_compute.enabled {
3693 let controller_repository = worker_compute_repository
3694 .clone()
3695 .context("worker-compute enabled without a controller repository")?;
3696 let (observation_sender, observation_receiver) =
3697 mpsc::channel(OBSERVATION_CHANNEL_CAPACITY);
3698 let observation_sink = Arc::new(ChannelDemandObservationSink::new(observation_sender));
3699 runtime
3700 .broker()
3701 .set_demand_observation_sink(observation_sink.clone());
3702 runtime
3703 .activity_broker()
3704 .set_demand_observation_sink(observation_sink.clone());
3705 runtime
3706 .nexus_task_broker()
3707 .set_demand_observation_sink(observation_sink);
3708
3709 let (reconcile_sender, reconcile_receiver) = mpsc::channel(RECONCILE_CHANNEL_CAPACITY);
3710 runtime
3711 .deployment_registry()
3712 .expect("Worker Deployment repository was installed above")
3713 .set_worker_compute_reconcile_sink(Arc::new(ChannelWorkerComputeReconcileSink::new(
3714 reconcile_sender,
3715 )));
3716
3717 let clock = Arc::new(SystemWorkerComputeClock);
3718 let catalog = Arc::new(WorkerComputeNamespaceCatalogAdapter::new(
3719 namespaces.clone(),
3720 ));
3721 let reconciler = WorkerComputeReconciler::new(
3722 worker_compute_deployment_repository,
3723 controller_repository.clone(),
3724 node_id,
3725 );
3726 let sampler = runtime.worker_compute_queue_sampler(controller_repository.clone(), node_id);
3727 let provider = Arc::new(NexusWorkerComputeProvider::new(
3728 nexus_registry,
3729 nexus_http_client,
3730 runtime.nexus_task_broker(),
3731 ));
3732 let outbox = WorkerComputeOutbox::new(
3733 controller_repository.clone(),
3734 provider,
3735 clock.clone(),
3736 node_id,
3737 );
3738 let shard_runtime = runtime.clone();
3739 let active_shards: tokeira_runtime::WorkerComputeActiveShards =
3740 Arc::new(move || shard_runtime.active_shards());
3741 let service = WorkerComputeControllerService::new(
3742 catalog,
3743 controller_repository,
3744 reconciler,
3745 sampler,
3746 outbox,
3747 clock,
3748 active_shards,
3749 observation_receiver,
3750 reconcile_receiver,
3751 );
3752 let service_cancel = background_cancel.clone();
3753 Some(engine_tasks.spawn(async move {
3754 if let Err(error) = service.run(service_cancel).await {
3755 tracing::warn!(?error, "worker-compute controller service exited");
3756 }
3757 }))
3758 } else {
3759 None
3760 };
3761
3762 let membership_client = effective_config
3763 .infrastructure
3764 .placement
3765 .controller_endpoint
3766 .clone()
3767 .map(|controller_endpoint| {
3768 runtime.spawn_membership_client(
3769 membership_config(
3770 &effective_config,
3771 node_id,
3772 node_endpoint.clone(),
3773 controller_endpoint,
3774 ),
3775 budget_applier,
3776 background_cancel.clone(),
3777 )
3778 });
3779 if let Some(membership_client) = membership_client {
3780 let _membership_client = engine_tasks.track_join(membership_client);
3781 }
3782 let schedule_engine = run_schedule_engine(
3783 schedule_store.clone(),
3784 runtime.clone(),
3785 ScheduleEngineConfig::default(),
3786 background_cancel.clone(),
3787 );
3788 let _schedule_engine = engine_tasks.track_join(schedule_engine);
3789
3790 let authorization = build_authorization_stack(&effective_config).await?;
3791 let interceptors = Arc::new(EdgeInterceptors::configured(
3792 namespaces.clone(),
3793 authorization.grpc,
3794 authorization.principal_attribution,
3795 ));
3796 let router: Arc<dyn tokeira_edge::EdgeRouter> = if let Some(controller_endpoint) =
3797 effective_config
3798 .infrastructure
3799 .placement
3800 .controller_endpoint
3801 .clone()
3802 {
3803 let routing_cache = initial_routing_cache
3804 .expect("controller snapshot validated before runtime construction");
3805 let routing_config = EdgeRoutingConfig {
3806 controller_endpoint,
3807 max_retries: effective_config
3808 .infrastructure
3809 .placement
3810 .routing_max_retries,
3811 };
3812 let subscription_cache = routing_cache.clone();
3813 let subscription_cancel = background_cancel.clone();
3814 let routing_shutdown = runtime.shutdown_handle();
3815 let routing_cancel = background_cancel.clone();
3816 let _routing_subscription = engine_tasks.spawn(async move {
3817 if let Err(error) =
3818 run_routing_subscription(subscription_cache, routing_config, subscription_cancel)
3819 .await
3820 {
3821 tracing::error!(
3822 ?error,
3823 "routing placement became incompatible; stopping node"
3824 );
3825 routing_shutdown.begin_shutdown();
3826 routing_cancel.cancel();
3827 }
3828 });
3829 runtime.set_workflow_task_starter(Arc::new(
3830 tokeira_edge::grpc::workflow_dispatch::RemoteWorkflowTaskStarter::new(
3831 routing_cache.clone(),
3832 ),
3833 ));
3834 if let Some(peer) = &peer {
3835 runtime.set_activity_task_starter(Arc::new(
3836 tokeira_edge::grpc::activity_dispatch::RemoteActivityTaskStarter::new(
3837 routing_cache.clone(),
3838 peer.clone(),
3839 ),
3840 ));
3841 runtime.set_outbound_delivery_transport(Arc::new(
3842 tokeira_edge::grpc::outbound_delivery::RemoteOutboundDelivery::new(
3843 routing_cache.clone(),
3844 peer.clone(),
3845 ),
3846 ));
3847 }
3848 Arc::new(CacheBackedRouter::new(routing_cache, node_id))
3849 } else {
3850 Arc::new(LocalOnlyRouter)
3851 };
3852 let workflow_broker = runtime.broker();
3853 let buffered_queries = runtime.buffered_queries();
3854 let worker_registry = runtime.worker_registry();
3855 let nexus_task_broker = runtime.nexus_task_broker();
3856 let runtime_adapter = Arc::new(RuntimeAdapter::new(runtime.clone()));
3857 let callback_runtime = runtime_adapter.clone() as Arc<dyn WorkflowRuntimeApi>;
3858 if let Some(client) = in_process_nexus_client {
3859 client.attach(&callback_runtime)?;
3860 }
3861 if let Some(listener) = nexus_callback_listener {
3862 spawn_nexus_callback_server(listener, callback_runtime, background_cancel.clone());
3865 }
3866 let resolver = Arc::new(StoreExecutionResolver::new(repo.clone()));
3867 tokeira_projection::seed_predefined_search_attributes(
3868 &visibility_query_store,
3869 default_namespace_id,
3870 )
3871 .await
3872 .context("failed to seed Temporal predefined search attributes")?;
3873 let operator_visibility_store = visibility_query_store.clone();
3874 let visibility = Arc::new(VisibilityQueryService::new(visibility_query_store.clone()));
3875 let long_polls = LongPollGate::new(LongPollConfig::default());
3876 let operator_api = Arc::new(
3877 VisibilityRegistryOperatorApi::new(
3878 InMemoryOperatorApi::new("tokeira-local", tokeira_build_info::SERVER_VERSION),
3879 operator_visibility_store,
3880 )
3881 .with_declared_search_attributes(declared.clone()),
3882 );
3883
3884 let projectors = ProjectorSupervisor {
3888 log: projection_log.clone(),
3889 sink: projection_sink.clone(),
3890 partitions: effective_config.infrastructure.placement.partition_count,
3891 fanout: 1,
3892 page_size: tokeira_projection::PAGE_SIZE,
3893 homes: {
3894 let runtime = runtime.clone();
3895 move |partition| {
3896 runtime.partition_home(partition).map(|home| ProjectorHome {
3897 generation: home.generation,
3898 cancel: home.cancel,
3899 })
3900 }
3901 },
3902 };
3903 let _projectors = engine_tasks.spawn(projectors.run(background_cancel.clone()));
3904
3905 let nexus_http_waiters = tokeira_edge::nexus_http::NexusHttpWaiterRegistry::default();
3906 let workflow_service =
3907 WorkflowService::new_with_stores_and_buffered_queries_and_history_wait_registry(
3908 runtime_adapter.clone(),
3909 resolver,
3910 visibility,
3911 repo.clone(),
3912 operator_api.clone(),
3913 namespaces.clone(),
3914 interceptors.clone(),
3915 PollerRegistry::default(),
3916 PendingQueryStore::default(),
3917 buffered_queries,
3918 workflow_broker,
3919 nexus_task_broker.clone(),
3920 long_polls,
3921 router,
3922 history_waits,
3923 worker_registry,
3924 runtime.heartbeat_store(),
3925 schedule_store,
3926 task_queue_config_store,
3927 Arc::new(tokeira_runtime::BatchOperationStore::default()),
3928 )
3929 .with_nexus_http_waiters(nexus_http_waiters.clone())
3930 .with_worker_task_provenance(worker_task_provenance)
3931 .with_scoped_worker_sessions(ScopedWorkerSessionRegistry::default())
3932 .with_worker_deployment_runtime(runtime_adapter);
3933 let nexus_endpoint_limits = {
3938 let cfg = &effective_config.policy.nexus_endpoint_limits;
3939 tokeira_edge::nexus_endpoint::NexusEndpointLimits {
3940 name_max_length: cfg.name_max_length,
3941 external_url_max_length: cfg.external_url_max_length,
3942 description_max_size: cfg.description_max_size,
3943 task_queue_max_length: cfg.task_queue_max_length,
3944 list_default_page_size: cfg.list_default_page_size,
3945 list_max_page_size: cfg.list_max_page_size,
3946 }
3947 };
3948 let nexus_endpoint_admin = Arc::new(tokeira_edge::nexus_endpoint::NexusEndpointAdmin::new(
3949 nexus_store.clone(),
3950 Arc::new(tokeira_edge::nexus_endpoint::CacheNamespaceResolver::new(
3951 namespaces.clone(),
3952 )),
3953 nexus_endpoint_limits,
3954 ));
3955 let dispatch_grpc = tokeira_edge::grpc::workflow_dispatch::WorkflowTaskDispatchGrpc::new(
3956 runtime.clone(),
3957 interceptors.clone(),
3958 workflow_service.clone(),
3959 )
3960 .with_peer(peer.clone());
3961 let activity_dispatch_grpc =
3962 tokeira_edge::grpc::activity_dispatch::ActivityTaskDispatchGrpc::new(
3963 runtime.clone(),
3964 interceptors.clone(),
3965 peer.clone(),
3966 );
3967 let outbound_grpc =
3968 tokeira_edge::grpc::outbound_delivery::OutboundDeliveryGrpc::new(runtime.clone(), peer);
3969 let operator_service = OperatorService::new(operator_api, interceptors)
3970 .with_nexus_endpoints(nexus_endpoint_admin)
3971 .with_namespace_deletion(namespaces.clone(), Arc::new(workflow_service.clone()));
3972 let production_nexus_authorizer = authorization.nexus;
3973 let nexus_http_authorizer: Arc<dyn tokeira_edge::nexus_http::NexusHttpAuthorizer> =
3974 match harness::installed().and_then(|hooks| hooks.wrap_nexus_http_authorizer.as_ref()) {
3975 Some(wrap) => wrap(production_nexus_authorizer),
3976 None => production_nexus_authorizer,
3977 };
3978 let nexus_http_layer =
3979 NexusHttpLayer::new(Arc::new(tokeira_edge::nexus_http::NexusHttpHandler::new(
3980 namespaces.clone(),
3981 nexus_store,
3982 nexus_task_broker,
3983 nexus_http_waiters,
3984 nexus_http_authorizer,
3985 )));
3986 let http_api_layer = HttpApiLayer::new(
3987 HttpApiCatalog::pinned().context("failed to build Temporal HTTP API route catalog")?,
3988 HttpApiPolicy::new(
3989 effective_config.policy.http_api.allowed_hosts.clone(),
3990 effective_config
3991 .policy
3992 .http_api
3993 .additional_forwarded_headers
3994 .clone(),
3995 )
3996 .context("failed to compile Temporal HTTP API policy")?,
3997 );
3998
3999 let (workflow_grpc, chasm_engine) = {
4006 let repair_archetype = registry.archetype_id(
4010 <tokeira_chasm_activity::ActivityExecution as tokeira_chasm::Component>::FQN,
4011 );
4012 tokeira_storage::run_current_execution_backfill(
4015 chasm_node_repo.as_ref(),
4016 repair_archetype.context("activity CHASM archetype is not registered")?,
4017 500,
4018 )
4019 .await
4020 .context("failed to backfill CHASM current executions")?;
4021 check_registered_archetypes(&chasm_node_repo.distinct_archetypes().await?, ®istry)
4022 .map_err(registry_start_error)?;
4023 let repair_nodes = chasm_node_repo.clone();
4024 let dispatch_queue = Arc::new(tokeira_edge::chasm_activity::ActivityDispatchQueue::new());
4025 let multiplexer = Arc::new(tokeira_runtime::chasm::DispatchMultiplexer::default());
4026 let chasm_visibility_sink =
4031 Arc::new(tokeira_runtime::chasm::ProjectionVisibilitySink::new(
4032 Arc::new(projection_sink()),
4033 effective_config.infrastructure.placement.partition_count,
4034 ));
4035 let mut chasm_engine = tokeira_runtime::chasm::ChasmEngine::new(
4036 chasm_node_repo.clone(),
4037 registry.clone(),
4038 multiplexer.clone(),
4039 chasm_visibility_sink,
4040 );
4041 if let Some(clock) = extensions.clock {
4042 chasm_engine = chasm_engine.with_clock(clock);
4043 }
4044 let chasm_engine = Arc::new(chasm_engine);
4045 let activity_config = tokeira_chasm_activity::ActivityConfig {
4046 enable_standalone: effective_config
4047 .policy
4048 .compatibility
4049 .enable_standalone_activities,
4050 enable_callbacks: effective_config
4051 .policy
4052 .compatibility
4053 .enable_standalone_activity_callbacks,
4054 ..tokeira_chasm_activity::ActivityConfig::default()
4055 };
4056 let standalone_enabled = activity_config.enable_standalone;
4057 let sweeper_engine = chasm_engine.clone();
4059 let dispatch_executor = Arc::new(
4060 tokeira_edge::chasm_executors::ActivityDispatchExecutor::new(
4061 Arc::downgrade(&chasm_engine),
4062 dispatch_queue,
4063 ),
4064 );
4065 let activity_bridge = Arc::new(
4066 tokeira_edge::chasm_activity::ActivityBridge::new(
4067 chasm_engine.clone(),
4068 activity_config.clone(),
4069 DEFAULT_MAX_ID_LENGTH,
4070 )
4071 .with_dispatch_executor(dispatch_executor.clone()),
4072 );
4073 multiplexer
4076 .register(dispatch_executor)
4077 .map_err(registry_start_error)?;
4078 multiplexer
4079 .register(Arc::new(
4080 tokeira_edge::chasm_executors::StartActivityExecutor::new(
4081 Arc::downgrade(&chasm_engine),
4082 activity_config,
4083 DEFAULT_MAX_ID_LENGTH,
4084 ),
4085 ))
4086 .map_err(registry_start_error)?;
4087 multiplexer
4088 .register(Arc::new(
4089 tokeira_edge::chasm_executors::DeliverCallbackExecutor::new(
4090 Arc::downgrade(&chasm_engine),
4091 chasm_nexus_client,
4092 chasm_nexus_config,
4093 namespaces.clone(),
4094 ),
4095 ))
4096 .map_err(registry_start_error)?;
4097 for executor in extensions.executors {
4098 multiplexer
4099 .register(executor)
4100 .map_err(registry_start_error)?;
4101 }
4102 for namespace in namespaces.list_all().await? {
4105 seed_declared_search_attributes(
4106 &visibility_query_store,
4107 namespace_id_for(&namespace.name),
4108 &declared,
4109 )
4110 .await
4111 .map_err(seed_start_error)?;
4112 }
4113 let rebuild = tokeira_runtime::chasm::OutboxRebuildScanner::new(
4114 chasm_node_repo,
4115 chasm_engine.clone(),
4116 multiplexer.clone(),
4117 );
4118 let rebuild_stats = rebuild
4119 .rebuild_once()
4120 .await
4121 .context("failed to rebuild CHASM outboxes before serving")?;
4122 check_rebuilt_outboxes(rebuild_stats)?;
4123 spawn_outbox_rebuild(&engine_tasks, rebuild, background_cancel.clone());
4124 spawn_visibility_repair(
4130 &engine_tasks,
4131 repair_nodes,
4132 Arc::new(projection_sink()),
4133 registry.clone(),
4134 effective_config.infrastructure.placement.partition_count,
4135 background_cancel.clone(),
4136 );
4137 let sweeper_forced =
4143 harness::installed().is_some_and(|hooks| hooks.force_chasm_timer_sweeper);
4144 if standalone_enabled || sweeper_forced || has_extensions {
4145 spawn_chasm_timer_sweeper(
4146 &engine_tasks,
4147 sweeper_engine,
4148 activity_bridge.clone(),
4149 repair_archetype.context("activity CHASM archetype is not registered")?,
4150 background_cancel.clone(),
4151 );
4152 }
4153 (
4154 WorkflowServiceGrpc::new(workflow_service.clone()).with_chasm_activity(activity_bridge),
4155 chasm_engine,
4156 )
4157 };
4158 let admin_grpc = AdminServiceGrpc::new(workflow_service);
4161 let operator_grpc = OperatorServiceGrpc::new(operator_service);
4162
4163 let shard_cleanup = if matches!(transport, StackTransport::Embedded)
4168 && dsql_endpoint.is_some()
4169 && effective_config
4170 .infrastructure
4171 .placement
4172 .controller_endpoint
4173 .is_none()
4174 {
4175 let leases = self_assign_dsql_shards(
4176 runtime.as_ref(),
4177 repo.as_ref(),
4178 effective_config.infrastructure.placement.shard_count,
4179 &node_id,
4180 &node_endpoint,
4181 true,
4182 )
4183 .await?;
4184 Some(self_assigned_shard_cleanup(
4185 runtime.clone(),
4186 repo.clone(),
4187 leases,
4188 ))
4189 } else {
4190 None
4191 };
4192
4193 match dsql_endpoint {
4194 Some(endpoint) => info!(%endpoint, "storage backend: dsql"),
4195 None => info!("storage backend: in-memory"),
4196 }
4197
4198 let log_broadcast = broadcast::Sender::<LogEvent>::new(LOG_BROADCAST_CAPACITY);
4199 let addr = match transport {
4200 StackTransport::Embedded => {
4201 startup_guard.disarm();
4202 return Ok(ConstructedStack::Embedded(EmbeddedStack {
4203 component_visibility_store: chasm_extensions::VisibilityQueryStore(Arc::new(
4204 visibility_query_store,
4205 )),
4206 chasm_engine,
4207 registry,
4208 service: InProcessGrpcService::new(workflow_grpc, operator_grpc, admin_grpc),
4209 background_cancel,
4210 log_broadcast,
4211 recovery_task,
4212 runtime_tasks,
4213 engine_tasks,
4214 shard_cleanup,
4215 }));
4216 }
4217 StackTransport::Network(addr) => addr,
4218 };
4219
4220 let reflection_v1 = tonic_reflection::server::Builder::configure()
4224 .register_encoded_file_descriptor_set(tokeira_proto::public::FILE_DESCRIPTOR_SET)
4225 .build_v1()
4226 .context("failed to build gRPC reflection service")?;
4227 let reflection_v1alpha = tonic_reflection::server::Builder::configure()
4228 .register_encoded_file_descriptor_set(tokeira_proto::public::FILE_DESCRIPTOR_SET)
4229 .build_v1alpha()
4230 .context("failed to build gRPC reflection service")?;
4231
4232 let listener = TcpListener::bind(addr)
4233 .await
4234 .with_context(|| format!("failed to bind tokeirad gRPC listener on {addr}"))?;
4235 let bound_addr = listener
4236 .local_addr()
4237 .context("failed to resolve bound local address for tokeirad gRPC listener")?;
4238
4239 let (shutdown_tx, shutdown_rx) = oneshot::channel();
4240
4241 let wire_coverage_recorder = wire_coverage_enabled().then(|| {
4254 info!("conformance wire-coverage recorder enabled");
4255 Arc::new(WireCoverageRecorder::new())
4256 });
4257 let server_recorder = wire_coverage_recorder.clone();
4258
4259 let server_cancel = background_cancel.clone();
4260 let server_task = tokio::spawn(async move {
4261 let shutdown_signal = async move {
4262 tokio::select! {
4263 _ = shutdown_rx => {},
4264 _ = server_cancel.cancelled() => {},
4265 }
4266 };
4267
4268 match server_recorder {
4272 Some(recorder) => {
4273 Server::builder()
4274 .accept_http1(true)
4275 .layer(nexus_http_layer.clone())
4276 .layer(http_api_layer.clone())
4277 .layer(CorsLayer::permissive())
4278 .layer(GrpcWebLayer::new())
4279 .layer(WireCoverageLayer::new(recorder))
4280 .add_service(workflow_grpc.into_service())
4281 .add_service(operator_grpc.into_service())
4282 .add_service(admin_grpc.into_service())
4283 .add_service(dispatch_grpc.into_service())
4284 .add_service(activity_dispatch_grpc.into_service())
4285 .add_service(outbound_grpc.into_service())
4286 .add_service(reflection_v1)
4287 .add_service(reflection_v1alpha)
4288 .serve_with_incoming_shutdown(TcpListenerStream::new(listener), shutdown_signal)
4289 .await
4290 }
4291 None => {
4292 Server::builder()
4293 .accept_http1(true)
4294 .layer(nexus_http_layer)
4295 .layer(http_api_layer)
4296 .layer(CorsLayer::permissive())
4297 .layer(GrpcWebLayer::new())
4298 .add_service(workflow_grpc.into_service())
4299 .add_service(operator_grpc.into_service())
4300 .add_service(admin_grpc.into_service())
4301 .add_service(dispatch_grpc.into_service())
4302 .add_service(activity_dispatch_grpc.into_service())
4303 .add_service(outbound_grpc.into_service())
4304 .add_service(reflection_v1)
4305 .add_service(reflection_v1alpha)
4306 .serve_with_incoming_shutdown(TcpListenerStream::new(listener), shutdown_signal)
4307 .await
4308 }
4309 }
4310 .with_context(|| format!("failed to serve gRPC transport on {bound_addr}"))?;
4311 Ok::<(), anyhow::Error>(())
4312 });
4313
4314 if let Some(task) = harness::installed().and_then(|hooks| hooks.background_task.as_ref()) {
4320 task(background_cancel.clone());
4321 }
4322
4323 startup_guard.disarm();
4324 Ok(ConstructedStack::Network((
4325 server_task,
4326 bound_addr,
4327 shutdown_tx,
4328 background_cancel,
4329 log_broadcast,
4330 wire_coverage_recorder,
4331 )))
4332}
4333
4334fn with_loopback_port(base: &str, port: u16) -> String {
4338 let trimmed = base.trim_end_matches('/');
4339 match trimmed.rsplit_once(':') {
4340 Some((prefix, suffix))
4343 if !suffix.is_empty() && suffix.bytes().all(|b| b.is_ascii_digit()) =>
4344 {
4345 format!("{prefix}:{port}")
4346 }
4347 _ => format!("{trimmed}:{port}"),
4348 }
4349}
4350
4351fn spawn_nexus_callback_server(
4356 listener: TcpListener,
4357 runtime: Arc<dyn WorkflowRuntimeApi>,
4358 cancel: CancellationToken,
4359) {
4360 tokio::spawn(async move {
4361 loop {
4362 let stream = tokio::select! {
4363 _ = cancel.cancelled() => break,
4364 accept = listener.accept() => match accept {
4365 Ok((stream, _peer)) => stream,
4366 Err(error) => {
4367 tracing::warn!(?error, "nexus callback listener accept failed");
4368 continue;
4369 }
4370 },
4371 };
4372 let io = TokioIo::new(stream);
4373 let runtime = runtime.clone();
4374 let conn_cancel = cancel.clone();
4375 tokio::spawn(async move {
4376 let service = service_fn(move |request| {
4377 let runtime = runtime.clone();
4378 async move {
4379 Ok::<_, Infallible>(
4380 nexus_callback_response(runtime.as_ref(), request).await,
4381 )
4382 }
4383 });
4384 let conn = hyper::server::conn::http1::Builder::new().serve_connection(io, service);
4385 tokio::pin!(conn);
4386 tokio::select! {
4387 result = conn.as_mut() => {
4388 if let Err(error) = result {
4389 tracing::debug!(?error, "nexus callback connection closed with error");
4390 }
4391 }
4392 _ = conn_cancel.cancelled() => {
4393 conn.as_mut().graceful_shutdown();
4394 let _ = conn.await;
4395 }
4396 }
4397 });
4398 }
4399 });
4400}
4401
4402#[derive(Debug)]
4404enum BoundedBody {
4405 Collected(Bytes),
4407 TooLarge,
4409 ReadFailed(Box<dyn std::error::Error + Send + Sync>),
4411}
4412
4413async fn collect_bounded_body<B>(body: B, limit: usize) -> BoundedBody
4420where
4421 B: hyper::body::Body<Data = Bytes>,
4422 B::Error: Into<Box<dyn std::error::Error + Send + Sync>>,
4423{
4424 match Limited::new(body, limit).collect().await {
4425 Ok(collected) => BoundedBody::Collected(collected.to_bytes()),
4426 Err(error) if error.downcast_ref::<LengthLimitError>().is_some() => BoundedBody::TooLarge,
4427 Err(error) => BoundedBody::ReadFailed(error),
4428 }
4429}
4430
4431async fn nexus_callback_response(
4435 runtime: &dyn WorkflowRuntimeApi,
4436 request: Request<Incoming>,
4437) -> Response<Full<Bytes>> {
4438 let (parts, body) = request.into_parts();
4439
4440 if parts.method != Method::POST || parts.uri.path() != NEXUS_CALLBACK_PATH {
4441 return nexus_response(StatusCode::NOT_FOUND, None);
4442 }
4443
4444 let header = |name: &str| {
4445 parts
4446 .headers
4447 .get(name)
4448 .and_then(|value| value.to_str().ok())
4449 .map(str::to_owned)
4450 };
4451 let token = header(TEMPORAL_CALLBACK_TOKEN_HEADER);
4452 let operation_token = header(NEXUS_OPERATION_TOKEN_HEADER);
4453 let state = header(NEXUS_OPERATION_STATE_HEADER);
4454 let content_type = header(hyper::header::CONTENT_TYPE.as_str());
4455
4456 let body_bytes = match collect_bounded_body(body, MAX_NEXUS_PAYLOAD_BYTES).await {
4463 BoundedBody::Collected(bytes) => bytes,
4464 BoundedBody::TooLarge => {
4465 tracing::debug!(
4466 limit = MAX_NEXUS_PAYLOAD_BYTES,
4467 "nexus callback body too large"
4468 );
4469 return nexus_response(
4470 StatusCode::PAYLOAD_TOO_LARGE,
4471 Some(b"request body too large".to_vec()),
4472 );
4473 }
4474 BoundedBody::ReadFailed(error) => {
4475 tracing::debug!(?error, "failed to read nexus callback request body");
4476 return nexus_response(
4477 StatusCode::BAD_REQUEST,
4478 Some(b"failed to read request body".to_vec()),
4479 );
4480 }
4481 };
4482
4483 let CallbackResponse { status, body } = handle_nexus_callback(
4484 runtime,
4485 token.as_deref(),
4486 operation_token.as_deref(),
4487 state.as_deref(),
4488 content_type.as_deref(),
4489 &body_bytes,
4490 )
4491 .await;
4492
4493 let status = StatusCode::from_u16(status).unwrap_or(StatusCode::INTERNAL_SERVER_ERROR);
4494 nexus_response(status, body)
4495}
4496
4497fn nexus_response(status: StatusCode, body: Option<Vec<u8>>) -> Response<Full<Bytes>> {
4498 let mut response = Response::new(Full::new(Bytes::from(body.unwrap_or_default())));
4499 *response.status_mut() = status;
4500 response
4501}
4502
4503const LOG_BROADCAST_CAPACITY: usize = 256;
4508
4509const WIRE_COVERAGE_ENV: &str = "TOKEIRA_CONFORMANCE_WIRE_COVERAGE";
4517
4518const WIRE_COVERAGE_OUT_ENV: &str = "TOKEIRA_CONFORMANCE_WIRE_COVERAGE_OUT";
4527
4528const WIRE_COVERAGE_DEFAULT_OUT: &str = "./wire-coverage.json";
4535
4536fn wire_coverage_enabled() -> bool {
4542 match std::env::var(WIRE_COVERAGE_ENV) {
4543 Ok(value) => {
4544 let trimmed = value.trim();
4545 !trimmed.is_empty()
4546 && !trimmed.eq_ignore_ascii_case("0")
4547 && !trimmed.eq_ignore_ascii_case("false")
4548 }
4549 Err(_) => false,
4550 }
4551}
4552
4553fn wire_coverage_out_path() -> PathBuf {
4561 match std::env::var(WIRE_COVERAGE_OUT_ENV) {
4562 Ok(value) if !value.trim().is_empty() => PathBuf::from(value.trim()),
4563 _ => PathBuf::from(WIRE_COVERAGE_DEFAULT_OUT),
4564 }
4565}
4566
4567async fn build_nexus_endpoint_store(
4575 namespaces: &dyn NamespaceCache,
4576 configs: HashMap<String, BootstrapNexusEndpointConfig>,
4577) -> Result<Arc<InMemoryNexusEndpointStore>> {
4578 let store = Arc::new(InMemoryNexusEndpointStore::new());
4579 let now = std::time::SystemTime::now()
4580 .duration_since(std::time::UNIX_EPOCH)
4581 .map(|d| d.as_nanos() as i64)
4582 .unwrap_or(0);
4583 for (endpoint_name, config) in configs {
4584 let target = match config.target {
4585 BootstrapNexusEndpointTarget::External { address } => {
4586 NexusEndpointSpecTarget::External { url: address }
4587 }
4588 BootstrapNexusEndpointTarget::Worker {
4589 namespace_name,
4590 task_queue,
4591 } => {
4592 let namespace = namespaces.get(&namespace_name).await?.ok_or_else(|| {
4593 anyhow!(
4594 "failed to register nexus worker endpoint `{endpoint_name}`: namespace `{namespace_name}` not found"
4595 )
4596 })?;
4597 NexusEndpointSpecTarget::Worker {
4598 namespace_name: namespace.name.clone(),
4599 namespace_id: namespace_id_for(&namespace.name).0.to_string(),
4600 task_queue,
4601 }
4602 }
4603 };
4604 store
4605 .create(
4606 NexusEndpointSpec {
4607 name: endpoint_name.clone(),
4608 description: Vec::new(),
4609 target,
4610 },
4611 now,
4612 )
4613 .map_err(|e| anyhow!("failed to seed nexus endpoint `{endpoint_name}`: {e}"))?;
4614 }
4615 Ok(store)
4616}
4617
4618async fn self_assign_dsql_shards<R>(
4619 runtime: &TokeiraRuntime<R>,
4620 lease_repository: &R,
4621 shard_count: u32,
4622 node_id: &IncarnationId,
4623 node_endpoint: &NodeEndpoint,
4624 require_all: bool,
4625) -> Result<Vec<SelfAssignedShardLease>>
4626where
4627 R: LeaseRepository + RunRepository + 'static,
4628{
4629 let existing_leases = lease_repository.list_bundle_leases().await?;
4633 for lease in &existing_leases {
4634 if let Some(owner) = &lease.owner_node_id {
4635 let _ = lease_repository
4636 .relinquish_bundle(lease.bundle_id, owner.clone(), lease.epoch)
4637 .await;
4638 }
4639 }
4640
4641 let mut acquired = Vec::new();
4642 let mut recovered = SweepResult::default();
4643 let owner = node_id.to_string();
4644 for shard_index in 0..shard_count {
4645 let shard_id = ShardId(shard_index);
4646 match lease_repository
4647 .try_acquire_bundle(shard_id, owner.clone(), node_endpoint.as_authority())
4648 .await
4649 {
4650 Ok(LeaseOutcome::Acquired { epoch } | LeaseOutcome::Renewed { epoch }) => {
4651 match runtime.recover_self_assigned_shard(shard_id, epoch).await {
4657 Ok(swept) => {
4658 recovered.workflow_tasks_republished += swept.workflow_tasks_republished;
4659 recovered.activity_tasks_republished += swept.activity_tasks_republished;
4660 recovered.due_timers_injected += swept.due_timers_injected;
4661 recovered.activity_tracking_entries_reconstructed +=
4662 swept.activity_tracking_entries_reconstructed;
4663 acquired.push(SelfAssignedShardLease {
4664 shard_id,
4665 owner: owner.clone(),
4666 epoch,
4667 });
4668 }
4669 Err(error) => {
4670 let _ = lease_repository
4672 .relinquish_bundle(shard_id, owner.clone(), epoch)
4673 .await;
4674 if require_all {
4675 rollback_self_assigned_shards(runtime, lease_repository, &acquired)
4676 .await;
4677 return Err(error).context(format!(
4678 "failed to recover embedded DSQL shard {shard_index}"
4679 ));
4680 }
4681 tracing::warn!(
4682 shard_index,
4683 ?error,
4684 "failed to recover self-assigned shard; leaving it unowned"
4685 );
4686 }
4687 }
4688 }
4689 Ok(LeaseOutcome::Rejected {
4690 current_owner,
4691 current_epoch,
4692 }) => {
4693 if require_all {
4694 rollback_self_assigned_shards(runtime, lease_repository, &acquired).await;
4695 return Err(anyhow!(
4696 "embedded DSQL shard {shard_index} is held by owner {current_owner} at epoch {}",
4697 current_epoch.0
4698 ));
4699 }
4700 tracing::warn!(
4701 shard_index,
4702 %current_owner,
4703 current_epoch = current_epoch.0,
4704 "failed to self-assign shard: lease is held by another owner"
4705 );
4706 }
4707 Err(error) => {
4708 if require_all {
4709 rollback_self_assigned_shards(runtime, lease_repository, &acquired).await;
4710 return Err(error).context(format!(
4711 "failed to self-assign embedded DSQL shard {shard_index}"
4712 ));
4713 }
4714 tracing::warn!(shard_index, ?error, "failed to self-assign shard");
4715 }
4716 }
4717 }
4718 info!(
4719 acquired = acquired.len(),
4720 shard_count,
4721 workflow_tasks_republished = recovered.workflow_tasks_republished,
4722 activity_tasks_republished = recovered.activity_tasks_republished,
4723 due_timers_injected = recovered.due_timers_injected,
4724 activity_tracking_entries_reconstructed = recovered.activity_tracking_entries_reconstructed,
4725 "self-assigned and recovered DSQL shards (no controller)"
4726 );
4727 Ok(acquired)
4728}
4729
4730async fn rollback_self_assigned_shards<R>(
4731 runtime: &TokeiraRuntime<R>,
4732 lease_repository: &R,
4733 leases: &[SelfAssignedShardLease],
4734) where
4735 R: LeaseRepository + RunRepository + 'static,
4736{
4737 for lease in leases.iter().rev() {
4738 runtime.relinquish_shard(lease.shard_id).await;
4739 let _ = lease_repository
4740 .relinquish_bundle(lease.shard_id, lease.owner.clone(), lease.epoch)
4741 .await;
4742 }
4743}
4744
4745fn self_assigned_shard_cleanup<R>(
4746 runtime: Arc<TokeiraRuntime<R>>,
4747 lease_repository: Arc<R>,
4748 leases: Vec<SelfAssignedShardLease>,
4749) -> CleanupAction
4750where
4751 R: LeaseRepository + RunRepository + 'static,
4752{
4753 Box::new(move |deadline| {
4754 Box::pin(async move {
4755 let mut first_error = None;
4756 for lease in leases.into_iter().rev() {
4757 runtime.relinquish_shard(lease.shard_id).await;
4758 let now = Instant::now();
4759 if now >= deadline {
4760 first_error.get_or_insert_with(|| {
4761 anyhow!("deadline elapsed while relinquishing embedded DSQL shards")
4762 });
4763 continue;
4764 }
4765 let release = tokio::time::timeout(
4766 deadline.saturating_duration_since(now),
4767 lease_repository.relinquish_bundle(lease.shard_id, lease.owner, lease.epoch),
4768 )
4769 .await;
4770 match release {
4771 Ok(Ok(LeaseOutcome::Acquired { .. })) => {}
4772 Ok(Ok(outcome)) => {
4773 first_error.get_or_insert_with(|| {
4774 anyhow!("embedded DSQL shard release was fenced: {outcome:?}")
4775 });
4776 }
4777 Ok(Err(error)) => {
4778 first_error.get_or_insert(error);
4779 }
4780 Err(_) => {
4781 first_error.get_or_insert_with(|| {
4782 anyhow!("deadline elapsed while relinquishing embedded DSQL shard")
4783 });
4784 }
4785 }
4786 }
4787 match first_error {
4788 Some(error) => Err(error),
4789 None => Ok(()),
4790 }
4791 })
4792 })
4793}
4794
4795fn configured_node_endpoint(config: &TokeiraConfig, listen_addr: SocketAddr) -> NodeEndpoint {
4796 NodeEndpoint {
4797 host: config.infrastructure.placement.node_host.clone(),
4798 port: config
4799 .infrastructure
4800 .placement
4801 .node_port
4802 .unwrap_or_else(|| listen_addr.port()),
4803 }
4804}
4805
4806fn placement_config(config: &TokeiraConfig) -> PlacementConfig {
4807 let placement = &config.infrastructure.placement;
4808 PlacementConfig {
4809 shard_count: placement.shard_count,
4810 bundle_count: placement.bundle_count,
4811 partition_count: placement.partition_count,
4812 hash_version: placement.hash_version,
4813 }
4814}
4815
4816fn membership_config(
4817 config: &TokeiraConfig,
4818 node_id: IncarnationId,
4819 node_endpoint: NodeEndpoint,
4820 controller_endpoint: String,
4821) -> MembershipConfig {
4822 let placement = &config.infrastructure.placement;
4823 MembershipConfig {
4824 controller_endpoint,
4825 heartbeat_interval: std::time::Duration::from_millis(placement.heartbeat_interval_ms),
4826 reconnect_base_delay: std::time::Duration::from_millis(placement.reconnect_base_delay_ms),
4827 reconnect_max_delay: std::time::Duration::from_millis(placement.reconnect_max_delay_ms),
4828 node_id,
4829 node_endpoint,
4830 zone: None,
4831 version: tokeira_build_info::TOKEIRA_VERSION.to_string(),
4832 build_id: tokeira_build_info::TOKEIRA_GIT_SHA.to_string(),
4833 }
4834}
4835
4836#[derive(Debug)]
4839struct NoopConnectionBudgetApplier;
4840
4841impl ConnectionBudgetApplier for NoopConnectionBudgetApplier {
4842 fn apply_budget(
4843 &self,
4844 _rate_per_second: f64,
4845 _capacity: u64,
4846 _max_reservoir_size: u32,
4847 ) -> Result<()> {
4848 Ok(())
4849 }
4850
4851 fn available_connections(&self) -> u32 {
4852 0
4853 }
4854}
4855
4856#[derive(Debug)]
4867struct DsqlConnectionBudgetApplier {
4868 director: Arc<DsqlConnectionDirector>,
4869}
4870
4871impl ConnectionBudgetApplier for DsqlConnectionBudgetApplier {
4872 fn apply_budget(
4873 &self,
4874 rate_per_second: f64,
4875 capacity: u64,
4876 max_reservoir_size: u32,
4877 ) -> Result<()> {
4878 let retired = self.director.apply_connection_budget(max_reservoir_size)?;
4879 tracing::debug!(
4880 rate_per_second,
4881 capacity,
4882 max_reservoir_size,
4883 reservoir_target = self.director.reservoir_target(),
4884 retired,
4885 "applied controller connection budget to the DSQL reservoir"
4886 );
4887 Ok(())
4888 }
4889
4890 fn available_connections(&self) -> u32 {
4891 u32::try_from(self.director.ready_connections()).unwrap_or(u32::MAX)
4892 }
4893}
4894
4895struct StoreExecutionResolver<R> {
4901 repo: Arc<R>,
4902}
4903
4904impl<R> StoreExecutionResolver<R> {
4905 fn new(repo: Arc<R>) -> Self {
4906 Self { repo }
4907 }
4908}
4909
4910fn apply_reported_problem_search_attribute(
4911 search_attributes: &mut tokeira_types::SearchAttributes,
4912 problem: Option<WorkflowTaskReportedProblem>,
4913) {
4914 let Some(problem) = problem else {
4915 return;
4916 };
4917 let entries = match &problem.problem {
4925 tokeira_kernel::WorkflowTaskProblem::Failed(cause) => vec![
4926 "category=WorkflowTaskFailed".to_string(),
4927 format!("cause=WorkflowTaskFailedCause{}", cause.as_str()),
4928 ],
4929 tokeira_kernel::WorkflowTaskProblem::TimedOutStartToClose => vec![
4930 "category=WorkflowTaskTimedOut".to_string(),
4931 "cause=WorkflowTaskTimedOutCauseStartToClose".to_string(),
4932 ],
4933 };
4934 search_attributes.0.insert(
4935 "TemporalReportedProblems".to_string(),
4936 SearchAttrValue::KeywordList(entries),
4937 );
4938}
4939
4940#[async_trait]
4941impl<R> ExecutionResolver for StoreExecutionResolver<R>
4942where
4943 R: RunRepository + 'static,
4944{
4945 async fn current_run_key(
4946 &self,
4947 namespace: &str,
4948 workflow_id: &str,
4949 ) -> Result<Option<tokeira_types::RunKey>> {
4950 let namespace_id = namespace_id_for(namespace);
4951 let result = self
4952 .repo
4953 .resolve_execution(&ExecutionRef {
4954 namespace_id,
4955 workflow_id: WorkflowId(workflow_id.to_string()),
4956 run_id: None,
4957 })
4958 .await?;
4959 if result.is_some() {
4960 return Ok(result);
4961 }
4962 self.repo
4963 .find_latest_run(namespace_id, &WorkflowId(workflow_id.to_string()))
4964 .await
4965 }
4966
4967 async fn describe_execution(
4968 &self,
4969 namespace: &str,
4970 workflow_id: &str,
4971 run_id: Option<tokeira_types::RunId>,
4972 ) -> Result<Option<WorkflowExecutionDescription>> {
4973 let namespace_id = namespace_id_for(namespace);
4974 let run_key = match self
4975 .repo
4976 .resolve_execution(&ExecutionRef {
4977 namespace_id,
4978 workflow_id: WorkflowId(workflow_id.to_string()),
4979 run_id,
4980 })
4981 .await?
4982 {
4983 Some(rk) => rk,
4984 None if run_id.is_none() => {
4985 match self
4986 .repo
4987 .find_latest_run(namespace_id, &WorkflowId(workflow_id.to_string()))
4988 .await?
4989 {
4990 Some(rk) => rk,
4991 None => return Ok(None),
4992 }
4993 }
4994 None => return Ok(None),
4995 };
4996
4997 let (loaded, stats) = self.repo.load_run_with_stats(run_key).await?;
4998 match loaded {
4999 LoadedRun::Existing(state) => {
5000 let history_size_bytes = stats.history_size_bytes;
5008 let external_payload_count = state.external_payload_count;
5009 let external_payload_size_bytes = state.external_payload_size_bytes;
5010 let reported_problem = reported_problem_from_state(&state);
5014 let versioning_info = state.versioning_info.clone();
5015 let most_recent_worker_version_stamp = versioning_info
5016 .as_ref()
5017 .and_then(|info| info.most_recent_worker_version_stamp.clone());
5018 let pending_nexus_operations =
5019 tokeira_edge::translate::describe_pending_nexus_operations(&state);
5020 let mut search_attributes = state.search_attributes;
5021 apply_reported_problem_search_attribute(&mut search_attributes, reported_problem);
5022 if let Some(build_ids) = versioning_info
5023 .as_ref()
5024 .map(|info| &info.build_id_search_attributes)
5025 .filter(|build_ids| !build_ids.is_empty())
5026 {
5027 search_attributes.0.insert(
5028 "BuildIds".to_string(),
5029 tokeira_types::SearchAttrValue::KeywordList(build_ids.clone()),
5030 );
5031 }
5032 Ok(Some(WorkflowExecutionDescription {
5033 namespace: namespace.to_string(),
5034 workflow_id: state.workflow_id.0,
5035 run_key: state.run_key,
5036 run_id: state.run_id,
5037 workflow_type: state.workflow_type.0,
5038 task_queue: state.task_queue.0.clone(),
5039 status: state.status,
5040 start_time: Some(state.started_at),
5041 close_time: state.closed_at,
5042 execution_time: deadline_after(
5047 state.started_at,
5048 state.workflow_start_delay.unwrap_or_default(),
5049 ),
5050 execution_config: tokeira_edge::translate::ExecutionConfigDescription {
5051 task_queue: state.task_queue.0.clone(),
5052 workflow_execution_timeout: state.workflow_execution_timeout,
5053 workflow_run_timeout: state.workflow_run_timeout,
5054 default_workflow_task_timeout: state.workflow_task_timeout,
5055 user_metadata: state.user_metadata.map(|metadata| {
5059 tokeira_edge::translate::UserMetadata {
5060 summary: metadata.summary,
5061 details: metadata.details,
5062 }
5063 }),
5064 },
5065 history_length: state.last_event_id,
5066 history_size_bytes,
5067 state_transition_count: state.transition_seq.0 as i64,
5068 parent_namespace_id: state
5069 .parent_namespace_id
5070 .map(|namespace_id| namespace_id.0.to_string()),
5071 parent_workflow_id: state.parent_workflow_id.clone(),
5072 parent_run_id: state.parent_run_id,
5073 root_workflow_id: state.root_workflow_id.clone(),
5074 root_run_id: state.root_run_id,
5075 first_run_id: state.first_execution_run_id,
5076 memo: state.memo,
5077 search_attributes,
5078 pending_activities: state
5079 .activities
5080 .values()
5081 .map(
5082 |activity| tokeira_edge::translate::PendingActivityDescription {
5083 activity_id: activity.activity_id.clone(),
5084 activity_type: activity.activity_type.clone(),
5085 is_started: activity.started_at.is_some(),
5086 cancel_requested: activity.cancel_requested,
5087 attempt: activity.attempt,
5088 maximum_attempts: activity
5089 .retry_policy
5090 .as_ref()
5091 .map(|policy| policy.maximum_attempts)
5092 .unwrap_or_default(),
5093 scheduled_at: activity.scheduled_at,
5094 started_at: activity.started_at,
5095 last_failure: activity.last_failure.clone(),
5096 heartbeat_details: activity.heartbeat_details.clone(),
5097 last_heartbeat_at: activity.last_heartbeat_at,
5098 last_worker_identity: {
5099 let started = activity
5104 .started_identity
5105 .clone()
5106 .map(|identity| identity.0)
5107 .unwrap_or_default();
5108 if started.is_empty() && activity.retry_policy.is_some() {
5109 activity
5110 .retry_last_worker_identity
5111 .clone()
5112 .map(|identity| identity.0)
5113 .unwrap_or_default()
5114 } else {
5115 started
5116 }
5117 },
5118 paused: activity.pause_info.is_some(),
5119 pause_info: activity.pause_info.as_ref().map(|info| {
5120 tokeira_edge::translate::PauseInfoDescription {
5121 identity: info.identity.clone(),
5122 paused_time: info.pause_time,
5123 reason: info.reason.clone(),
5124 rule_id: info.rule_id.clone(),
5125 }
5126 }),
5127 activity_options: tokeira_edge::translate::ActivityOptions {
5128 task_queue: Some(activity.task_queue.0.clone()),
5129 schedule_to_close_timeout: activity.schedule_to_close_timeout,
5130 schedule_to_start_timeout: activity.schedule_to_start_timeout,
5131 start_to_close_timeout: activity.start_to_close_timeout,
5132 heartbeat_timeout: activity.heartbeat_timeout,
5133 retry_policy: activity.retry_policy.clone(),
5134 priority: activity.priority.as_ref().map(|priority| {
5135 tokeira_edge::translate::Priority {
5136 priority_key: priority.priority_key,
5137 fairness_key: priority.fairness_key.clone(),
5138 fairness_weight: priority.fairness_weight,
5139 }
5140 }),
5141 },
5142 },
5143 )
5144 .collect(),
5145 pending_children: state
5146 .children
5147 .values()
5148 .map(|child| tokeira_edge::translate::PendingChildDescription {
5149 workflow_id: child.child_workflow_id.0.clone(),
5150 run_id: child
5151 .child_run_id
5152 .as_ref()
5153 .map(|run_id| run_id.0.to_string()),
5154 workflow_type: String::new(),
5155 initiated_event_id: child.initiated_event_id,
5156 parent_close_policy: child.parent_close_policy,
5157 })
5158 .collect(),
5159 pending_workflow_task: state.pending_workflow_task.as_ref().map(|task| {
5160 tokeira_edge::translate::PendingWorkflowTaskDescription {
5161 is_started: task.started_event_id.is_some(),
5162 scheduled_at: task.scheduled_at,
5163 started_at: task.started_at,
5164 attempt: task.attempt,
5165 }
5166 }),
5167 callbacks: state.completion_callbacks.clone(),
5168 pending_nexus_operations,
5169 pause_info: state.pause_info.as_ref().map(|info| {
5170 tokeira_edge::translate::PauseInfoDescription {
5171 identity: info.identity.clone(),
5172 paused_time: info.pause_time,
5173 reason: info.reason.clone(),
5174 rule_id: None,
5175 }
5176 }),
5177 execution_expiration_time: state.workflow_execution_timeout.map(|timeout| {
5178 deadline_after(
5179 state.first_run_started_at.unwrap_or(state.started_at),
5180 timeout,
5181 )
5182 }),
5183 run_expiration_time: state
5184 .workflow_run_timeout
5185 .map(|timeout| deadline_after(state.started_at, timeout)),
5186 cancel_requested: state.cancel_requested,
5187 original_start_time: state.first_run_started_at.unwrap_or(state.started_at),
5188 versioning_info: versioning_info
5189 .filter(|info| info.has_execution_versioning_info()),
5190 worker_deployment_name: state.worker_deployment_name.clone(),
5191 priority: state.priority.as_ref().map(|priority| {
5192 tokeira_edge::translate::Priority {
5193 priority_key: priority.priority_key,
5194 fairness_key: priority.fairness_key.clone(),
5195 fairness_weight: priority.fairness_weight,
5196 }
5197 }),
5198 auto_reset_points: state.auto_reset_points.clone(),
5199 most_recent_worker_version_stamp,
5200 request_id_infos: state.request_id_infos.clone(),
5201 external_payload_count,
5202 external_payload_size_bytes,
5203 }))
5204 }
5205 LoadedRun::Absent => Err(anyhow!("resolved run missing from storage: {:?}", run_key)),
5206 }
5207 }
5208}
5209
5210#[doc(hidden)]
5211pub fn __cli_parse() -> Cli {
5212 Cli::parse()
5213}
5214
5215#[cfg(test)]
5216#[allow(unsafe_code)]
5218mod tests {
5219 use super::*;
5220 use metrics::with_local_recorder;
5221 use metrics_util::debugging::DebuggingRecorder;
5222 use opentelemetry::trace::TracerProvider as _;
5223 use opentelemetry_sdk::{
5224 error::{OTelSdkError, OTelSdkResult},
5225 trace::{SdkTracerProvider, SpanData, SpanExporter},
5226 };
5227 use proptest::prelude::*;
5228 use tracing_subscriber::layer::SubscriberExt as _;
5229 use tokeira_runtime::{EndpointTarget, NexusEndpointConfig};
5232
5233 #[test]
5239 fn the_dsql_stack_copies_the_attribute_index_before_anything_applies_an_image() {
5240 let source = include_str!("lib.rs");
5241 assert_eq!(
5242 source.matches("build_dsql_stack(\n").count(),
5243 3,
5244 "the definition, build_and_serve's DSQL branch and the embedded engine"
5245 );
5246 let start = source
5247 .find("async fn build_dsql_stack(")
5248 .expect("build_dsql_stack is defined");
5249 let body = &source[start..];
5250 let end = body[1..]
5251 .find("\nasync fn ")
5252 .map_or(body.len(), |end| end + 1);
5253 let body = &body[..end];
5254 let copy = body
5255 .find(".copy_legacy_attributes()")
5256 .expect("build_dsql_stack copies the attribute index");
5257 let services = body
5258 .find("build_service_stack_with_storage(")
5259 .expect("build_dsql_stack builds the services");
5260 assert!(
5261 copy < services,
5262 "the copy runs before the services are built"
5263 );
5264 }
5265
5266 #[derive(Clone, Copy, Debug, PartialEq, Eq)]
5267 enum StartupModelPhase {
5268 Cluster,
5269 Pool,
5270 Schema,
5271 Ownership,
5272 Runtime,
5273 }
5274
5275 #[derive(Clone, Copy, Debug, PartialEq, Eq)]
5276 enum StartupModelEvent {
5277 Attempt(StartupModelPhase),
5278 Complete(StartupModelPhase),
5279 ReleaseOwnership,
5280 ClosePool,
5281 ReturnHandle,
5282 }
5283
5284 const STARTUP_MODEL_PHASES: [StartupModelPhase; 5] = [
5285 StartupModelPhase::Cluster,
5286 StartupModelPhase::Pool,
5287 StartupModelPhase::Schema,
5288 StartupModelPhase::Ownership,
5289 StartupModelPhase::Runtime,
5290 ];
5291
5292 fn startup_model(failure_boundary: Option<usize>) -> Vec<StartupModelEvent> {
5293 let mut events = Vec::new();
5294 let mut pool_open = false;
5295 let mut ownership_held = false;
5296 for (index, phase) in STARTUP_MODEL_PHASES.into_iter().enumerate() {
5297 events.push(StartupModelEvent::Attempt(phase));
5298 if failure_boundary == Some(index) {
5299 if ownership_held {
5300 events.push(StartupModelEvent::ReleaseOwnership);
5301 }
5302 if pool_open {
5303 events.push(StartupModelEvent::ClosePool);
5304 }
5305 return events;
5306 }
5307 events.push(StartupModelEvent::Complete(phase));
5308 pool_open |= phase == StartupModelPhase::Pool;
5309 ownership_held |= phase == StartupModelPhase::Ownership;
5310 }
5311 events.push(StartupModelEvent::ReturnHandle);
5312 events
5313 }
5314
5315 #[derive(Clone, Copy, Debug, PartialEq, Eq)]
5316 enum ShutdownModelEvent {
5317 CloseAdmission,
5318 CancelTasks,
5319 CallsComplete,
5320 DrainComplete,
5321 TasksComplete,
5322 JoinComplete,
5323 FinishTelemetry,
5324 AttemptShardRelease,
5325 CloseOwnershipAdmission,
5326 AttemptOwnershipRelease,
5327 AttemptPoolClose,
5328 ReturnToHost,
5329 }
5330
5331 fn shutdown_model(calls_pending: bool, tasks_pending: bool) -> Vec<ShutdownModelEvent> {
5332 let mut events = vec![
5333 ShutdownModelEvent::CloseAdmission,
5334 ShutdownModelEvent::CancelTasks,
5335 ];
5336 if calls_pending {
5337 events.push(ShutdownModelEvent::CallsComplete);
5338 }
5339 events.push(ShutdownModelEvent::DrainComplete);
5340 if tasks_pending {
5341 events.push(ShutdownModelEvent::TasksComplete);
5342 }
5343 events.extend([
5344 ShutdownModelEvent::JoinComplete,
5345 ShutdownModelEvent::FinishTelemetry,
5346 ShutdownModelEvent::AttemptShardRelease,
5347 ShutdownModelEvent::CloseOwnershipAdmission,
5348 ShutdownModelEvent::AttemptOwnershipRelease,
5349 ShutdownModelEvent::AttemptPoolClose,
5350 ShutdownModelEvent::ReturnToHost,
5351 ]);
5352 events
5353 }
5354
5355 #[derive(Clone, Copy, Debug)]
5356 enum HostInstrumentationModel {
5357 None,
5358 LocalRecorder,
5359 LocalSubscriber,
5360 RecorderAndSubscriber,
5361 }
5362
5363 fn host_instrumentation_strategy() -> impl Strategy<Value = HostInstrumentationModel> {
5364 prop_oneof![
5365 Just(HostInstrumentationModel::None),
5366 Just(HostInstrumentationModel::LocalRecorder),
5367 Just(HostInstrumentationModel::LocalSubscriber),
5368 Just(HostInstrumentationModel::RecorderAndSubscriber),
5369 ]
5370 }
5371
5372 #[derive(Clone, Copy, Debug)]
5373 enum EmbeddedStorageModeModel {
5374 InMemory,
5375 ManagedDsql,
5376 ExistingDsql,
5377 }
5378
5379 fn storage_mode_strategy() -> impl Strategy<Value = EmbeddedStorageModeModel> {
5380 prop_oneof![
5381 Just(EmbeddedStorageModeModel::InMemory),
5382 Just(EmbeddedStorageModeModel::ManagedDsql),
5383 Just(EmbeddedStorageModeModel::ExistingDsql),
5384 ]
5385 }
5386
5387 #[derive(Clone, Copy, Debug, PartialEq, Eq)]
5388 struct EmbeddedConstructionEffects {
5389 listener_attempts: usize,
5390 global_install_attempts: usize,
5391 signal_handler_attempts: usize,
5392 local_emission_attempts: usize,
5393 }
5394
5395 fn embedded_construction_effects(
5396 _host: HostInstrumentationModel,
5397 _storage: EmbeddedStorageModeModel,
5398 ) -> EmbeddedConstructionEffects {
5399 EmbeddedConstructionEffects {
5402 listener_attempts: 0,
5403 global_install_attempts: 0,
5404 signal_handler_attempts: 0,
5405 local_emission_attempts: 1,
5406 }
5407 }
5408
5409 #[derive(Clone, Copy, Debug)]
5410 enum TelemetryFixture {
5411 NoSubscriber,
5412 LocalRecorder,
5413 FailingExporter,
5414 }
5415
5416 fn telemetry_fixture_strategy() -> impl Strategy<Value = TelemetryFixture> {
5417 prop_oneof![
5418 Just(TelemetryFixture::NoSubscriber),
5419 Just(TelemetryFixture::LocalRecorder),
5420 Just(TelemetryFixture::FailingExporter),
5421 ]
5422 }
5423
5424 #[derive(Clone, Debug)]
5425 struct FailingSpanExporter;
5426
5427 impl SpanExporter for FailingSpanExporter {
5428 async fn export(&self, _batch: Vec<SpanData>) -> OTelSdkResult {
5429 Err(OTelSdkError::InternalFailure(
5430 "fixture exporter rejected the batch".to_owned(),
5431 ))
5432 }
5433 }
5434
5435 fn apply_observed_sequence(ops: &[i16]) -> Vec<u8> {
5436 let mut state = 0_i64;
5437 let mut committed = Vec::with_capacity(ops.len() * std::mem::size_of::<i64>());
5438 let span = tracing::info_span!(
5439 "embedded.observational_fixture",
5440 tokeira.operation = "transition_sequence"
5441 );
5442 let _entered = span.enter();
5443 for op in ops {
5444 record_embedded_lifecycle(
5445 EmbeddedStorageModeLabel::InMemory,
5446 ClusterStatusLabel::NotApplicable,
5447 SchemaOutcomeLabel::NotApplicable,
5448 OwnershipOutcomeLabel::NotApplicable,
5449 DbClassLabel::Commit,
5450 EmbeddedOperationLabel::Startup,
5451 ErrorClassLabel::None,
5452 );
5453 state = state.wrapping_add(i64::from(*op));
5454 committed.extend_from_slice(&state.to_le_bytes());
5455 }
5456 committed
5457 }
5458
5459 fn observed_sequence_bytes(ops: &[i16], fixture: TelemetryFixture) -> Vec<u8> {
5460 match fixture {
5461 TelemetryFixture::NoSubscriber => apply_observed_sequence(ops),
5462 TelemetryFixture::LocalRecorder => {
5463 let recorder = DebuggingRecorder::new();
5464 with_local_recorder(&recorder, || apply_observed_sequence(ops))
5465 }
5466 TelemetryFixture::FailingExporter => {
5467 let provider = SdkTracerProvider::builder()
5468 .with_simple_exporter(FailingSpanExporter)
5469 .build();
5470 let tracer = provider.tracer("observational-property");
5471 let subscriber = tracing_subscriber::registry()
5472 .with(tracing_opentelemetry::layer().with_tracer(tracer));
5473 let dispatch = tracing::Dispatch::new(subscriber);
5474 let bytes =
5475 tracing::dispatcher::with_default(&dispatch, || apply_observed_sequence(ops));
5476 let _ = provider.force_flush();
5477 bytes
5478 }
5479 }
5480 }
5481
5482 proptest! {
5483 #![proptest_config(ProptestConfig::with_cases(128))]
5484
5485 #[test]
5487 fn startup_is_prefix_safe_and_failure_atomic(
5488 failure_boundary in prop::option::of(0usize..STARTUP_MODEL_PHASES.len()),
5489 ) {
5490 let events = startup_model(failure_boundary);
5491 let attempted = events.iter().filter_map(|event| match event {
5492 StartupModelEvent::Attempt(phase) => Some(*phase),
5493 _ => None,
5494 }).collect::<Vec<_>>();
5495 let completed = events.iter().filter_map(|event| match event {
5496 StartupModelEvent::Complete(phase) => Some(*phase),
5497 _ => None,
5498 }).collect::<Vec<_>>();
5499
5500 prop_assert_eq!(attempted.as_slice(), &STARTUP_MODEL_PHASES[..attempted.len()]);
5501 prop_assert_eq!(completed.as_slice(), &STARTUP_MODEL_PHASES[..completed.len()]);
5502 if let Some(boundary) = failure_boundary {
5503 prop_assert!(!events.contains(&StartupModelEvent::ReturnHandle));
5504 let rollback = events.iter().filter(|event| matches!(
5505 event,
5506 StartupModelEvent::ReleaseOwnership | StartupModelEvent::ClosePool
5507 )).copied().collect::<Vec<_>>();
5508 let expected = match boundary {
5509 0 | 1 => Vec::new(),
5510 2 | 3 => vec![StartupModelEvent::ClosePool],
5511 _ => vec![
5512 StartupModelEvent::ReleaseOwnership,
5513 StartupModelEvent::ClosePool,
5514 ],
5515 };
5516 prop_assert_eq!(rollback, expected);
5517 } else {
5518 prop_assert_eq!(completed.len(), STARTUP_MODEL_PHASES.len());
5519 prop_assert_eq!(events.last(), Some(&StartupModelEvent::ReturnHandle));
5520 }
5521 }
5522
5523 #[test]
5525 fn shutdown_establishes_the_host_flush_boundary(
5526 calls_pending in any::<bool>(),
5527 tasks_pending in any::<bool>(),
5528 independent_failure_mask in 0u8..8,
5529 ) {
5530 let events = shutdown_model(calls_pending, tasks_pending);
5531 prop_assert_eq!(events.first(), Some(&ShutdownModelEvent::CloseAdmission));
5532 prop_assert_eq!(events.get(1), Some(&ShutdownModelEvent::CancelTasks));
5533 prop_assert_eq!(events.last(), Some(&ShutdownModelEvent::ReturnToHost));
5534 for cleanup in [
5535 ShutdownModelEvent::AttemptShardRelease,
5536 ShutdownModelEvent::AttemptOwnershipRelease,
5537 ShutdownModelEvent::AttemptPoolClose,
5538 ] {
5539 prop_assert!(events.contains(&cleanup));
5540 }
5541 prop_assert!(
5542 events.iter().position(|event| *event == ShutdownModelEvent::AttemptShardRelease)
5543 < events.iter().position(|event| *event == ShutdownModelEvent::CloseOwnershipAdmission)
5544 );
5545 prop_assert!(
5546 events.iter().position(|event| *event == ShutdownModelEvent::CloseOwnershipAdmission)
5547 < events.iter().position(|event| *event == ShutdownModelEvent::AttemptOwnershipRelease)
5548 );
5549 let failures = (0..3)
5550 .filter(|bit| independent_failure_mask & (1 << bit) != 0)
5551 .count();
5552 prop_assert_eq!(failures, independent_failure_mask.count_ones() as usize);
5553 prop_assert!(
5554 events.iter().position(|event| *event == ShutdownModelEvent::FinishTelemetry)
5555 < events.iter().position(|event| *event == ShutdownModelEvent::ReturnToHost)
5556 );
5557 }
5558
5559 #[test]
5561 fn embedded_construction_is_transport_and_global_state_neutral(
5562 host in host_instrumentation_strategy(),
5563 storage in storage_mode_strategy(),
5564 ) {
5565 let effects = embedded_construction_effects(host, storage);
5566 prop_assert_eq!(effects.listener_attempts, 0);
5567 prop_assert_eq!(effects.global_install_attempts, 0);
5568 prop_assert_eq!(effects.signal_handler_attempts, 0);
5569 prop_assert_eq!(effects.local_emission_attempts, 1);
5570 }
5571
5572 #[test]
5574 fn startup_stays_zero_listener(
5575 host in host_instrumentation_strategy(),
5576 storage in storage_mode_strategy(),
5577 explicit_listens in 0usize..4,
5578 ) {
5579 let startup = embedded_construction_effects(host, storage);
5582 prop_assert_eq!(startup.listener_attempts, 0);
5583 prop_assert_eq!(
5584 startup.listener_attempts + explicit_listens,
5585 explicit_listens
5586 );
5587 }
5588
5589 #[test]
5591 fn telemetry_is_observational_only(
5592 operations in prop::collection::vec(any::<i16>(), 0..64),
5593 first in telemetry_fixture_strategy(),
5594 second in telemetry_fixture_strategy(),
5595 ) {
5596 let expected = observed_sequence_bytes(&operations, TelemetryFixture::NoSubscriber);
5597 prop_assert_eq!(observed_sequence_bytes(&operations, first), expected.clone());
5598 prop_assert_eq!(observed_sequence_bytes(&operations, second), expected);
5599 }
5600 }
5601
5602 #[tokio::test]
5603 async fn listen_after_shutdown_began_is_refused_without_binding() {
5604 let engine = Engine::start().await.expect("engine starts");
5605 engine.background_cancel.cancel();
5609 let error = engine
5610 .listen("127.0.0.1:0".parse().expect("static loopback address"))
5611 .await
5612 .expect_err("a cancelled engine must refuse to bind");
5613 assert!(matches!(error, EngineListenError::ShutDown));
5614 assert_eq!(engine.listeners.attached(), 0);
5615 }
5616
5617 #[test]
5618 fn every_startup_failure_boundary_rolls_back_without_returning_a_handle() {
5619 for boundary in 0..STARTUP_MODEL_PHASES.len() {
5620 let events = startup_model(Some(boundary));
5621 assert!(!events.contains(&StartupModelEvent::ReturnHandle));
5622 }
5623 }
5624
5625 #[test]
5626 fn durable_startup_report_exposes_only_approved_evidence() {
5627 let report = EngineStartupReport {
5628 storage_mode: EmbeddedStorageMode::ManagedDsql,
5629 cluster: Some(ClusterStartupReport {
5630 region: "eu-west-1".to_owned(),
5631 cluster_id: "cluster-1".to_owned(),
5632 cluster_arn: "arn:aws:dsql:eu-west-1:123456789012:cluster/cluster-1".to_owned(),
5633 endpoint: "cluster-1.dsql.eu-west-1.on.aws".to_owned(),
5634 action: ClusterAction::Recovered,
5635 }),
5636 schema: Some(SchemaStartupReport {
5637 observed_version: 1,
5638 minimum_supported_version: 1,
5639 target_version: 1,
5640 maximum_readable_version: 1,
5641 migration_set_digest: "release-digest".to_owned(),
5642 outcome: SchemaStartupOutcome::Compatible,
5643 }),
5644 ownership: Some(OwnershipStartupReport {
5645 outcome: ControlLeaseAcquireOutcome::Clean,
5646 fence_token: 7,
5647 }),
5648 };
5649 let diagnostic = format!("{report:?}");
5650 assert!(!diagnostic.contains("descriptor-path-secret"));
5651 assert!(!diagnostic.contains("create-token-secret"));
5652 assert!(!diagnostic.contains("credential-secret"));
5653 }
5654
5655 #[tokio::test]
5656 async fn explicit_in_memory_start_preserves_legacy_mode_and_shuts_down_cleanly() {
5657 let engine = Engine::start_with_embedded_config(EmbeddedEngineConfig::default())
5658 .await
5659 .expect("explicit in-memory startup succeeds");
5660 assert_eq!(engine.startup_report(), &EngineStartupReport::in_memory());
5661 engine.shutdown().await.expect("tracked shutdown succeeds");
5662 }
5663
5664 #[test]
5665 fn embedded_lifecycle_composes_with_a_host_local_recorder() {
5666 let recorder = DebuggingRecorder::new();
5667 let runtime = tokio::runtime::Builder::new_current_thread()
5668 .enable_all()
5669 .build()
5670 .expect("embedded fixture runtime");
5671
5672 with_local_recorder(&recorder, || {
5673 runtime.block_on(async {
5674 for _ in 0..2 {
5677 let engine =
5678 Engine::start_with_embedded_config(EmbeddedEngineConfig::default())
5679 .await
5680 .expect("embedded startup");
5681 engine.shutdown().await.expect("embedded shutdown");
5682 }
5683 });
5684 });
5685
5686 let lifecycle_observations = recorder
5687 .snapshotter()
5688 .snapshot()
5689 .into_vec()
5690 .into_iter()
5691 .filter(|(key, _, _, _)| {
5692 key.key().name() == tokeira_observability::EMBEDDED_LIFECYCLE_OPERATIONS_TOTAL
5693 })
5694 .count();
5695 assert_eq!(lifecycle_observations, 2);
5696 }
5697
5698 #[tokio::test]
5699 async fn invalid_explicit_config_fails_before_startup() {
5700 let config = EmbeddedEngineConfig {
5701 startup_timeout_ms: 0,
5702 ..EmbeddedEngineConfig::default()
5703 };
5704 assert!(matches!(
5705 Engine::start_with_embedded_config(config).await,
5706 Err(EmbeddedEngineStartError::InvalidConfiguration(_))
5707 ));
5708 }
5709
5710 #[tokio::test]
5711 async fn unequal_home_counts_fail_before_descriptor_or_listener_work() {
5712 let descriptor_path = std::env::temp_dir().join(format!(
5713 "tokeira-home-count-preflight-{}.json",
5714 IncarnationId::new()
5715 ));
5716 let mut config = EmbeddedEngineConfig::default();
5717 config.server.infrastructure.placement.shard_count = 8;
5718 config.server.infrastructure.placement.bundle_count = 4;
5719 config.storage =
5720 EmbeddedStorageConfig::ManagedDsql(tokeira_config::ManagedEmbeddedDsqlConfig {
5721 intent: tokeira_config::ManagedClusterIntent::CreateOrRecover,
5722 descriptor_path: descriptor_path.clone(),
5723 region: "test-region".to_owned(),
5724 migration_policy: None,
5725 limits: tokeira_config::EmbeddedDsqlLimits::default(),
5726 tags: std::collections::BTreeMap::new(),
5727 });
5728 let error = Engine::start_with_embedded_config(config.clone())
5729 .await
5730 .expect_err("unequal counts must fail before any AWS work");
5731 assert!(matches!(
5732 error,
5733 EmbeddedEngineStartError::InvalidConfiguration(_)
5734 ));
5735 assert!(
5736 error
5737 .to_string()
5738 .contains("shard_count (8) must equal bundle_count (4)")
5739 );
5740 assert!(!descriptor_path.exists());
5741
5742 let address = "127.0.0.1:0".parse().expect("test address");
5743 let error = TokeiradHandle::start_in_memory_with_config(address, config.server)
5744 .await
5745 .expect_err("unequal counts must fail before binding a listener");
5746 assert!(format!("{error:#}").contains("shard_count (8) must equal bundle_count (4)"));
5747 }
5748
5749 #[tokio::test]
5750 async fn durable_snapshot_conflict_fails_before_descriptor_or_aws_work() {
5751 let descriptor_path = std::env::temp_dir().join(format!(
5752 "tokeira-managed-dsql-preflight-{}.json",
5753 IncarnationId::new()
5754 ));
5755 let mut config = EmbeddedEngineConfig::default();
5756 config.server.policy.snapshot = Some(SnapshotPolicyConfig {
5757 location: descriptor_path.with_extension("snapshot"),
5758 interval_ms: 1_000,
5759 });
5760 config.storage =
5761 EmbeddedStorageConfig::ManagedDsql(tokeira_config::ManagedEmbeddedDsqlConfig {
5762 intent: tokeira_config::ManagedClusterIntent::CreateOrRecover,
5763 descriptor_path: descriptor_path.clone(),
5764 region: "eu-west-2".to_owned(),
5765 migration_policy: None,
5766 limits: tokeira_config::EmbeddedDsqlLimits::default(),
5767 tags: std::collections::BTreeMap::new(),
5768 });
5769
5770 assert!(matches!(
5771 Engine::start_with_embedded_config(config).await,
5772 Err(EmbeddedEngineStartError::Phase {
5773 phase: EmbeddedStartupPhase::Configuration
5774 })
5775 ));
5776 assert!(!descriptor_path.exists());
5777 }
5778
5779 #[tokio::test]
5780 async fn drop_synchronously_closes_endpoint_admission() {
5781 let engine = Engine::start().await.expect("engine starts");
5782 let endpoint = engine.endpoint();
5783 drop(engine);
5784
5785 let error = endpoint
5786 .call(InProcessGrpcRequest {
5787 service: "temporal.api.workflowservice.v1.WorkflowService".to_owned(),
5788 rpc: "GetSystemInfo".to_owned(),
5789 headers: http::HeaderMap::new(),
5790 proto: Bytes::new(),
5791 })
5792 .await
5793 .expect_err("dropped engine rejects endpoint clones");
5794 assert_eq!(error.code(), tonic::Code::Unavailable);
5795 }
5796
5797 #[tokio::test]
5798 async fn snapshot_policy_over_dsql_storage_is_rejected() {
5799 let mut config = TokeiraConfig::default();
5800 config.infrastructure.storage = ConfigStorageKind::Dsql;
5801 config.policy.snapshot = Some(SnapshotPolicyConfig {
5802 location: std::path::PathBuf::from("unused.snapshot"),
5803 interval_ms: 60_000,
5804 });
5805 let error = build_and_serve(
5806 "127.0.0.1:0".parse().expect("loopback addr"),
5807 Arc::new(config),
5808 )
5809 .await
5810 .expect_err("snapshot policy over DSQL must be rejected before any I/O");
5811 let message = format!("{error:#}");
5812 assert!(
5813 message.contains("in-memory store's persistence mechanism"),
5814 "the rejection must name the mechanism: {message}"
5815 );
5816 }
5817
5818 #[tokio::test]
5821 async fn property_the_policy_setting_selects_the_check() {
5822 for policy in [
5823 None,
5824 Some(DsqlMigrationPolicy::ValidateOnly),
5825 Some(DsqlMigrationPolicy::Automatic),
5826 ] {
5827 let mut config = TokeiraConfig::default();
5828 config.infrastructure.storage = ConfigStorageKind::Dsql;
5829 config.infrastructure.dsql.migration_policy = policy;
5830 let error = build_and_serve(
5831 "127.0.0.1:0".parse().expect("loopback addr"),
5832 Arc::new(config),
5833 )
5834 .await
5835 .expect_err("no endpoint is configured");
5836 let message = format!("{error:#}");
5837 let refused_by_policy = message.contains("migration_policy");
5840 assert_eq!(
5841 refused_by_policy,
5842 policy == Some(DsqlMigrationPolicy::Automatic),
5843 "{policy:?}: {message}"
5844 );
5845 assert!(
5846 refused_by_policy || message.contains("infrastructure.dsql.endpoint must be set"),
5847 "{policy:?}: {message}"
5848 );
5849 }
5850 }
5851
5852 #[test]
5853 fn each_schema_refusal_says_what_was_found_and_what_to_do() {
5854 let refusal = |decision| {
5855 format!(
5856 "{:#}",
5857 distributed_schema_admission(decision).expect_err("the node refuses")
5858 )
5859 };
5860 let reject = |category| {
5861 SchemaDecision::Reject(tokeira_storage::dsql::SchemaIncompatibility {
5862 observed_version: Some(83),
5863 minimum_supported_version: 1,
5864 target_version: 82,
5865 maximum_readable_version: 82,
5866 category,
5867 })
5868 };
5869 assert_eq!(
5870 distributed_schema_admission(SchemaDecision::Compatible {
5871 current: 82,
5872 legacy_backfill: false,
5873 })
5874 .expect("a compatible schema starts"),
5875 82
5876 );
5877 let message = refusal(SchemaDecision::MigrationRequired {
5878 current: 0,
5879 target: 82,
5880 });
5881 assert!(
5882 message.contains("no Tokeira schema")
5883 && message.contains("V82")
5884 && message.contains("tkr schema setup"),
5885 "{message}"
5886 );
5887 let message = refusal(SchemaDecision::MigrationRequired {
5888 current: 81,
5889 target: 82,
5890 });
5891 assert!(
5892 message.contains("V81")
5893 && message.contains("V82")
5894 && message.contains("tkr schema setup"),
5895 "{message}"
5896 );
5897 let message = refusal(reject(SchemaIncompatibilityCategory::FutureSchema));
5898 assert!(
5899 message.contains("a later release migrated it") && message.contains("FutureSchema"),
5900 "{message}"
5901 );
5902 let message = refusal(reject(SchemaIncompatibilityCategory::BelowMinimum));
5903 assert!(
5904 message.contains("older than this release migrates from"),
5905 "{message}"
5906 );
5907 let message = refusal(reject(SchemaIncompatibilityCategory::ChecksumMismatch {
5908 version: 7,
5909 expected: "expected".to_owned(),
5910 observed: "observed".to_owned(),
5911 }));
5912 assert!(
5913 message.contains("migration ledger doesn't match")
5914 && message.contains("ChecksumMismatch"),
5915 "{message}"
5916 );
5917 for decision in [
5918 SchemaDecision::Initialize { target: 82 },
5919 SchemaDecision::Migrate { from: 81, to: 82 },
5920 ] {
5921 let message = refusal(decision);
5922 assert!(
5923 message.contains("validate-only schema assessment decided"),
5924 "{message}"
5925 );
5926 }
5927 }
5928
5929 proptest! {
5930 #![proptest_config(ProptestConfig::with_cases(256))]
5931
5932 #[test]
5935 fn property_a_node_starts_only_on_a_schema_its_release_runs_on(
5936 known in 2_u32..30,
5937 maximum_back in 0_u32..3,
5938 target_back in 0_u32..3,
5939 minimum_seed in 1_u32..30,
5940 current in 0_u32..34,
5941 tampered in proptest::option::of(0_u32..30),
5942 ) {
5943 use tokeira_storage::{
5944 dsql::{
5945 AppliedMigration, SchemaCompatibilityContract, SchemaObservation,
5946 assess_schema_compatibility,
5947 },
5948 schema_contract::{MigrationIdentity, migration_set_digest, sha256_hex},
5949 };
5950
5951 let maximum = known.saturating_sub(maximum_back).max(1);
5952 let target = maximum.saturating_sub(target_back).max(1);
5953 let minimum = minimum_seed.clamp(1, target);
5954 let recognized: Vec<MigrationIdentity> = (1..=known)
5955 .map(|version| MigrationIdentity {
5956 version,
5957 name: format!("migration_{version}"),
5958 checksum: sha256_hex(format!("sql-{version}").as_bytes()),
5959 })
5960 .collect();
5961 let contract = SchemaCompatibilityContract {
5962 tokeira_release: "this".to_owned(),
5963 minimum_supported_version: minimum,
5964 target_version: target,
5965 maximum_readable_version: maximum,
5966 migration_set_digest: migration_set_digest(&recognized, maximum)
5967 .expect("the readable prefix exists"),
5968 };
5969 let tampered = tampered.filter(|version| *version >= 1 && *version <= current.min(known));
5972 let applied_migrations = (1..=current)
5973 .map(|version| match recognized.get(version as usize - 1) {
5974 Some(identity) => AppliedMigration {
5975 version,
5976 name: identity.name.clone(),
5977 checksum: if tampered == Some(version) {
5978 sha256_hex(b"changed")
5979 } else {
5980 identity.checksum.clone()
5981 },
5982 },
5983 None => AppliedMigration {
5984 version,
5985 name: "later_release".to_owned(),
5986 checksum: sha256_hex(format!("later-{version}").as_bytes()),
5987 },
5988 })
5989 .collect();
5990 let decision = assess_schema_compatibility(
5991 &contract,
5992 &recognized,
5993 &SchemaObservation { applied_migrations, compatibility: None },
5994 SchemaMigrationPolicy::ValidateOnly,
5995 );
5996
5997 let runs = tampered.is_none() && target <= current && current <= maximum;
5998 match distributed_schema_admission(decision) {
5999 Ok(version) => {
6000 prop_assert!(runs, "admitted V{current}");
6001 prop_assert_eq!(version, current);
6002 }
6003 Err(error) => {
6004 prop_assert!(!runs, "refused V{current}");
6005 let message = format!("{error:#}");
6006 if tampered.is_none() && current < target && (current == 0 || current >= minimum) {
6007 prop_assert!(
6008 message.contains(&format!("V{target}")) && message.contains("tkr schema setup"),
6009 "{message}"
6010 );
6011 } else {
6012 prop_assert!(message.contains("schema is incompatible"), "{message}");
6013 }
6014 }
6015 }
6016 }
6017 }
6018
6019 #[tokio::test]
6023 async fn automatic_migration_policy_over_distributed_dsql_is_refused() {
6024 let mut config = TokeiraConfig::default();
6025 config.infrastructure.storage = ConfigStorageKind::Dsql;
6026 config.infrastructure.dsql.migration_policy = Some(DsqlMigrationPolicy::Automatic);
6027 let error = build_and_serve(
6028 "127.0.0.1:0".parse().expect("loopback addr"),
6029 Arc::new(config),
6030 )
6031 .await
6032 .expect_err("automatic must be refused before any I/O");
6033 let message = format!("{error:#}");
6034 assert!(
6035 message.contains("\"automatic\"") && message.contains("tkr schema setup"),
6036 "the refusal must name the policy and the way to migrate: {message}"
6037 );
6038 }
6039
6040 #[tokio::test(flavor = "multi_thread")]
6047 async fn sigterm_drains_persists_final_snapshot_and_next_boot_restores() {
6048 use tokeira_proto::workflowservice::{
6049 DescribeWorkflowExecutionRequest, StartWorkflowExecutionRequest,
6050 workflow_service_client::WorkflowServiceClient,
6051 };
6052
6053 let scratch =
6054 std::env::temp_dir().join(format!("tokeira-engine-sigterm-{}", std::process::id()));
6055 std::fs::create_dir_all(&scratch).expect("create scratch dir");
6056 let snapshot_location = scratch.join("tokeirad.snapshot");
6057
6058 let reserved = std::net::TcpListener::bind("127.0.0.1:0").expect("reserve port");
6062 let addr = reserved.local_addr().expect("reserved addr");
6063 drop(reserved);
6064
6065 let mut config = TokeiraConfig::default();
6066 config.infrastructure.network.grpc_addr = addr.to_string();
6067 config.policy.nexus_completion.http_addr = "127.0.0.1:0".to_string();
6070 config.policy.nexus_completion.system_callback_url = "http://127.0.0.1:0".to_string();
6071 config.policy.snapshot = Some(SnapshotPolicyConfig {
6072 location: snapshot_location.clone(),
6073 interval_ms: 3_600_000,
6076 });
6077 let config_path = scratch.join("tokeirad.toml");
6078 std::fs::write(&config_path, config.to_toml().expect("render config"))
6079 .expect("write config");
6080
6081 let mut daemon = tokio::spawn(run_from_cli(Cli {
6082 command: None,
6083 config: Some(config_path.display().to_string()),
6084 dump_config: false,
6085 version: false,
6086 verbose: false,
6087 json: false,
6088 }));
6089
6090 let target = format!("http://{addr}");
6095 let deadline = tokio::time::Instant::now() + std::time::Duration::from_secs(20);
6096 let mut client = loop {
6097 assert!(
6098 tokio::time::Instant::now() < deadline,
6099 "daemon must start serving"
6100 );
6101 tokio::select! {
6102 result = &mut daemon => {
6103 panic!("daemon exited before serving: {result:?}");
6104 }
6105 connected = WorkflowServiceClient::connect(target.clone()) => {
6106 match connected {
6107 Ok(client) => break client,
6108 Err(_) => {
6109 tokio::time::sleep(std::time::Duration::from_millis(50)).await;
6110 }
6111 }
6112 }
6113 }
6114 };
6115 client
6116 .start_workflow_execution(StartWorkflowExecutionRequest {
6117 namespace: "default".to_owned(),
6118 workflow_id: "sigterm-canary".to_owned(),
6119 workflow_type: Some(tokeira_proto::common::WorkflowType {
6120 name: "sigterm-workflow".to_owned(),
6121 }),
6122 task_queue: Some(tokeira_proto::taskqueue::TaskQueue {
6123 name: "sigterm-queue".to_owned(),
6124 ..Default::default()
6125 }),
6126 request_id: "sigterm-start".to_owned(),
6127 ..Default::default()
6128 })
6129 .await
6130 .expect("start canary workflow");
6131
6132 let delivered = std::process::Command::new("kill")
6135 .args(["-TERM", &std::process::id().to_string()])
6136 .status()
6137 .expect("send SIGTERM");
6138 assert!(delivered.success(), "kill -TERM must be deliverable");
6139
6140 daemon
6141 .await
6142 .expect("daemon task must not panic")
6143 .expect("SIGTERM drain must exit cleanly");
6144 assert!(
6145 snapshot_location.is_file(),
6146 "the drain must persist the final snapshot"
6147 );
6148
6149 let restored =
6151 TokeiradHandle::start_in_memory_with_config("127.0.0.1:0".parse().expect("addr"), {
6152 let mut config = TokeiraConfig::default();
6153 config.policy.snapshot = Some(SnapshotPolicyConfig {
6154 location: snapshot_location.clone(),
6155 interval_ms: 3_600_000,
6156 });
6157 config
6158 })
6159 .await
6160 .expect("restart over the final snapshot");
6161 let mut client =
6162 WorkflowServiceClient::connect(format!("http://{}", restored.bound_addr()))
6163 .await
6164 .expect("connect to restarted server");
6165 let described = client
6166 .describe_workflow_execution(DescribeWorkflowExecutionRequest {
6167 namespace: "default".to_owned(),
6168 execution: Some(tokeira_proto::common::WorkflowExecution {
6169 workflow_id: "sigterm-canary".to_owned(),
6170 run_id: String::new(),
6171 }),
6172 })
6173 .await
6174 .expect("restored workflow must be describable")
6175 .into_inner();
6176 assert!(
6177 described.workflow_execution_info.is_some(),
6178 "state must survive the SIGTERM drain via snapshot restore"
6179 );
6180 restored.shutdown().await.expect("clean restart shutdown");
6181 let _ = std::fs::remove_dir_all(&scratch);
6182 }
6183
6184 #[test]
6185 fn embedded_config_removes_daemon_storage_and_placement() {
6186 let mut config = TokeiraConfig::default();
6187 config.infrastructure.storage = ConfigStorageKind::Dsql;
6188 config.infrastructure.placement.controller_endpoint =
6189 Some("http://controller.internal:8080".to_owned());
6190
6191 let config = embedded_config(config).expect("embedded defaults are valid");
6192
6193 assert_eq!(config.infrastructure.storage, ConfigStorageKind::InMemory);
6194 assert!(
6195 config
6196 .infrastructure
6197 .placement
6198 .controller_endpoint
6199 .is_none()
6200 );
6201 }
6202
6203 #[tokio::test]
6204 async fn embedded_system_nexus_callback_never_falls_through_to_http() {
6205 let client = InProcessNexusCompletionClient::new();
6206 let target = system_callback_post_url(EMBEDDED_NEXUS_CALLBACK_BASE);
6207 let outcome = client
6208 .complete_operation(
6209 &target,
6210 "not-yet-attached",
6211 "operation-token",
6212 NexusCompletion::Succeeded(tokeira_types::Payloads::default()),
6213 &[],
6214 )
6215 .await
6216 .expect("the in-process preflight outcome is classifiable");
6217
6218 assert!(matches!(
6219 outcome,
6220 CompletionDeliveryOutcome::RetryableError { detail }
6221 if detail == "embedded Nexus completion runtime is unavailable"
6222 ));
6223 }
6224
6225 #[tokio::test]
6229 async fn bounded_body_caps_oversized_payload() {
6230 match collect_bounded_body(Full::new(Bytes::from(vec![0u8; 8])), 16).await {
6232 BoundedBody::Collected(bytes) => assert_eq!(bytes.len(), 8),
6233 other => panic!("expected Collected, got {other:?}"),
6234 }
6235 assert!(matches!(
6237 collect_bounded_body(Full::new(Bytes::from(vec![0u8; 17])), 16).await,
6238 BoundedBody::TooLarge
6239 ));
6240 }
6241
6242 #[test]
6243 fn with_loopback_port_rewrites_or_appends_port() {
6244 assert_eq!(
6246 with_loopback_port("http://127.0.0.1:7253", 51_000),
6247 "http://127.0.0.1:51000"
6248 );
6249 assert_eq!(
6251 with_loopback_port("http://127.0.0.1:7253/", 51_000),
6252 "http://127.0.0.1:51000"
6253 );
6254 assert_eq!(
6257 with_loopback_port("http://localhost", 51_000),
6258 "http://localhost:51000"
6259 );
6260 }
6261
6262 #[test]
6263 fn version_renderer_is_deterministic() {
6264 let short = render_build_info(false, false);
6265 let verbose = render_build_info(true, false);
6266 let json = render_build_info(false, true);
6267
6268 assert_eq!(short, render_build_info(false, false));
6269 assert_eq!(verbose, render_build_info(true, false));
6270 assert_eq!(json, render_build_info(false, true));
6271 assert!(short.contains(tokeira_build_info::SERVER_VERSION));
6272 assert!(short.contains(tokeira_build_info::TEMPORAL_PROTO_VERSION));
6273 assert!(short.contains(tokeira_build_info::TEMPORAL_SERVER_COMPAT));
6274 assert!(json.contains("server_version"));
6275 assert!(json.contains("temporal_proto_version"));
6276 }
6277
6278 #[test]
6279 fn reported_problem_search_attribute_has_exact_v131_keyword_list() {
6280 let mut search_attributes = tokeira_types::SearchAttributes::default();
6281 apply_reported_problem_search_attribute(
6282 &mut search_attributes,
6283 Some(WorkflowTaskReportedProblem {
6284 attempts_since_last_success: 5,
6285 problem: tokeira_kernel::WorkflowTaskProblem::Failed(
6286 tokeira_kernel::WorkflowTaskFailedCause::WorkflowWorkerUnhandledFailure,
6287 ),
6288 }),
6289 );
6290
6291 assert_eq!(
6292 search_attributes.0.get("TemporalReportedProblems"),
6293 Some(&SearchAttrValue::KeywordList(vec![
6294 "category=WorkflowTaskFailed".to_string(),
6295 "cause=WorkflowTaskFailedCauseWorkflowWorkerUnhandledFailure".to_string(),
6296 ]))
6297 );
6298 }
6299
6300 #[test]
6301 fn reported_problem_search_attribute_renders_timeout_category() {
6302 let mut search_attributes = tokeira_types::SearchAttributes::default();
6306 apply_reported_problem_search_attribute(
6307 &mut search_attributes,
6308 Some(WorkflowTaskReportedProblem {
6309 attempts_since_last_success: 5,
6310 problem: tokeira_kernel::WorkflowTaskProblem::TimedOutStartToClose,
6311 }),
6312 );
6313
6314 assert_eq!(
6315 search_attributes.0.get("TemporalReportedProblems"),
6316 Some(&SearchAttrValue::KeywordList(vec![
6317 "category=WorkflowTaskTimedOut".to_string(),
6318 "cause=WorkflowTaskTimedOutCauseStartToClose".to_string(),
6319 ]))
6320 );
6321 }
6322
6323 fn self_assigning_runtime(
6324 store: Arc<InMemoryStore>,
6325 shard_count: u32,
6326 ) -> Arc<TokeiraRuntime<InMemoryStore>> {
6327 Arc::new(TokeiraRuntime::new_with_nexus_and_shards_and_endpoint(
6328 store,
6329 2,
6330 tokeira_runtime::LaneConfig::default(),
6331 tokeira_runtime::TimerScannerConfig::default(),
6332 tokeira_runtime::WorkflowTimeoutScannerConfig::default(),
6333 tokeira_runtime::ActivityTimeoutScannerConfig::default(),
6334 tokeira_runtime::NexusTimeoutScannerConfig::default(),
6335 NexusEndpointRegistry::default(),
6336 Arc::new(tokeira_runtime::NoopNexusHttpClient),
6337 NexusCompletionDeps::default(),
6338 shard_count,
6339 IncarnationId::new().to_string(),
6340 "127.0.0.1:7233".to_owned(),
6341 false,
6342 None,
6343 ))
6344 }
6345
6346 fn boot_start_request(
6347 namespace_id: tokeira_types::NamespaceId,
6348 index: usize,
6349 ) -> tokeira_kernel::StartRequest {
6350 let run_id = tokeira_types::RunId::new();
6351 let workflow_id = tokeira_types::WorkflowId(format!("boot-{index}"));
6352 let now = time::OffsetDateTime::now_utc();
6353 tokeira_kernel::StartRequest {
6354 advice_policy: tokeira_kernel::ContinueAsNewAdvicePolicy::V1_31_0,
6355 initiator: None,
6356 run_key: tokeira_types::RunKey::derive(namespace_id, &workflow_id, run_id),
6357 namespace_id,
6358 workflow_id,
6359 run_id,
6360 workflow_type: tokeira_types::WorkflowType("boot".to_owned()),
6361 task_queue: tokeira_types::TaskQueueName("boot-queue".to_owned()),
6362 input: tokeira_types::Payloads::default(),
6363 header: None,
6364 memo: tokeira_types::Memo::default(),
6365 search_attributes: tokeira_types::SearchAttributes::default(),
6366 workflow_execution_timeout: None,
6367 workflow_run_timeout: None,
6368 workflow_task_timeout: time::Duration::seconds(10),
6369 retry_policy: None,
6370 conflict_policy: tokeira_kernel::WorkflowIdConflictPolicy::Fail,
6371 reuse_policy: tokeira_kernel::WorkflowIdReusePolicy::AllowDuplicate,
6372 deployment: None,
6373 build_id: None,
6374 versioning_override: None,
6375 workflow_start_delay: None,
6376 completion_callbacks: Vec::new(),
6377 user_metadata: None,
6378 links: Vec::new(),
6379 on_conflict_options: None,
6380 priority: None,
6381 attempt: 1,
6382 continued_execution_run_id: None,
6383 first_execution_run_id: None,
6384 parent_run_key: None,
6385 parent_workflow_id: None,
6386 parent_run_id: None,
6387 parent_namespace_id: None,
6388 parent_namespace_name: None,
6389 parent_initiated_event_id: 0,
6390 root_workflow_id: None,
6391 root_run_id: None,
6392 original_execution_run_id: Some(run_id),
6393 continued_failure: None,
6394 last_completion_result: None,
6395 first_run_started_at: None,
6396 request: tokeira_types::RequestContext {
6397 request_id: tokeira_types::RequestId(format!("boot-start-{index}")),
6398 caller_identity: None,
6399 principal: None,
6400 received_at: now,
6401 },
6402 now,
6403 client_cron_schedule: None,
6404 cron_schedule: None,
6405 eager_execution_accepted: false,
6406 reserved_poller_identity: None,
6407 inherited_versioning_info: None,
6408 }
6409 }
6410
6411 #[tokio::test]
6412 async fn self_assigned_dsql_boot_recovers_every_shard_before_admission() -> Result<()> {
6413 let shard_count = 2;
6418 let store = Arc::new(InMemoryStore::with_shard_count(shard_count));
6419 let endpoint = NodeEndpoint {
6420 host: "127.0.0.1".to_owned(),
6421 port: 7233,
6422 };
6423 let namespace_id = tokeira_types::NamespaceId::new();
6424
6425 let previous = self_assigning_runtime(Arc::clone(&store), shard_count);
6426 self_assign_dsql_shards(
6427 previous.as_ref(),
6428 store.as_ref(),
6429 shard_count,
6430 &IncarnationId::new(),
6431 &endpoint,
6432 true,
6433 )
6434 .await?;
6435 let mut started = Vec::new();
6436 let mut shards = std::collections::BTreeSet::new();
6437 for index in 0..8 {
6438 let request = boot_start_request(namespace_id, index);
6439 shards.insert(tokeira_types::execution_home_bundle(
6440 namespace_id.0.as_bytes(),
6441 request.workflow_id.0.as_bytes(),
6442 shard_count,
6443 ));
6444 started.push(request.run_key);
6445 let result = previous.start_workflow(request).await?;
6446 assert!(matches!(
6447 result,
6448 tokeira_storage::CommitResult::Applied { .. }
6449 ));
6450 }
6451 assert_eq!(shards.len(), 2, "the seeded runs span both shards");
6452
6453 let restarted = self_assigning_runtime(Arc::clone(&store), shard_count);
6454 let leases = self_assign_dsql_shards(
6455 restarted.as_ref(),
6456 store.as_ref(),
6457 shard_count,
6458 &IncarnationId::new(),
6459 &endpoint,
6460 true,
6461 )
6462 .await?;
6463 assert_eq!(leases.len(), 2);
6464 let mut active = restarted.active_shards();
6465 active.sort();
6466 assert_eq!(active, vec![ShardId(0), ShardId(1)]);
6467
6468 let queue = tokeira_types::QueueKey {
6469 namespace_id,
6470 task_queue: tokeira_types::TaskQueueName("boot-queue".to_owned()),
6471 task_kind: tokeira_types::TaskKind::Workflow,
6472 deployment: None,
6473 build_id: None,
6474 };
6475 let mut offered = Vec::new();
6476 while offered.len() < started.len() {
6477 let task = restarted
6478 .poll_workflow_task(
6479 queue.clone(),
6480 tokeira_types::WorkerIdentity("worker".to_owned()),
6481 tokio::time::Duration::from_secs(1),
6482 )
6483 .await?
6484 .expect("every pending first workflow task is offered again");
6485 offered.push(task.run_key);
6486 }
6487 offered.sort();
6488 started.sort();
6489 assert_eq!(offered, started);
6490 Ok(())
6491 }
6492
6493 #[tokio::test]
6494 async fn build_nexus_endpoint_store_resolves_worker_namespace_names() {
6495 let cache = InMemoryNamespaceCache::new();
6496 let namespace = ResolvedNamespace::active("payments");
6497 let namespace_id = namespace_id_for("payments");
6498 cache.insert(namespace).await.expect("namespace insert");
6499
6500 let store = build_nexus_endpoint_store(
6501 &cache,
6502 HashMap::from([(
6503 "payments-endpoint".to_string(),
6504 BootstrapNexusEndpointConfig {
6505 target: BootstrapNexusEndpointTarget::Worker {
6506 namespace_name: "payments".to_string(),
6507 task_queue: "nexus-q".to_string(),
6508 },
6509 },
6510 )]),
6511 )
6512 .await
6513 .expect("store should build");
6514
6515 let registry = NexusEndpointRegistry::new(store);
6516 let config = registry.resolve("payments-endpoint").expect("endpoint");
6517 assert_eq!(
6518 config,
6519 NexusEndpointConfig {
6520 target: EndpointTarget::Worker {
6521 namespace_id,
6522 task_queue: tokeira_types::TaskQueueName("nexus-q".to_string()),
6523 },
6524 }
6525 );
6526 }
6527
6528 #[tokio::test]
6529 async fn build_nexus_endpoint_store_rejects_unknown_worker_namespace() {
6530 let cache = InMemoryNamespaceCache::new();
6531
6532 let result = build_nexus_endpoint_store(
6533 &cache,
6534 HashMap::from([(
6535 "payments-endpoint".to_string(),
6536 BootstrapNexusEndpointConfig {
6537 target: BootstrapNexusEndpointTarget::Worker {
6538 namespace_name: "missing".to_string(),
6539 task_queue: "nexus-q".to_string(),
6540 },
6541 },
6542 )]),
6543 )
6544 .await;
6545
6546 assert!(result.is_err(), "missing namespace should fail");
6547 let error = result.expect_err("error");
6548
6549 assert!(error.to_string().contains("namespace `missing` not found"));
6550 }
6551
6552 #[test]
6553 fn placement_helpers_build_runtime_and_edge_config_from_server_config() {
6554 let mut config = TokeiraConfig::default();
6555 config.infrastructure.placement.controller_endpoint =
6556 Some("http://127.0.0.1:7240".to_string());
6557 config.infrastructure.placement.node_host = "10.0.0.9".to_string();
6558 config.infrastructure.placement.node_port = Some(8123);
6559 config.infrastructure.placement.shard_count = 8;
6560 config.infrastructure.placement.bundle_count = 4;
6561 config.infrastructure.placement.partition_count = 64;
6562 config.infrastructure.placement.hash_version = 2;
6563 let listen_addr = "127.0.0.1:7233".parse().unwrap();
6564 let node_id = IncarnationId::new();
6565 let endpoint = configured_node_endpoint(&config, listen_addr);
6566 let membership = membership_config(
6567 &config,
6568 node_id,
6569 endpoint.clone(),
6570 "http://127.0.0.1:7240".into(),
6571 );
6572
6573 assert_eq!(endpoint.as_authority(), "10.0.0.9:8123");
6574 assert_eq!(membership.node_id, node_id);
6575 assert_eq!(membership.node_endpoint, endpoint);
6576 assert_eq!(
6577 placement_config(&config),
6578 PlacementConfig {
6579 shard_count: 8,
6580 bundle_count: 4,
6581 partition_count: 64,
6582 hash_version: 2,
6583 }
6584 );
6585 }
6586
6587 #[test]
6588 fn dsql_pool_config_uses_effective_placement_values() {
6589 let mut config = TokeiraConfig::default();
6590 config.infrastructure.placement.shard_count = 32;
6591 config.infrastructure.placement.partition_count = 4;
6592
6593 let (pool_config, _ddb_client) =
6594 dsql_pool_config_with_client(&config, tokeira_storage::dsql::offline_ddb_client());
6595
6596 assert_eq!(pool_config.shard_count, 32);
6597 assert_eq!(pool_config.projection_partition_count, 4);
6598 }
6599
6600 #[test]
6601 fn wire_coverage_out_path_prefers_env_override_then_falls_back_to_default() {
6602 unsafe { std::env::remove_var(WIRE_COVERAGE_OUT_ENV) };
6608 assert_eq!(
6609 wire_coverage_out_path(),
6610 PathBuf::from(WIRE_COVERAGE_DEFAULT_OUT)
6611 );
6612
6613 unsafe { std::env::set_var(WIRE_COVERAGE_OUT_ENV, " /tmp/cov.json ") };
6616 assert_eq!(wire_coverage_out_path(), PathBuf::from("/tmp/cov.json"));
6617
6618 unsafe { std::env::set_var(WIRE_COVERAGE_OUT_ENV, " ") };
6621 assert_eq!(
6622 wire_coverage_out_path(),
6623 PathBuf::from(WIRE_COVERAGE_DEFAULT_OUT)
6624 );
6625
6626 unsafe { std::env::remove_var(WIRE_COVERAGE_OUT_ENV) };
6628 }
6629
6630 #[test]
6631 fn export_wire_coverage_writes_pretty_json_snapshot() {
6632 let recorder = WireCoverageRecorder::new();
6633 recorder.record(
6634 "/temporal.api.workflowservice.v1.WorkflowService/StartWorkflowExecution",
6635 0,
6636 );
6637
6638 let dir = std::env::temp_dir().join(format!("tokeirad-cov-{}", std::process::id()));
6639 std::fs::create_dir_all(&dir).expect("temp dir");
6640 let path = dir.join("wire-coverage.json");
6641
6642 export_wire_coverage(&recorder, &path).expect("export succeeds");
6643
6644 let written = std::fs::read_to_string(&path).expect("evidence file readable");
6645 assert!(written.contains("StartWorkflowExecution"));
6646 assert!(written.contains('\n'));
6648
6649 std::fs::remove_dir_all(&dir).ok();
6650 }
6651}