1#![deny(rust_2018_idioms)]
15#![allow(clippy::too_many_arguments)]
17
18use std::{
19 collections::HashMap,
20 convert::Infallible,
21 future::Future,
22 net::SocketAddr,
23 path::{Path, PathBuf},
24 pin::Pin,
25 sync::{
26 Arc, RwLock, Weak,
27 atomic::{AtomicU64, Ordering},
28 },
29 time::{Duration as StdDuration, Instant},
30};
31
32use anyhow::{Context, Result, anyhow};
33use async_trait::async_trait;
34use clap::Parser;
35use http_body_util::{BodyExt, Full, LengthLimitError, Limited};
36use hyper::{
37 Method, Request, Response, StatusCode,
38 body::{Bytes, Incoming},
39 service::service_fn,
40};
41use hyper_util::rt::TokioIo;
42use tokio::{
43 fs::{self, OpenOptions},
44 io::AsyncWriteExt as _,
45 net::TcpListener,
46 sync::{broadcast, mpsc, oneshot},
47 task::JoinHandle,
48};
49use tokio_stream::wrappers::TcpListenerStream;
50use tokio_util::sync::CancellationToken;
51use tonic::transport::Server;
52use tonic_web::GrpcWebLayer;
53use tower_http::cors::CorsLayer;
54use tracing::{error, info};
55
56#[cfg(feature = "chasm-extensions")]
57pub mod chasm;
58mod chasm_extensions;
59#[cfg(feature = "chasm-extensions")]
60pub use chasm_extensions::EngineBuilder;
61use chasm_extensions::{
62 ChasmExtensions, check_rebuilt_outboxes, check_registered_archetypes,
63 declared_search_attributes, embedded_stack_error, registry_start_error,
64 seed_declared_search_attributes, seed_start_error,
65};
66
67pub mod correlation_format;
68#[cfg(all(test, feature = "dsql-integration"))]
69mod distributed_schema_live_tests;
70#[cfg(test)]
71mod embedded_dsql_tests;
72#[doc(hidden)]
73pub mod harness;
74mod http_api_transport;
75mod listener;
76mod nexus_http_transport;
77pub mod observability;
78
79pub use listener::{EngineListenError, EngineListener, EngineListenerShutdownError};
80
81use http_api_transport::HttpApiLayer;
82use nexus_http_transport::NexusHttpLayer;
83use tokeira_auth::{
84 Authorizer, ClaimMapper, DefaultAuthorizer, GrantRule, GrantRules, JwksKeyProvider,
85 JwtAuthenticator, JwtIssuerProfile, MultiSourceClaimMapper, StsAuthenticator, WorkerScope,
86 WorkerScopeRule, WorkerScopeRules,
87};
88use tokeira_chasm::Library as _;
89use tokeira_config::{Cli, ConfigStorageKind};
90pub use tokeira_config::{
91 DsqlMigrationPolicy, EmbeddedEngineConfig, EmbeddedStorageConfig, SnapshotPolicyConfig,
92 TokeiraConfig,
93};
94use tokeira_edge::{
95 Authenticator, CacheBackedRouter, CallbackResponse, EdgeInterceptors, EdgeRoutingConfig,
96 HistoryNotifyingRepository, HistoryWaitRegistry, HttpApiCatalog, HttpApiPolicy,
97 InMemoryNamespaceCache, InMemoryOperatorApi, InProcessGrpcService, LocalOnlyRouter,
98 LongPollConfig, LongPollGate, NamespaceCache, OperatorService, PendingQueryStore,
99 PollerRegistry, ResolvedNamespace, RoutingCache, ScopedWorkerSessionRegistry,
100 WorkerComputeNamespaceCatalogAdapter, WorkflowExecutionDescription, WorkflowService,
101 conformance::{WireCoverageLayer, WireCoverageRecorder},
102 grpc::{
103 admin_service::AdminServiceGrpc, operator_service::OperatorServiceGrpc,
104 runtime_adapter::RuntimeAdapter, workflow_service::WorkflowServiceGrpc,
105 },
106 handle_nexus_callback,
107 nexus_http::MAX_NEXUS_PAYLOAD_BYTES,
108 operator_service::{ClusterInfo, OperatorApi, SearchAttributeDefinition},
109 run_routing_subscription,
110 translate::to_internal::namespace_id_for,
111 workflow_service::{ExecutionResolver, WorkflowRuntimeApi},
112};
113pub use tokeira_edge::{InProcessGrpcRequest, InProcessGrpcResponse};
114
115struct AuthorizationStack {
116 grpc: Arc<dyn Authenticator>,
117 nexus: Arc<dyn tokeira_edge::nexus_http::NexusHttpAuthorizer>,
118 principal_attribution: bool,
119}
120
121async fn build_authorization_stack(config: &TokeiraConfig) -> Result<AuthorizationStack> {
122 let Some(authorization) = config
123 .policy
124 .authorization
125 .as_ref()
126 .filter(|authorization| authorization.has_identity_source())
127 else {
128 let grpc = harness::installed()
129 .and_then(|hooks| hooks.fallback_grpc_authenticator.clone())
130 .unwrap_or_else(|| Arc::new(tokeira_edge::AllowAllAuthenticator));
131 let stack = AuthorizationStack {
132 grpc,
133 nexus: Arc::new(tokeira_edge::nexus_http::PermissiveNexusHttpAuthorizer),
134 principal_attribution: false,
135 };
136 return Ok(stack);
137 };
138
139 let mut profiles = Vec::with_capacity(authorization.jwt.issuers.len());
140 for issuer in &authorization.jwt.issuers {
141 let grants = GrantRules::new(
142 issuer
143 .grants
144 .iter()
145 .map(|rule| {
146 GrantRule::new(
147 rule.match_sub.clone(),
148 rule.grant.iter().map(String::as_str),
149 )
150 .with_context(|| format!("invalid JWT grants for issuer {}", issuer.name))
151 })
152 .collect::<Result<Vec<_>>>()?,
153 );
154 let worker_scopes = WorkerScopeRules::new(
155 issuer
156 .worker_scopes
157 .iter()
158 .map(|rule| {
159 let scope = WorkerScope::try_new(
160 rule.namespace.clone(),
161 rule.task_queues.clone(),
162 rule.deployment_name.clone(),
163 rule.build_id.clone(),
164 )
165 .with_context(|| {
166 format!("invalid JWT Worker scope for issuer {}", issuer.name)
167 })?;
168 WorkerScopeRule::new(rule.match_sub.clone(), scope).with_context(|| {
169 format!(
170 "invalid JWT Worker-scope subject for issuer {}",
171 issuer.name
172 )
173 })
174 })
175 .collect::<Result<Vec<_>>>()?,
176 );
177 let keys =
178 JwksKeyProvider::start(issuer.jwks_uri.clone(), issuer.refresh_interval_duration())
179 .await;
180 profiles.push(
181 JwtIssuerProfile::new(
182 issuer.name.clone(),
183 issuer.issuer.clone(),
184 issuer.audience.clone(),
185 issuer.permissions_claim.clone(),
186 grants,
187 keys,
188 )
189 .with_worker_scopes(worker_scopes),
190 );
191 }
192 let jwt = (!profiles.is_empty()).then(|| JwtAuthenticator::new(profiles));
193 let sts = authorization
194 .aws_iam
195 .as_ref()
196 .map(|aws_iam| {
197 let grants = aws_iam
198 .grants
199 .iter()
200 .map(|rule| {
201 GrantRule::new(
202 rule.match_arn.clone(),
203 rule.grant.iter().map(String::as_str),
204 )
205 .context("invalid AWS IAM grants")
206 })
207 .collect::<Result<Vec<_>>>()
208 .map(GrantRules::new)?;
209 let worker_scopes = WorkerScopeRules::new(
210 aws_iam
211 .worker_scopes
212 .iter()
213 .map(|rule| {
214 let scope = WorkerScope::try_new(
215 rule.namespace.clone(),
216 rule.task_queues.clone(),
217 rule.deployment_name.clone(),
218 rule.build_id.clone(),
219 )
220 .context("invalid AWS IAM Worker scope")?;
221 WorkerScopeRule::new(rule.match_arn.clone(), scope)
222 .context("invalid AWS IAM Worker-scope ARN")
223 })
224 .collect::<Result<Vec<_>>>()?,
225 );
226 Ok::<_, anyhow::Error>(StsAuthenticator::new(grants).with_worker_scopes(worker_scopes))
227 })
228 .transpose()?;
229 let mapper: Arc<dyn ClaimMapper> = Arc::new(MultiSourceClaimMapper::new(jwt, sts));
230 let authorizer: Arc<dyn Authorizer> = Arc::new(DefaultAuthorizer);
231 Ok(AuthorizationStack {
232 grpc: Arc::new(tokeira_edge::PolicyAuthenticator::new(
233 mapper.clone(),
234 authorizer.clone(),
235 authorization.expose_authorizer_errors,
236 )),
237 nexus: Arc::new(tokeira_edge::nexus_http::PolicyNexusHttpAuthorizer::new(
238 mapper,
239 authorizer,
240 authorization.expose_authorizer_errors,
241 )),
242 principal_attribution: authorization.principal_attribution,
243 })
244}
245use tokeira_kernel::{Link, LoadedRun};
246use tokeira_managed_dsql::{
247 AwsDsqlControlPlane, CanonicalClusterIdentity, ClusterAction, CreateOrRecoverRequest,
248 DsqlControlPlane, LifecycleEnvironment, LocalClusterDescriptorStore, ManagedDsqlLifecycle,
249 Readiness, ResolvedCluster, StartupDeadline, SystemLifecycleEnvironment,
250};
251use tokeira_observability::{
252 ClusterStatusLabel, DbClassLabel, EmbeddedOperationLabel, EmbeddedStorageModeLabel,
253 ErrorClassLabel, OwnershipOutcomeLabel, SchemaOutcomeLabel, record_embedded_lifecycle,
254};
255use tokeira_projection::{
256 DsqlVisibilityStore, InMemoryVisibilityStore, ProjectionSink, ProjectionWorker, SearchAttrType,
257 VisibilityQueryService, VisibilitySink, VisibilityStore,
258};
259use tokeira_runtime::{
260 ChannelDemandObservationSink, ChannelWorkerComputeReconcileSink,
261 CompletionCallbackScannerConfig, CompletionDeliveryOutcome, ConnectionBudgetApplier,
262 HttpNexusClient, HttpNexusCompletionClient, InMemoryNexusEndpointStore, MembershipConfig,
263 NEXUS_CALLBACK_PATH, NEXUS_OPERATION_STATE_HEADER, NEXUS_OPERATION_TOKEN_HEADER,
264 NexusCompletion, NexusCompletionClient, NexusCompletionDeps, NexusCompletionRuntimeConfig,
265 NexusEndpointRegistry, NexusEndpointSpec, NexusEndpointSpecTarget, NexusEndpointStore,
266 NexusNamespaceResolver, NexusWorkerComputeProvider, OBSERVATION_CHANNEL_CAPACITY,
267 RECONCILE_CHANNEL_CAPACITY, RepositoryBackedTaskQueueConfigStore, RuntimeConfig,
268 RuntimeShutdownHandle, ScheduleEngineConfig, ScheduleStore, SweepResult,
269 SystemWorkerComputeClock, TEMPORAL_CALLBACK_TOKEN_HEADER, TokeiraRuntime,
270 WorkerComputeControllerService, WorkerComputeOutbox, WorkerComputeReconciler,
271 WorkflowTaskReportedProblem, nexus_payload_to_body, reported_problem_from_state,
272 run_schedule_engine, system_callback_post_url,
273};
274use tokeira_storage::{
275 ConnectionDirector, DbClass, InMemoryStore, InMemoryWorkerComputeRepository, LeaseOutcome,
276 LeaseRepository, ProjectionLog, RunRepository, TaskQueueConfigRepository,
277 WorkerComputeRepository, WorkerDeploymentRepository, WorkerTaskProvenanceStore,
278 dsql::{
279 ConnectionControlLeaseRepository, ControlLeaseAcquireOutcome, ControlLeaseAcquireRequest,
280 ControlLeaseClusterIdentity, ControlLeaseError, ControlLeaseGuard, DsqlAuthConfig,
281 DsqlConnectionDirector, DsqlCoordinationConfig, DsqlPoolConfig, DsqlStore,
282 EmbeddedDsqlPoolConfig, MigrationRunner, OWNER_ADMISSION_MARGIN, OWNER_LEASE_DURATION,
283 OWNER_RENEW_INTERVAL, OwnershipAdmissionGate, OwnershipAdmissionState,
284 SCHEMA_MIGRATION_ADMISSION_MARGIN, SCHEMA_MIGRATION_LEASE_DURATION,
285 SchemaCompatibilityError, SchemaDecision, SchemaIncompatibilityCategory,
286 SchemaMigrationPolicy, WarmupDeadline,
287 },
288};
289use tokeira_types::{
290 ExecutionRef, IncarnationId, NamespaceId, NodeEndpoint, PlacementConfig, ProjectionCursor,
291 SearchAttrValue, ShardEpoch, ShardId, WorkflowId,
292};
293
294#[allow(dead_code)]
298#[derive(Clone, Debug)]
299pub enum BootstrapNexusEndpointTarget {
300 External {
301 address: String,
302 },
303 Worker {
304 namespace_name: String,
305 task_queue: String,
306 },
307}
308
309#[derive(Debug)]
310struct VisibilityRegistryOperatorApi<V> {
311 inner: InMemoryOperatorApi,
312 visibility_store: V,
313 declared_search_attributes: Vec<(String, SearchAttrType)>,
314}
315
316impl<V> VisibilityRegistryOperatorApi<V> {
317 fn new(inner: InMemoryOperatorApi, visibility_store: V) -> Self {
318 Self {
319 inner,
320 visibility_store,
321 declared_search_attributes: Vec::new(),
322 }
323 }
324
325 fn with_declared_search_attributes(mut self, declared: Vec<(String, SearchAttrType)>) -> Self {
326 self.declared_search_attributes = declared;
327 self
328 }
329}
330
331#[async_trait]
332impl<V> OperatorApi for VisibilityRegistryOperatorApi<V>
333where
334 V: VisibilityStore + Clone + 'static,
335{
336 async fn cluster_info(&self) -> Result<ClusterInfo> {
337 self.inner.cluster_info().await
338 }
339
340 async fn list_search_attributes(
341 &self,
342 namespace: Option<&str>,
343 ) -> Result<Vec<SearchAttributeDefinition>> {
344 self.inner.list_search_attributes(namespace).await
345 }
346
347 async fn upsert_search_attribute(
348 &self,
349 namespace: &str,
350 attr: SearchAttributeDefinition,
351 ) -> Result<()> {
352 let attr_type = visibility_search_attr_type(&attr.attr_type)?;
353 self.visibility_store
358 .register_attr(namespace_id_for(namespace), attr.name.clone(), attr_type)
359 .await?;
360 self.inner.upsert_search_attribute(namespace, attr).await
361 }
362
363 async fn remove_search_attribute(&self, namespace: &str, attr_name: &str) -> Result<()> {
364 self.inner
365 .remove_search_attribute(namespace, attr_name)
366 .await
367 }
368
369 async fn seed_predefined_search_attributes(&self, namespace: &str) -> Result<()> {
375 tokeira_projection::seed_predefined_search_attributes(
376 &self.visibility_store,
377 namespace_id_for(namespace),
378 )
379 .await?;
380 seed_declared_search_attributes(
384 &self.visibility_store,
385 namespace_id_for(namespace),
386 &self.declared_search_attributes,
387 )
388 .await?;
389 Ok(())
390 }
391}
392
393struct CacheNexusNamespaceResolver {
404 namespaces: Arc<dyn NamespaceCache>,
405}
406
407#[async_trait]
408impl NexusNamespaceResolver for CacheNexusNamespaceResolver {
409 async fn name_for_id(&self, namespace_id: NamespaceId) -> Option<String> {
410 self.namespaces
411 .list_all()
412 .await
413 .ok()?
414 .into_iter()
415 .map(|namespace| namespace.name)
416 .find(|name| namespace_id_for(name) == namespace_id)
417 }
418}
419
420fn visibility_search_attr_type(value: &str) -> Result<SearchAttrType> {
421 match value {
422 "keyword" => Ok(SearchAttrType::Keyword),
423 "keyword_list" => Ok(SearchAttrType::KeywordList),
424 "int" => Ok(SearchAttrType::Int),
425 "bool" => Ok(SearchAttrType::Bool),
426 "double" => Ok(SearchAttrType::Double),
427 "datetime" => Ok(SearchAttrType::Datetime),
428 "text" => Ok(SearchAttrType::Text),
429 other => Err(anyhow!("unsupported search attribute type `{other}`")),
430 }
431}
432
433#[allow(dead_code)]
436#[derive(Clone, Debug)]
437pub struct BootstrapNexusEndpointConfig {
438 pub target: BootstrapNexusEndpointTarget,
439}
440
441#[derive(Debug)]
452pub struct Engine {
453 #[cfg_attr(not(feature = "chasm-extensions"), allow(dead_code))]
455 component_visibility_store: chasm_extensions::VisibilityQueryStore,
456 #[cfg_attr(not(feature = "chasm-extensions"), allow(dead_code))]
458 chasm_engine: Arc<tokeira_runtime::chasm::ChasmEngine>,
459 #[cfg_attr(not(feature = "chasm-extensions"), allow(dead_code))]
460 registry: Arc<tokeira_chasm::Registry>,
461 endpoint: TemporalEndpoint,
462 background_cancel: CancellationToken,
463 log_broadcast: broadcast::Sender<LogEvent>,
464 snapshot_policy: Option<EngineSnapshotPolicy>,
465 recovery_task: Option<JoinHandle<Result<()>>>,
466 startup_report: EngineStartupReport,
467 shutdown_coordinator: Option<EmbeddedShutdownCoordinator>,
468 listeners: listener::ListenerRegistry,
469}
470
471#[derive(Clone, Copy, Debug, PartialEq, Eq)]
473pub enum EmbeddedStorageMode {
474 InMemory,
476 ManagedDsql,
478 ExistingDsql,
480}
481
482#[derive(Clone, Debug, PartialEq, Eq)]
484pub struct EngineStartupReport {
485 pub storage_mode: EmbeddedStorageMode,
487 pub cluster: Option<ClusterStartupReport>,
489 pub schema: Option<SchemaStartupReport>,
491 pub ownership: Option<OwnershipStartupReport>,
493}
494
495impl EngineStartupReport {
496 fn in_memory() -> Self {
497 Self {
498 storage_mode: EmbeddedStorageMode::InMemory,
499 cluster: None,
500 schema: None,
501 ownership: None,
502 }
503 }
504}
505
506#[derive(Clone, Debug, PartialEq, Eq)]
508pub struct ClusterStartupReport {
509 pub region: String,
511 pub cluster_id: String,
513 pub cluster_arn: String,
515 pub endpoint: String,
521 pub action: ClusterAction,
523}
524
525#[derive(Clone, Copy, Debug, PartialEq, Eq)]
527pub enum SchemaStartupOutcome {
528 Compatible,
530 Initialized,
532 Migrated,
534 MetadataBackfilled,
536}
537
538#[derive(Clone, Debug, PartialEq, Eq)]
540pub struct SchemaStartupReport {
541 pub observed_version: u32,
543 pub minimum_supported_version: u32,
545 pub target_version: u32,
547 pub maximum_readable_version: u32,
549 pub migration_set_digest: String,
551 pub outcome: SchemaStartupOutcome,
553}
554
555#[derive(Clone, Copy, Debug, PartialEq, Eq)]
557pub struct OwnershipStartupReport {
558 pub outcome: ControlLeaseAcquireOutcome,
560 pub fence_token: i64,
562}
563
564#[derive(Clone, Copy, Debug, PartialEq, Eq)]
566pub enum EmbeddedStartupPhase {
567 Configuration,
569 ClusterResolution,
571 ConnectionWarmup,
573 Schema,
575 Ownership,
577 RuntimeRestore,
579}
580
581impl std::fmt::Display for EmbeddedStartupPhase {
582 fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
583 formatter.write_str(match self {
584 Self::Configuration => "configuration validation",
585 Self::ClusterResolution => "cluster resolution",
586 Self::ConnectionWarmup => "DSQL connection warmup",
587 Self::Schema => "schema compatibility",
588 Self::Ownership => "embedded ownership",
589 Self::RuntimeRestore => "runtime restoration",
590 })
591 }
592}
593
594#[derive(Debug)]
596pub enum EmbeddedEngineStartError {
597 Registry(tokeira_chasm::ChasmError),
599 UnregisteredArchetype {
601 archetype_id: u32,
603 executions: u64,
605 },
606 SearchAttributeType {
608 namespace: NamespaceId,
610 key: String,
612 declared: SearchAttrType,
614 existing: SearchAttrType,
616 },
617 UnserviceableOutbox {
619 archetype_id: u32,
621 cause: tokeira_runtime::chasm::RebuildFailure,
623 executions: usize,
625 },
626 InvalidConfiguration(tokeira_config::EmbeddedConfigError),
628 Phase {
630 phase: EmbeddedStartupPhase,
632 },
633 DeadlineExceeded {
635 phase: EmbeddedStartupPhase,
637 },
638}
639
640impl std::fmt::Display for EmbeddedEngineStartError {
641 fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
642 match self {
643 Self::Registry(error) => write!(
644 formatter,
645 "CHASM registration failed: {error}; correct the conflicting library or executor before restarting"
646 ),
647 Self::UnregisteredArchetype {
648 archetype_id,
649 executions,
650 } => write!(
651 formatter,
652 "CHASM archetype {archetype_id} is not registered but has {executions} stored executions; load its library before restarting"
653 ),
654 Self::SearchAttributeType {
655 namespace,
656 key,
657 declared,
658 existing,
659 } => write!(
660 formatter,
661 "namespace {} search attribute `{key}` declares {declared:?} but storage has {existing:?}; restore the matching declaration or correct the registered type before restarting",
662 namespace.0
663 ),
664 Self::UnserviceableOutbox {
665 archetype_id,
666 cause,
667 executions,
668 } => write!(
669 formatter,
670 "CHASM rebuild cannot serve {executions} executions; first failure in archetype {archetype_id}: {cause}; restore the matching library or repair its persisted data before restarting"
671 ),
672 Self::InvalidConfiguration(error) => {
673 write!(formatter, "invalid embedded engine configuration: {error}")
674 }
675 Self::Phase { phase } => {
676 write!(formatter, "embedded engine startup failed during {phase}")
677 }
678 Self::DeadlineExceeded { phase } => {
679 write!(
680 formatter,
681 "embedded engine startup deadline exceeded during {phase}"
682 )
683 }
684 }
685 }
686}
687
688impl std::error::Error for EmbeddedEngineStartError {}
689
690impl From<tokeira_config::EmbeddedConfigError> for EmbeddedEngineStartError {
691 fn from(error: tokeira_config::EmbeddedConfigError) -> Self {
692 Self::InvalidConfiguration(error)
693 }
694}
695
696#[derive(Clone, Copy, Debug, PartialEq, Eq)]
698pub enum EmbeddedShutdownFailure {
699 RpcDrain,
701 RuntimeTasks,
703 ShardRelease,
705 OwnershipRelease,
707 Storage,
709 Snapshot,
711 ListenerDrain,
713}
714
715#[derive(Debug)]
717pub struct EmbeddedEngineShutdownError {
718 failures: Vec<EmbeddedShutdownFailure>,
719}
720
721impl std::fmt::Display for EmbeddedEngineShutdownError {
722 fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
723 write!(
724 formatter,
725 "embedded engine shutdown completed with {} cleanup failure(s)",
726 self.failures.len()
727 )
728 }
729}
730
731impl std::error::Error for EmbeddedEngineShutdownError {}
732
733impl EmbeddedEngineShutdownError {
734 pub fn failures(&self) -> &[EmbeddedShutdownFailure] {
736 &self.failures
737 }
738}
739
740type CleanupFuture = Pin<Box<dyn Future<Output = Result<()>> + Send>>;
741type CleanupAction = Box<dyn FnOnce(Instant) -> CleanupFuture + Send>;
742
743#[derive(Clone)]
744struct EmbeddedOwnership {
745 repository: ConnectionControlLeaseRepository,
746 guard: Arc<tokio::sync::Mutex<Option<ControlLeaseGuard>>>,
747 gate: OwnershipAdmissionGate,
748}
749
750impl std::fmt::Debug for EmbeddedOwnership {
751 fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
752 formatter
753 .debug_struct("EmbeddedOwnership")
754 .field("gate", &self.gate)
755 .finish_non_exhaustive()
756 }
757}
758
759struct EmbeddedShutdownCoordinator {
760 service: InProcessGrpcService,
761 runtime_tasks: RuntimeShutdownHandle,
762 engine_tasks: RuntimeShutdownHandle,
763 shard_cleanup: Option<CleanupAction>,
764 director: Option<Arc<DsqlConnectionDirector>>,
765 ownership: Option<EmbeddedOwnership>,
766}
767
768impl std::fmt::Debug for EmbeddedShutdownCoordinator {
769 fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
770 formatter
771 .debug_struct("EmbeddedShutdownCoordinator")
772 .field("service", &self.service)
773 .field("runtime_tasks", &self.runtime_tasks)
774 .field("engine_tasks", &self.engine_tasks)
775 .field("has_shard_cleanup", &self.shard_cleanup.is_some())
776 .field("has_director", &self.director.is_some())
777 .field("ownership", &self.ownership)
778 .finish()
779 }
780}
781
782#[derive(Debug)]
783struct EngineSnapshotPolicy {
784 store: InMemoryStore,
785 location: PathBuf,
786 periodic_task: Option<JoinHandle<()>>,
787}
788
789static SNAPSHOT_TEMP_SEQUENCE: AtomicU64 = AtomicU64::new(0);
790
791impl EngineSnapshotPolicy {
792 fn start(
793 store: InMemoryStore,
794 config: SnapshotPolicyConfig,
795 cancel: CancellationToken,
796 ) -> Self {
797 let interval_duration = std::time::Duration::from_millis(config.interval_ms);
798 let location = config.location;
799 let periodic_store = store.clone();
800 let periodic_location = location.clone();
801 let first_tick = tokio::time::Instant::now() + interval_duration;
805 let periodic_task = tokio::spawn(async move {
806 let mut interval = tokio::time::interval_at(first_tick, interval_duration);
811 interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
812 loop {
813 tokio::select! {
814 biased;
815 _ = cancel.cancelled() => break,
816 _ = interval.tick() => {
817 if let Err(error) = persist_snapshot(&periodic_store, &periodic_location).await {
818 error!(
819 snapshot_path = %periodic_location.display(),
820 error = %error,
821 "failed to persist periodic engine snapshot"
822 );
823 }
824 }
825 }
826 }
827 });
828 Self {
829 store,
830 location,
831 periodic_task: Some(periodic_task),
832 }
833 }
834}
835
836#[derive(Clone, Debug)]
843pub struct TemporalEndpoint {
844 service: InProcessGrpcService,
845 shutdown: CancellationToken,
846}
847
848impl Engine {
849 #[cfg(feature = "chasm-extensions")]
855 pub fn chasm_visibility<C>(
856 &self,
857 namespace_id: tokeira_types::NamespaceId,
858 ) -> Result<tokeira_projection::ComponentVisibility, tokeira_chasm::ChasmError>
859 where
860 C: tokeira_chasm::EngineComponent
861 + tokeira_chasm::RootComponent
862 + tokeira_chasm::SearchAttributeProvider
863 + tokeira_chasm::VisibilityContributor,
864 {
865 let archetype_id = self.registry.archetype_id(C::FQN).ok_or_else(|| {
866 tokeira_chasm::ChasmError::Internal(format!("archetype `{}` is not registered", C::FQN))
867 })?;
868 Ok(tokeira_projection::ComponentVisibility::new(
869 self.component_visibility_store.0.clone(),
870 namespace_id,
871 tokeira_types::ArchetypeId(archetype_id),
872 ))
873 }
874 #[cfg(feature = "chasm-extensions")]
876 pub fn builder(config: EmbeddedEngineConfig) -> EngineBuilder {
877 EngineBuilder {
878 config,
879 extensions: ChasmExtensions::default(),
880 }
881 }
882
883 #[cfg(feature = "chasm-extensions")]
886 pub fn chasm<C>(
887 &self,
888 ) -> Result<tokeira_runtime::chasm::TypedEngine<C>, tokeira_chasm::ChasmError>
889 where
890 C: tokeira_chasm::EngineComponent
891 + tokeira_chasm::RootComponent
892 + tokeira_chasm::SearchAttributeProvider
893 + tokeira_chasm::VisibilityContributor,
894 {
895 if self.registry.archetype_id(C::FQN).is_none() {
896 return Err(tokeira_chasm::ChasmError::Internal(format!(
897 "archetype `{}` is not registered",
898 C::FQN
899 )));
900 }
901 Ok(tokeira_runtime::chasm::TypedEngine::new(
902 self.chasm_engine.clone(),
903 ))
904 }
905
906 pub async fn start() -> Result<Self> {
908 Self::start_with_config(TokeiraConfig::default()).await
909 }
910
911 pub async fn embedded() -> Result<Self> {
913 Self::start().await
914 }
915
916 pub async fn start_with_config(config: TokeiraConfig) -> Result<Self> {
922 Self::start_in_memory_with_extensions(config, ChasmExtensions::default()).await
923 }
924
925 async fn start_in_memory_with_extensions(
926 config: TokeiraConfig,
927 extensions: ChasmExtensions,
928 ) -> Result<Self> {
929 let config = embedded_config(config)?;
930 let snapshot_config = config.policy.snapshot.clone();
931 let (store, restored) = restore_snapshot_store(&config).await?;
932 let stack = build_embedded(Arc::new(config), store.clone(), restored, extensions).await?;
933 let snapshot_policy = snapshot_config.map(|config| {
934 EngineSnapshotPolicy::start(store, config, stack.background_cancel.clone())
935 });
936 stack.engine_tasks.close_registration();
937 let shutdown_coordinator = EmbeddedShutdownCoordinator {
938 service: stack.service.clone(),
939 runtime_tasks: stack.runtime_tasks,
940 engine_tasks: stack.engine_tasks,
941 shard_cleanup: stack.shard_cleanup,
942 director: None,
943 ownership: None,
944 };
945 let engine = Self {
946 component_visibility_store: stack.component_visibility_store,
947 chasm_engine: stack.chasm_engine,
948 registry: stack.registry,
949 endpoint: TemporalEndpoint {
950 service: stack.service,
951 shutdown: stack.background_cancel.clone(),
952 },
953 background_cancel: stack.background_cancel,
954 log_broadcast: stack.log_broadcast,
955 snapshot_policy,
956 recovery_task: stack.recovery_task,
957 startup_report: EngineStartupReport::in_memory(),
958 shutdown_coordinator: Some(shutdown_coordinator),
959 listeners: listener::ListenerRegistry::default(),
960 };
961 record_embedded_startup(&engine.startup_report);
962 Ok(engine)
963 }
964
965 pub async fn start_with_embedded_config(
971 config: EmbeddedEngineConfig,
972 ) -> Result<Self, EmbeddedEngineStartError> {
973 Self::start_with_extensions(config, ChasmExtensions::default()).await
974 }
975
976 async fn start_with_extensions(
977 config: EmbeddedEngineConfig,
978 extensions: ChasmExtensions,
979 ) -> Result<Self, EmbeddedEngineStartError> {
980 if !matches!(config.storage, EmbeddedStorageConfig::InMemory)
981 && config.server.policy.snapshot.is_some()
982 {
983 return Err(EmbeddedEngineStartError::Phase {
984 phase: EmbeddedStartupPhase::Configuration,
985 });
986 }
987 config.validate()?;
988 if matches!(config.storage, EmbeddedStorageConfig::InMemory) {
989 return Self::start_in_memory_with_extensions(config.server, extensions)
990 .await
991 .map_err(embedded_stack_error);
992 }
993 start_embedded_dsql(config, extensions).await
994 }
995
996 pub fn startup_report(&self) -> &EngineStartupReport {
998 &self.startup_report
999 }
1000
1001 pub fn endpoint(&self) -> TemporalEndpoint {
1003 self.endpoint.clone()
1004 }
1005
1006 pub fn log_sink(&self) -> broadcast::Receiver<LogEvent> {
1008 self.log_broadcast.subscribe()
1009 }
1010
1011 #[cfg(feature = "temporalio-client")]
1016 pub fn service_override(&self) -> temporalio_client::callback_based::CallbackBasedGrpcService {
1017 self.endpoint.service_override()
1018 }
1019
1020 pub async fn shutdown(mut self) -> Result<(), EmbeddedEngineShutdownError> {
1034 let deadline = Instant::now() + StdDuration::from_secs(30);
1035 let mut failures = Vec::new();
1036 if let Some(mut coordinator) = self.shutdown_coordinator.take() {
1037 coordinator.begin_shutdown();
1038 self.listeners.stop_all(deadline, &mut failures).await;
1039 coordinator.shutdown(deadline, &mut failures).await;
1040 } else {
1041 self.listeners.stop_all(deadline, &mut failures).await;
1042 self.background_cancel.cancel();
1043 }
1044 if let Some(task) = self.recovery_task.take()
1045 && task.await.is_err()
1046 {
1047 failures.push(EmbeddedShutdownFailure::ShardRelease);
1048 }
1049 if let Some(policy) = self.snapshot_policy.take()
1050 && finish_snapshot_policy(policy).await.is_err()
1051 {
1052 failures.push(EmbeddedShutdownFailure::Snapshot);
1053 }
1054 record_embedded_shutdown(&self.startup_report, &failures);
1055 if failures.is_empty() {
1056 Ok(())
1057 } else {
1058 Err(EmbeddedEngineShutdownError { failures })
1059 }
1060 }
1061}
1062
1063impl EmbeddedShutdownCoordinator {
1064 fn begin_shutdown(&self) {
1065 self.service.begin_shutdown();
1066 self.engine_tasks.begin_shutdown();
1067 self.runtime_tasks.begin_shutdown();
1068 }
1069
1070 async fn shutdown(&mut self, deadline: Instant, failures: &mut Vec<EmbeddedShutdownFailure>) {
1071 if self.service.drain(deadline).await.is_err() {
1072 failures.push(EmbeddedShutdownFailure::RpcDrain);
1073 }
1074 if self.engine_tasks.wait(deadline).await.is_err() {
1075 failures.push(EmbeddedShutdownFailure::RuntimeTasks);
1076 }
1077 if self.runtime_tasks.wait(deadline).await.is_err() {
1078 failures.push(EmbeddedShutdownFailure::RuntimeTasks);
1079 }
1080 if let Some(cleanup) = self.shard_cleanup.take()
1081 && cleanup(deadline).await.is_err()
1082 {
1083 failures.push(EmbeddedShutdownFailure::ShardRelease);
1084 }
1085 if let Some(ownership) = &self.ownership {
1091 ownership.gate.begin_closing();
1092 }
1093 if let (Some(ownership), Some(director)) = (&self.ownership, &self.director) {
1094 let now = Instant::now();
1095 let ownership_drained = now < deadline
1096 && tokio::time::timeout(
1097 deadline.saturating_duration_since(now),
1098 ownership.gate.wait_for_drain(),
1099 )
1100 .await
1101 .is_ok();
1102 if ownership_drained {
1103 let now = Instant::now();
1104 let release_result = if now < deadline {
1105 tokio::time::timeout(
1106 deadline.saturating_duration_since(now),
1107 release_embedded_ownership(ownership, director),
1108 )
1109 .await
1110 .map_err(|_| anyhow!("embedded ownership release deadline elapsed"))
1111 .and_then(|result| result)
1112 } else {
1113 Err(anyhow!("embedded ownership release deadline elapsed"))
1114 };
1115 if release_result.is_err() {
1116 failures.push(EmbeddedShutdownFailure::OwnershipRelease);
1117 }
1118 } else {
1119 failures.push(EmbeddedShutdownFailure::OwnershipRelease);
1123 }
1124 }
1125 if let Some(director) = &self.director
1126 && director.shutdown_with_deadline(deadline).await.is_err()
1127 {
1128 failures.push(EmbeddedShutdownFailure::Storage);
1129 }
1130 }
1131}
1132
1133async fn start_embedded_dsql(
1134 config: EmbeddedEngineConfig,
1135 extensions: ChasmExtensions,
1136) -> Result<Engine, EmbeddedEngineStartError> {
1137 let deadline = Instant::now() + StdDuration::from_millis(config.startup_timeout_ms);
1138 let storage_mode = match &config.storage {
1139 EmbeddedStorageConfig::ManagedDsql(_) => EmbeddedStorageMode::ManagedDsql,
1140 EmbeddedStorageConfig::ExistingDsql(_) => EmbeddedStorageMode::ExistingDsql,
1141 EmbeddedStorageConfig::InMemory => unreachable!("in-memory handled by caller"),
1142 };
1143 let region = match &config.storage {
1144 EmbeddedStorageConfig::ManagedDsql(managed) => managed.region.clone(),
1145 EmbeddedStorageConfig::ExistingDsql(existing) => existing.region.clone(),
1146 EmbeddedStorageConfig::InMemory => unreachable!("in-memory handled by caller"),
1147 };
1148 let control = startup_infallible_phase(
1149 deadline,
1150 EmbeddedStartupPhase::ClusterResolution,
1151 AwsDsqlControlPlane::from_region(region.clone()),
1152 )
1153 .await?;
1154 let migration_policy = config
1155 .effective_migration_policy()
1156 .expect("durable embedded mode always has a migration policy");
1157 let storage = config.storage.clone();
1158 let ConnectedEmbeddedDsql {
1159 server,
1160 cluster,
1161 connection_endpoint,
1162 store: dsql_store,
1163 wake_required,
1164 } = connect_embedded_dsql(
1165 config,
1166 control.clone(),
1167 SystemLifecycleEnvironment,
1168 deadline,
1169 DsqlStore::connect_embedded,
1170 )
1171 .await?;
1172 let lifecycle_deadline = StartupDeadline::at(deadline);
1173 let director = dsql_store.connection_director_arc();
1174
1175 let cluster = if wake_required {
1176 match refresh_cluster_after_wake(
1177 &storage,
1178 control,
1179 SystemLifecycleEnvironment,
1180 cluster,
1181 lifecycle_deadline,
1182 deadline,
1183 )
1184 .await
1185 {
1186 Ok(cluster) => cluster,
1187 Err(error) => {
1188 let _ = director.shutdown_with_deadline(deadline).await;
1189 return Err(error);
1190 }
1191 }
1192 } else {
1193 cluster
1194 };
1195
1196 let schema = match startup_phase(
1197 deadline,
1198 EmbeddedStartupPhase::Schema,
1199 apply_embedded_schema(
1200 director.clone(),
1201 &cluster,
1202 schema_migration_policy(migration_policy),
1203 deadline,
1204 ),
1205 )
1206 .await
1207 {
1208 Ok(report) => report,
1209 Err(error) => {
1210 let _ = director.shutdown_with_deadline(deadline).await;
1211 return Err(error);
1212 }
1213 };
1214
1215 let (ownership, ownership_report) = match startup_phase(
1216 deadline,
1217 EmbeddedStartupPhase::Ownership,
1218 acquire_embedded_ownership(director.clone(), &cluster, deadline),
1219 )
1220 .await
1221 {
1222 Ok(ownership) => ownership,
1223 Err(error) => {
1224 let _ = director.shutdown_with_deadline(deadline).await;
1225 return Err(error);
1226 }
1227 };
1228 if director
1229 .install_ownership_gate(ownership.gate.clone())
1230 .is_err()
1231 {
1232 rollback_embedded_dsql(&ownership, &director, deadline).await;
1233 return Err(EmbeddedEngineStartError::Phase {
1234 phase: EmbeddedStartupPhase::Ownership,
1235 });
1236 }
1237
1238 let stack =
1239 match startup_infallible_phase(deadline, EmbeddedStartupPhase::RuntimeRestore, async {
1240 build_dsql_stack(
1241 StackTransport::Embedded,
1242 Arc::new(server),
1243 dsql_store,
1244 connection_endpoint.clone(),
1245 extensions,
1246 )
1247 .await
1248 .map_err(embedded_stack_error)
1249 })
1250 .await
1251 .and_then(|result| result)
1252 {
1253 Ok(ConstructedStack::Embedded(stack)) => stack,
1254 Ok(ConstructedStack::Network(_)) => {
1255 rollback_embedded_dsql(&ownership, &director, deadline).await;
1256 return Err(EmbeddedEngineStartError::Phase {
1257 phase: EmbeddedStartupPhase::RuntimeRestore,
1258 });
1259 }
1260 Err(error) => {
1261 rollback_embedded_dsql(&ownership, &director, deadline).await;
1262 return Err(error);
1263 }
1264 };
1265
1266 spawn_ownership_renewal(
1267 &stack.engine_tasks,
1268 director.clone(),
1269 ownership.clone(),
1270 stack.service.clone(),
1271 );
1272 stack.engine_tasks.close_registration();
1273 let report = EngineStartupReport {
1274 storage_mode,
1275 cluster: Some(cluster_startup_report(&cluster, &connection_endpoint)),
1276 schema: Some(schema),
1277 ownership: Some(ownership_report),
1278 };
1279 let coordinator = EmbeddedShutdownCoordinator {
1280 service: stack.service.clone(),
1281 runtime_tasks: stack.runtime_tasks,
1282 engine_tasks: stack.engine_tasks,
1283 shard_cleanup: stack.shard_cleanup,
1284 director: Some(director),
1285 ownership: Some(ownership),
1286 };
1287 let engine = Engine {
1288 component_visibility_store: stack.component_visibility_store,
1289 chasm_engine: stack.chasm_engine,
1290 registry: stack.registry,
1291 endpoint: TemporalEndpoint {
1292 service: stack.service,
1293 shutdown: stack.background_cancel.clone(),
1294 },
1295 background_cancel: stack.background_cancel,
1296 log_broadcast: stack.log_broadcast,
1297 snapshot_policy: None,
1298 recovery_task: stack.recovery_task,
1299 startup_report: report,
1300 shutdown_coordinator: Some(coordinator),
1301 listeners: listener::ListenerRegistry::default(),
1302 };
1303 record_embedded_startup(&engine.startup_report);
1304 Ok(engine)
1305}
1306
1307#[derive(Debug)]
1311struct ConnectedEmbeddedDsql<S> {
1312 server: TokeiraConfig,
1313 cluster: ResolvedCluster,
1314 connection_endpoint: String,
1315 store: S,
1316 wake_required: bool,
1317}
1318
1319async fn connect_embedded_dsql<C, T, F, Fut, S>(
1322 config: EmbeddedEngineConfig,
1323 control: C,
1324 time: T,
1325 deadline: Instant,
1326 connect: F,
1327) -> Result<ConnectedEmbeddedDsql<S>, EmbeddedEngineStartError>
1328where
1329 C: DsqlControlPlane + Clone,
1330 T: LifecycleEnvironment + Clone,
1331 F: FnOnce(DsqlAuthConfig, EmbeddedDsqlPoolConfig, WarmupDeadline) -> Fut,
1332 Fut: Future<Output = Result<S>>,
1333{
1334 let lifecycle_deadline = StartupDeadline::at(deadline);
1335 let resolved = match &config.storage {
1336 EmbeddedStorageConfig::ManagedDsql(managed) => {
1337 let lifecycle = ManagedDsqlLifecycle::new(
1338 control.clone(),
1339 LocalClusterDescriptorStore::new(&managed.descriptor_path),
1340 time.clone(),
1341 );
1342 startup_phase(
1343 deadline,
1344 EmbeddedStartupPhase::ClusterResolution,
1345 lifecycle.create_or_recover(
1346 CreateOrRecoverRequest {
1347 region: managed.region.clone(),
1348 tags: managed.tags.clone(),
1349 },
1350 lifecycle_deadline,
1351 ),
1352 )
1353 .await?
1354 }
1355 EmbeddedStorageConfig::ExistingDsql(existing) => {
1356 let unused_descriptor = LocalClusterDescriptorStore::new(PathBuf::new());
1360 let lifecycle =
1361 ManagedDsqlLifecycle::new(control.clone(), unused_descriptor, time.clone());
1362 let identity = CanonicalClusterIdentity::new(
1363 &existing.region,
1364 &existing.cluster_id,
1365 &existing.cluster_arn,
1366 )
1367 .map_err(|_| EmbeddedEngineStartError::Phase {
1368 phase: EmbeddedStartupPhase::ClusterResolution,
1369 })?;
1370 startup_phase(
1371 deadline,
1372 EmbeddedStartupPhase::ClusterResolution,
1373 lifecycle.resolve_existing(identity, lifecycle_deadline),
1374 )
1375 .await?
1376 }
1377 EmbeddedStorageConfig::InMemory => unreachable!("in-memory handled by caller"),
1378 };
1379 let readiness = refresh_cluster_until_storage_handoff(
1380 &config.storage,
1381 control.clone(),
1382 time,
1383 resolved,
1384 lifecycle_deadline,
1385 deadline,
1386 )
1387 .await?;
1388 let cluster = match &readiness {
1389 Readiness::Active(usable) => usable.resolved().clone(),
1390 Readiness::WakeRequired(cluster) => cluster.clone(),
1391 };
1392
1393 let connection_endpoint = match &config.storage {
1400 EmbeddedStorageConfig::ExistingDsql(existing) => existing.endpoint.clone(),
1401 EmbeddedStorageConfig::ManagedDsql(_) => cluster.endpoint.clone(),
1402 EmbeddedStorageConfig::InMemory => unreachable!("in-memory handled by caller"),
1403 };
1404 let mut server = config.server;
1405 server.infrastructure.storage = ConfigStorageKind::Dsql;
1406 server.infrastructure.placement.controller_endpoint = None;
1407 server.infrastructure.dsql.endpoint = Some(connection_endpoint.clone());
1408 server.infrastructure.dsql.region = Some(cluster.identity.region.clone());
1409 server
1410 .validate()
1411 .map_err(|_| EmbeddedEngineStartError::Phase {
1412 phase: EmbeddedStartupPhase::Configuration,
1413 })?;
1414
1415 let limits = match &config.storage {
1416 EmbeddedStorageConfig::ManagedDsql(managed) => &managed.limits,
1417 EmbeddedStorageConfig::ExistingDsql(existing) => &existing.limits,
1418 EmbeddedStorageConfig::InMemory => unreachable!("in-memory handled by caller"),
1419 };
1420 let mut pool_config = EmbeddedDsqlPoolConfig::with_limits(
1421 limits.max_connections,
1422 limits.concurrent_connection_creations,
1423 limits.connection_rate_per_second,
1424 limits.connection_burst,
1425 );
1426 pool_config.shard_count = server.infrastructure.placement.shard_count;
1427 pool_config.projection_partition_count = server.infrastructure.placement.partition_count;
1428 let auth = DsqlAuthConfig {
1429 endpoint: connection_endpoint.clone(),
1430 region: Some(cluster.identity.region.clone()),
1431 admin_role_arn: server.infrastructure.dsql.admin_role_arn.clone(),
1432 runtime_role_arn: server.infrastructure.dsql.runtime_role_arn.clone(),
1433 readonly_role_arn: server.infrastructure.dsql.readonly_role_arn.clone(),
1434 };
1435 let store = startup_phase(
1436 deadline,
1437 EmbeddedStartupPhase::ConnectionWarmup,
1438 connect(auth, pool_config, WarmupDeadline::new(deadline)),
1439 )
1440 .await?;
1441 Ok(ConnectedEmbeddedDsql {
1442 server,
1443 cluster,
1444 connection_endpoint,
1445 store,
1446 wake_required: matches!(readiness, Readiness::WakeRequired(_)),
1447 })
1448}
1449
1450fn cluster_startup_report(
1451 cluster: &ResolvedCluster,
1452 connection_endpoint: &str,
1453) -> ClusterStartupReport {
1454 ClusterStartupReport {
1455 region: cluster.identity.region.clone(),
1456 cluster_id: cluster.identity.cluster_id.clone(),
1457 cluster_arn: cluster.identity.cluster_arn.clone(),
1458 endpoint: connection_endpoint.to_owned(),
1459 action: cluster.action,
1460 }
1461}
1462
1463fn storage_mode_label(mode: EmbeddedStorageMode) -> EmbeddedStorageModeLabel {
1464 match mode {
1465 EmbeddedStorageMode::InMemory => EmbeddedStorageModeLabel::InMemory,
1466 EmbeddedStorageMode::ManagedDsql => EmbeddedStorageModeLabel::ManagedDsql,
1467 EmbeddedStorageMode::ExistingDsql => EmbeddedStorageModeLabel::ExistingDsql,
1468 }
1469}
1470
1471fn schema_outcome_label(report: &EngineStartupReport) -> SchemaOutcomeLabel {
1472 match report.schema.as_ref().map(|schema| schema.outcome) {
1473 None => SchemaOutcomeLabel::NotApplicable,
1474 Some(SchemaStartupOutcome::Compatible) => SchemaOutcomeLabel::Compatible,
1475 Some(SchemaStartupOutcome::Initialized) => SchemaOutcomeLabel::Initialized,
1476 Some(SchemaStartupOutcome::Migrated) => SchemaOutcomeLabel::Migrated,
1477 Some(SchemaStartupOutcome::MetadataBackfilled) => SchemaOutcomeLabel::MetadataBackfilled,
1478 }
1479}
1480
1481fn ownership_outcome_label(report: &EngineStartupReport) -> OwnershipOutcomeLabel {
1482 match report.ownership.map(|ownership| ownership.outcome) {
1483 None => OwnershipOutcomeLabel::NotApplicable,
1484 Some(ControlLeaseAcquireOutcome::Clean) => OwnershipOutcomeLabel::AcquiredClean,
1485 Some(ControlLeaseAcquireOutcome::ExpiredTakeover) => OwnershipOutcomeLabel::AcquiredExpired,
1486 }
1487}
1488
1489fn record_embedded_startup(report: &EngineStartupReport) {
1495 record_embedded_lifecycle(
1496 storage_mode_label(report.storage_mode),
1497 if report.cluster.is_some() {
1498 ClusterStatusLabel::Active
1499 } else {
1500 ClusterStatusLabel::NotApplicable
1501 },
1502 schema_outcome_label(report),
1503 ownership_outcome_label(report),
1504 DbClassLabel::Control,
1505 EmbeddedOperationLabel::Startup,
1506 ErrorClassLabel::None,
1507 );
1508}
1509
1510fn record_embedded_shutdown(report: &EngineStartupReport, failures: &[EmbeddedShutdownFailure]) {
1511 let ownership = if report.ownership.is_some() && failures.is_empty() {
1512 OwnershipOutcomeLabel::Released
1513 } else {
1514 ownership_outcome_label(report)
1515 };
1516 record_embedded_lifecycle(
1517 storage_mode_label(report.storage_mode),
1518 if report.cluster.is_some() {
1519 ClusterStatusLabel::Active
1520 } else {
1521 ClusterStatusLabel::NotApplicable
1522 },
1523 schema_outcome_label(report),
1524 ownership,
1525 DbClassLabel::Control,
1526 EmbeddedOperationLabel::Shutdown,
1527 if failures.is_empty() {
1528 ErrorClassLabel::None
1529 } else {
1530 ErrorClassLabel::Internal
1531 },
1532 );
1533}
1534
1535async fn refresh_cluster_until_storage_handoff<C, T>(
1536 storage: &EmbeddedStorageConfig,
1537 control: C,
1538 time: T,
1539 cluster: ResolvedCluster,
1540 lifecycle_deadline: StartupDeadline,
1541 deadline: Instant,
1542) -> Result<Readiness, EmbeddedEngineStartError>
1543where
1544 C: DsqlControlPlane,
1545 T: LifecycleEnvironment,
1546{
1547 match storage {
1548 EmbeddedStorageConfig::ManagedDsql(managed) => {
1549 let lifecycle = ManagedDsqlLifecycle::new(
1550 control,
1551 LocalClusterDescriptorStore::new(&managed.descriptor_path),
1552 time,
1553 );
1554 startup_phase(
1555 deadline,
1556 EmbeddedStartupPhase::ClusterResolution,
1557 lifecycle.refresh_until_usable(cluster, lifecycle_deadline),
1558 )
1559 .await
1560 }
1561 EmbeddedStorageConfig::ExistingDsql(_) => {
1562 let lifecycle = ManagedDsqlLifecycle::new(
1563 control,
1564 LocalClusterDescriptorStore::new(PathBuf::new()),
1565 time,
1566 );
1567 startup_phase(
1568 deadline,
1569 EmbeddedStartupPhase::ClusterResolution,
1570 lifecycle.refresh_until_usable(cluster, lifecycle_deadline),
1571 )
1572 .await
1573 }
1574 EmbeddedStorageConfig::InMemory => unreachable!("in-memory handled by caller"),
1575 }
1576}
1577
1578async fn refresh_cluster_after_wake<C, T>(
1579 storage: &EmbeddedStorageConfig,
1580 control: C,
1581 time: T,
1582 cluster: ResolvedCluster,
1583 lifecycle_deadline: StartupDeadline,
1584 deadline: Instant,
1585) -> Result<ResolvedCluster, EmbeddedEngineStartError>
1586where
1587 C: DsqlControlPlane,
1588 T: LifecycleEnvironment,
1589{
1590 let usable = match storage {
1591 EmbeddedStorageConfig::ManagedDsql(managed) => {
1592 let lifecycle = ManagedDsqlLifecycle::new(
1593 control,
1594 LocalClusterDescriptorStore::new(&managed.descriptor_path),
1595 time,
1596 );
1597 startup_phase(
1598 deadline,
1599 EmbeddedStartupPhase::ConnectionWarmup,
1600 lifecycle.refresh_after_wake(cluster, lifecycle_deadline),
1601 )
1602 .await?
1603 }
1604 EmbeddedStorageConfig::ExistingDsql(_) => {
1605 let lifecycle = ManagedDsqlLifecycle::new(
1606 control,
1607 LocalClusterDescriptorStore::new(PathBuf::new()),
1608 time,
1609 );
1610 startup_phase(
1611 deadline,
1612 EmbeddedStartupPhase::ConnectionWarmup,
1613 lifecycle.refresh_after_wake(cluster, lifecycle_deadline),
1614 )
1615 .await?
1616 }
1617 EmbeddedStorageConfig::InMemory => unreachable!("in-memory handled by caller"),
1618 };
1619 Ok(usable.into_resolved())
1620}
1621
1622fn schema_migration_policy(policy: DsqlMigrationPolicy) -> SchemaMigrationPolicy {
1623 match policy {
1624 DsqlMigrationPolicy::Automatic => SchemaMigrationPolicy::Automatic,
1625 DsqlMigrationPolicy::ValidateOnly => SchemaMigrationPolicy::ValidateOnly,
1626 }
1627}
1628
1629async fn apply_embedded_schema(
1630 director: Arc<DsqlConnectionDirector>,
1631 cluster: &ResolvedCluster,
1632 policy: SchemaMigrationPolicy,
1633 deadline: Instant,
1634) -> Result<SchemaStartupReport> {
1635 let runner = MigrationRunner::embedded();
1636 let contract = MigrationRunner::compatibility_contract();
1637 let leases = ConnectionControlLeaseRepository::new();
1638 let mut permit = director.acquire(DbClass::Control).await?;
1639 let decision = runner
1640 .assess_connection(permit.connection()?, &contract, policy)
1641 .await?;
1642 let outcome = match &decision {
1643 SchemaDecision::Compatible {
1644 legacy_backfill: false,
1645 ..
1646 } => SchemaStartupOutcome::Compatible,
1647 SchemaDecision::Compatible {
1648 legacy_backfill: true,
1649 ..
1650 } => SchemaStartupOutcome::MetadataBackfilled,
1651 SchemaDecision::Initialize { .. } => SchemaStartupOutcome::Initialized,
1652 SchemaDecision::Migrate { .. } => SchemaStartupOutcome::Migrated,
1653 SchemaDecision::MigrationRequired { current, target } => {
1654 return Err(anyhow!(
1655 "schema migration required from V{current} to V{target}"
1656 ));
1657 }
1658 SchemaDecision::Reject(_) => return Err(anyhow!("schema compatibility rejected")),
1659 };
1660 if !matches!(outcome, SchemaStartupOutcome::Compatible) {
1661 runner
1662 .bootstrap_migration_coordination(permit.connection()?, &decision)
1663 .await?;
1664 leases.bootstrap(permit.connection()?).await?;
1665 let mut migration_guard = leases
1666 .acquire(
1667 permit.connection()?,
1668 &ControlLeaseAcquireRequest {
1669 claim_name: "schema-migration".to_owned(),
1670 cluster: control_lease_identity(cluster),
1671 owner_id: format!("schema-{}", IncarnationId::new()),
1672 lease_duration: SCHEMA_MIGRATION_LEASE_DURATION,
1673 admission_margin: SCHEMA_MIGRATION_ADMISSION_MARGIN,
1674 acquire_deadline: deadline,
1675 },
1676 )
1677 .await?;
1678 let migration_gate = OwnershipAdmissionGate::for_guard(&migration_guard);
1679 let apply_result = runner
1680 .apply_decision(
1681 permit.connection()?,
1682 &decision,
1683 &leases,
1684 &mut migration_guard,
1685 &migration_gate,
1686 )
1687 .await;
1688 let release_result = leases
1689 .release(permit.connection()?, &migration_guard, &migration_gate)
1690 .await;
1691 apply_result?;
1692 release_result?;
1693 }
1694 let final_decision = runner
1695 .assess_connection(
1696 permit.connection()?,
1697 &contract,
1698 SchemaMigrationPolicy::ValidateOnly,
1699 )
1700 .await?;
1701 let observed_version = match final_decision {
1702 SchemaDecision::Compatible { current, .. } => current,
1703 _ => return Err(anyhow!("schema did not converge to a readable version")),
1704 };
1705 Ok(SchemaStartupReport {
1706 observed_version,
1707 minimum_supported_version: contract.minimum_supported_version,
1708 target_version: contract.target_version,
1709 maximum_readable_version: contract.maximum_readable_version,
1710 migration_set_digest: contract.migration_set_digest,
1711 outcome,
1712 })
1713}
1714
1715async fn acquire_embedded_ownership(
1716 director: Arc<DsqlConnectionDirector>,
1717 cluster: &ResolvedCluster,
1718 deadline: Instant,
1719) -> Result<(EmbeddedOwnership, OwnershipStartupReport)> {
1720 let repository = ConnectionControlLeaseRepository::new();
1721 let mut permit = director.acquire(DbClass::Control).await?;
1722 repository.bootstrap(permit.connection()?).await?;
1723 let guard = repository
1724 .acquire(
1725 permit.connection()?,
1726 &ControlLeaseAcquireRequest {
1727 claim_name: "embedded-owner".to_owned(),
1728 cluster: control_lease_identity(cluster),
1729 owner_id: format!("engine-{}", IncarnationId::new()),
1730 lease_duration: OWNER_LEASE_DURATION,
1731 admission_margin: OWNER_ADMISSION_MARGIN,
1732 acquire_deadline: deadline,
1733 },
1734 )
1735 .await?;
1736 let gate = OwnershipAdmissionGate::for_guard(&guard);
1737 if let Some(quiescence_deadline) = guard.quiescence_deadline() {
1738 let now = Instant::now();
1739 if now >= deadline || quiescence_deadline > deadline {
1740 return Err(anyhow!("expired owner quiescence exceeds startup deadline"));
1741 }
1742 tokio::time::sleep_until(tokio::time::Instant::from_std(quiescence_deadline)).await;
1743 gate.finish_quiescence(&guard, Instant::now())?;
1744 }
1745 let report = OwnershipStartupReport {
1746 outcome: guard.outcome(),
1747 fence_token: guard.fence_token(),
1748 };
1749 Ok((
1750 EmbeddedOwnership {
1751 repository,
1752 guard: Arc::new(tokio::sync::Mutex::new(Some(guard))),
1753 gate,
1754 },
1755 report,
1756 ))
1757}
1758
1759fn spawn_ownership_renewal(
1760 tasks: &RuntimeShutdownHandle,
1761 director: Arc<DsqlConnectionDirector>,
1762 ownership: EmbeddedOwnership,
1763 service: InProcessGrpcService,
1764) {
1765 let cancel = tasks.cancellation_token();
1766 let _renewal = tasks.spawn(async move {
1767 let interval_duration =
1768 StdDuration::try_from(OWNER_RENEW_INTERVAL).expect("positive owner renewal interval");
1769 let mut interval = tokio::time::interval(interval_duration);
1770 interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
1771 interval.tick().await;
1772 loop {
1773 tokio::select! {
1774 _ = cancel.cancelled() => break,
1775 _ = interval.tick() => {
1776 let mut guard_slot = ownership.guard.lock().await;
1777 let Some(guard) = guard_slot.as_mut() else {
1778 break;
1779 };
1780 let result = match director.acquire(DbClass::Control).await {
1781 Ok(mut permit) => match permit.connection() {
1782 Ok(connection) => ownership.repository.renew(
1783 connection,
1784 guard,
1785 OWNER_LEASE_DURATION,
1786 OWNER_ADMISSION_MARGIN,
1787 &ownership.gate,
1788 )
1789 .await
1790 .map_err(anyhow::Error::from),
1791 Err(error) => Err(error),
1792 },
1793 Err(error) => Err(error),
1794 };
1795 if let Err(error) = result {
1796 if matches!(
1797 error.downcast_ref::<ControlLeaseError>(),
1798 Some(ControlLeaseError::Fenced)
1799 ) {
1800 ownership.gate.fence();
1801 } else {
1802 guard.enforce_admission_deadline(Instant::now(), &ownership.gate);
1803 }
1804 }
1805 if ownership.gate.state() != OwnershipAdmissionState::Open {
1806 service.begin_shutdown();
1807 break;
1808 }
1809 }
1810 }
1811 }
1812 });
1813}
1814
1815fn control_lease_identity(cluster: &ResolvedCluster) -> ControlLeaseClusterIdentity {
1816 ControlLeaseClusterIdentity {
1817 cluster_id: cluster.identity.cluster_id.clone(),
1818 cluster_arn: cluster.identity.cluster_arn.clone(),
1819 }
1820}
1821
1822async fn rollback_embedded_dsql(
1823 ownership: &EmbeddedOwnership,
1824 director: &Arc<DsqlConnectionDirector>,
1825 deadline: Instant,
1826) {
1827 ownership.gate.begin_closing();
1828 let now = Instant::now();
1829 if now < deadline {
1830 let _ = tokio::time::timeout(
1831 deadline.saturating_duration_since(now),
1832 release_embedded_ownership(ownership, director),
1833 )
1834 .await;
1835 }
1836 let _ = director.shutdown_with_deadline(deadline).await;
1837}
1838
1839async fn release_embedded_ownership(
1840 ownership: &EmbeddedOwnership,
1841 director: &Arc<DsqlConnectionDirector>,
1842) -> Result<()> {
1843 let mut permit = director.acquire_shutdown_control().await?;
1846 let mut guard = ownership.guard.lock().await;
1847 if let Some(guard) = guard.take() {
1848 ownership
1849 .repository
1850 .release(permit.connection()?, &guard, &ownership.gate)
1851 .await?;
1852 }
1853 Ok(())
1854}
1855
1856async fn startup_phase<T, E, F>(
1857 deadline: Instant,
1858 phase: EmbeddedStartupPhase,
1859 future: F,
1860) -> Result<T, EmbeddedEngineStartError>
1861where
1862 F: Future<Output = Result<T, E>>,
1863{
1864 let now = Instant::now();
1865 if now >= deadline {
1866 return Err(EmbeddedEngineStartError::DeadlineExceeded { phase });
1867 }
1868 match tokio::time::timeout(deadline.saturating_duration_since(now), future).await {
1869 Ok(Ok(value)) => Ok(value),
1870 Ok(Err(_)) => Err(EmbeddedEngineStartError::Phase { phase }),
1871 Err(_) => Err(EmbeddedEngineStartError::DeadlineExceeded { phase }),
1872 }
1873}
1874
1875async fn startup_infallible_phase<T, F>(
1876 deadline: Instant,
1877 phase: EmbeddedStartupPhase,
1878 future: F,
1879) -> Result<T, EmbeddedEngineStartError>
1880where
1881 F: Future<Output = T>,
1882{
1883 let now = Instant::now();
1884 if now >= deadline {
1885 return Err(EmbeddedEngineStartError::DeadlineExceeded { phase });
1886 }
1887 tokio::time::timeout(deadline.saturating_duration_since(now), future)
1888 .await
1889 .map_err(|_| EmbeddedEngineStartError::DeadlineExceeded { phase })
1890}
1891
1892async fn finish_snapshot_policy(mut policy: EngineSnapshotPolicy) -> Result<()> {
1899 if let Some(task) = policy.periodic_task.take() {
1900 task.await
1901 .context("engine snapshot task panicked during shutdown")?;
1902 }
1903 tokio::task::yield_now().await;
1908 persist_snapshot(&policy.store, &policy.location)
1909 .await
1910 .with_context(|| {
1911 format!(
1912 "failed to persist final engine snapshot to {}",
1913 policy.location.display()
1914 )
1915 })
1916}
1917
1918async fn restore_snapshot_store(config: &TokeiraConfig) -> Result<(InMemoryStore, bool)> {
1919 let Some(snapshot) = config.policy.snapshot.as_ref() else {
1920 return Ok((InMemoryStore::default(), false));
1921 };
1922 match fs::read(&snapshot.location).await {
1923 Ok(bytes) => {
1924 let store = InMemoryStore::from_snapshot(&bytes).with_context(|| {
1925 format!(
1926 "failed to restore engine snapshot from {}",
1927 snapshot.location.display()
1928 )
1929 })?;
1930 retire_snapshot_leases(&store).await.with_context(|| {
1931 format!(
1932 "failed to retire prior embedded-engine leases from {}",
1933 snapshot.location.display()
1934 )
1935 })?;
1936 info!(
1937 snapshot_path = %snapshot.location.display(),
1938 "restored engine snapshot"
1939 );
1940 Ok((store, true))
1941 }
1942 Err(error) if error.kind() == std::io::ErrorKind::NotFound => {
1943 Ok((InMemoryStore::default(), false))
1944 }
1945 Err(error) => Err(error).with_context(|| {
1946 format!(
1947 "failed to read engine snapshot from {}",
1948 snapshot.location.display()
1949 )
1950 }),
1951 }
1952}
1953
1954async fn retire_snapshot_leases(store: &InMemoryStore) -> Result<()> {
1955 for lease in store
1956 .list_bundle_leases()
1957 .await
1958 .context("failed to list engine snapshot leases")?
1959 {
1960 let Some(owner) = lease.owner_node_id else {
1961 continue;
1962 };
1963 match store
1964 .relinquish_bundle(lease.bundle_id, owner, lease.epoch)
1965 .await
1966 .context("failed to relinquish an engine snapshot lease")?
1967 {
1968 LeaseOutcome::Acquired { .. } => {}
1969 LeaseOutcome::Rejected {
1970 current_owner,
1971 current_epoch,
1972 } => {
1973 return Err(anyhow!(
1974 "engine snapshot lease retirement was fenced by owner {current_owner} at epoch {}",
1975 current_epoch.0
1976 ));
1977 }
1978 LeaseOutcome::Renewed { epoch } => {
1979 return Err(anyhow!(
1980 "engine snapshot lease retirement unexpectedly renewed epoch {}",
1981 epoch.0
1982 ));
1983 }
1984 }
1985 }
1986 Ok(())
1987}
1988
1989async fn persist_snapshot(store: &InMemoryStore, location: &Path) -> Result<()> {
1990 let bytes = store
1991 .snapshot()
1992 .await
1993 .context("failed to capture engine snapshot")?;
1994 write_snapshot_atomically(location, &bytes).await
1995}
1996
1997async fn write_snapshot_atomically(location: &Path, bytes: &[u8]) -> Result<()> {
1998 if let Some(parent) = location
1999 .parent()
2000 .filter(|parent| !parent.as_os_str().is_empty())
2001 {
2002 fs::create_dir_all(parent).await.with_context(|| {
2003 format!(
2004 "failed to create engine snapshot directory {}",
2005 parent.display()
2006 )
2007 })?;
2008 }
2009
2010 let sequence = SNAPSHOT_TEMP_SEQUENCE.fetch_add(1, Ordering::Relaxed);
2014 let mut temporary_name = location.as_os_str().to_os_string();
2015 temporary_name.push(format!(".tmp-{}-{sequence}", std::process::id()));
2016 let temporary = PathBuf::from(temporary_name);
2017
2018 let write_result: Result<()> = async {
2019 let mut file = OpenOptions::new()
2020 .create(true)
2021 .truncate(true)
2022 .write(true)
2023 .open(&temporary)
2024 .await
2025 .with_context(|| {
2026 format!(
2027 "failed to create temporary engine snapshot {}",
2028 temporary.display()
2029 )
2030 })?;
2031 file.write_all(bytes).await.with_context(|| {
2032 format!(
2033 "failed to write temporary engine snapshot {}",
2034 temporary.display()
2035 )
2036 })?;
2037 file.sync_all().await.with_context(|| {
2038 format!(
2039 "failed to sync temporary engine snapshot {}",
2040 temporary.display()
2041 )
2042 })?;
2043 drop(file);
2044 fs::rename(&temporary, location)
2045 .await
2046 .with_context(|| format!("failed to replace engine snapshot {}", location.display()))?;
2047 Ok(())
2048 }
2049 .await;
2050
2051 if write_result.is_err() {
2052 let _ = fs::remove_file(&temporary).await;
2055 }
2056 write_result
2057}
2058
2059fn embedded_config(mut config: TokeiraConfig) -> Result<TokeiraConfig> {
2060 config.infrastructure.storage = ConfigStorageKind::InMemory;
2061 config.infrastructure.placement.controller_endpoint = None;
2065 config
2066 .validate()
2067 .context("invalid embedded engine config")?;
2068 Ok(config)
2069}
2070
2071impl Drop for Engine {
2072 fn drop(&mut self) {
2073 if let Some(coordinator) = &self.shutdown_coordinator {
2074 coordinator.begin_shutdown();
2075 }
2076 self.background_cancel.cancel();
2077 }
2078}
2079
2080impl TemporalEndpoint {
2081 pub async fn call(
2083 &self,
2084 request: InProcessGrpcRequest,
2085 ) -> Result<InProcessGrpcResponse, tonic::Status> {
2086 tokio::select! {
2087 biased;
2088 _ = self.shutdown.cancelled() => {
2089 Err(tonic::Status::unavailable("embedded Tokeira engine is shut down"))
2090 }
2091 response = self.service.call(request) => response,
2092 }
2093 }
2094
2095 pub fn is_shutdown(&self) -> bool {
2097 self.shutdown.is_cancelled()
2098 }
2099
2100 #[cfg(feature = "temporalio-client")]
2102 pub fn service_override(&self) -> temporalio_client::callback_based::CallbackBasedGrpcService {
2103 let endpoint = self.clone();
2104 temporalio_client::callback_based::CallbackBasedGrpcService {
2105 callback: Arc::new(move |request| {
2106 let endpoint = endpoint.clone();
2107 Box::pin(async move {
2108 endpoint
2109 .call(InProcessGrpcRequest {
2110 service: request.service,
2111 rpc: request.rpc,
2112 headers: request.headers,
2113 proto: request.proto,
2114 })
2115 .await
2116 .map(
2120 |response| temporalio_client::callback_based::GrpcSuccessResponse {
2121 headers: response.headers,
2122 proto: response.proto,
2123 },
2124 )
2125 })
2126 }),
2127 }
2128 }
2129}
2130
2131const EMBEDDED_NEXUS_CALLBACK_BASE: &str = "http://tokeira-engine.invalid";
2132
2133struct InProcessNexusCompletionClient {
2136 runtime: RwLock<Option<Weak<dyn WorkflowRuntimeApi>>>,
2137 http: HttpNexusCompletionClient,
2138}
2139
2140impl std::fmt::Debug for InProcessNexusCompletionClient {
2141 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
2142 f.debug_struct("InProcessNexusCompletionClient")
2143 .finish_non_exhaustive()
2144 }
2145}
2146
2147impl InProcessNexusCompletionClient {
2148 fn new() -> Self {
2149 Self {
2150 runtime: RwLock::new(None),
2151 http: HttpNexusCompletionClient::new(),
2152 }
2153 }
2154
2155 fn attach(&self, runtime: &Arc<dyn WorkflowRuntimeApi>) -> Result<()> {
2156 let mut slot = self
2157 .runtime
2158 .write()
2159 .map_err(|_| anyhow!("embedded Nexus completion runtime lock poisoned"))?;
2160 *slot = Some(Arc::downgrade(runtime));
2161 Ok(())
2162 }
2163}
2164
2165#[async_trait]
2166impl NexusCompletionClient for InProcessNexusCompletionClient {
2167 async fn complete_operation(
2168 &self,
2169 url: &str,
2170 token: &str,
2171 operation_token: &str,
2172 completion: NexusCompletion,
2173 links: &[Link],
2174 ) -> Result<CompletionDeliveryOutcome> {
2175 if url != system_callback_post_url(EMBEDDED_NEXUS_CALLBACK_BASE) {
2176 return self
2177 .http
2178 .complete_operation(url, token, operation_token, completion, links)
2179 .await;
2180 }
2181
2182 let runtime = self
2183 .runtime
2184 .read()
2185 .ok()
2186 .and_then(|slot| slot.as_ref().and_then(Weak::upgrade));
2187 let Some(runtime) = runtime else {
2188 return Ok(CompletionDeliveryOutcome::RetryableError {
2189 detail: "embedded Nexus completion runtime is unavailable".to_owned(),
2190 });
2191 };
2192
2193 let state = completion.operation_state();
2194 let (body, content_type) = match completion {
2195 NexusCompletion::Succeeded(payloads) => nexus_payload_to_body(&payloads),
2196 NexusCompletion::Failed(body) | NexusCompletion::Canceled(body) => {
2197 (body, Some("application/json".to_owned()))
2198 }
2199 };
2200 let response = handle_nexus_callback(
2201 runtime.as_ref(),
2202 Some(token),
2203 Some(operation_token),
2204 Some(state),
2205 content_type.as_deref(),
2206 &body,
2207 )
2208 .await;
2209 let detail = response
2210 .body
2211 .map(|body| String::from_utf8_lossy(&body).into_owned())
2212 .unwrap_or_else(|| format!("Nexus callback returned status {}", response.status));
2213 Ok(match response.status {
2214 200..=299 => CompletionDeliveryOutcome::Delivered,
2215 400 | 401 | 403 | 404 | 409 | 501 => {
2216 CompletionDeliveryOutcome::NonRetryableError { detail }
2217 }
2218 _ => CompletionDeliveryOutcome::RetryableError { detail },
2219 })
2220 }
2221}
2222
2223#[derive(Debug)]
2236pub struct TokeiradHandle {
2237 bound_addr: SocketAddr,
2238 shutdown_tx: Option<oneshot::Sender<()>>,
2239 server_task: Option<JoinHandle<Result<()>>>,
2240 background_cancel: CancellationToken,
2246 log_broadcast: broadcast::Sender<LogEvent>,
2247 snapshot_policy: Option<EngineSnapshotPolicy>,
2251}
2252
2253#[derive(Clone, Debug)]
2260pub struct LogEvent {
2261 pub rpc: String,
2262 pub namespace: String,
2263 pub heartbeat_count: usize,
2264}
2265
2266impl TokeiradHandle {
2267 pub async fn start_in_memory(addr: SocketAddr) -> Result<Self> {
2274 Self::start_in_memory_with_config(addr, TokeiraConfig::default()).await
2275 }
2276
2277 pub async fn start_in_memory_with_config(
2284 addr: SocketAddr,
2285 mut config: TokeiraConfig,
2286 ) -> Result<Self> {
2287 config.infrastructure.storage = ConfigStorageKind::InMemory;
2288 config.policy.nexus_completion.http_addr = "127.0.0.1:0".to_string();
2292 config.policy.nexus_completion.system_callback_url = "http://127.0.0.1:0".to_string();
2293 config
2294 .validate()
2295 .context("invalid in-memory server config")?;
2296 let effective_config = Arc::new(config);
2297 let (
2298 (server_task, bound_addr, shutdown_tx, background_cancel, log_broadcast, _recorder),
2299 snapshot_policy,
2300 ) = build_and_serve(addr, effective_config).await?;
2301 Ok(Self {
2302 bound_addr,
2303 shutdown_tx: Some(shutdown_tx),
2304 server_task: Some(server_task),
2305 background_cancel,
2306 log_broadcast,
2307 snapshot_policy,
2308 })
2309 }
2310
2311 pub async fn start_in_memory_with_toml(addr: SocketAddr, config: &str) -> Result<Self> {
2317 let config = toml::from_str(config).context("parse in-memory server config")?;
2318 Self::start_in_memory_with_config(addr, config).await
2319 }
2320
2321 pub fn bound_addr(&self) -> SocketAddr {
2323 self.bound_addr
2324 }
2325
2326 pub fn log_sink(&self) -> broadcast::Receiver<LogEvent> {
2329 self.log_broadcast.subscribe()
2330 }
2331
2332 pub async fn shutdown(mut self) -> Result<()> {
2335 if let Some(tx) = self.shutdown_tx.take() {
2336 let _ = tx.send(());
2337 }
2338 self.background_cancel.cancel();
2339 if let Some(task) = self.server_task.take() {
2340 task.await
2341 .context("tokeirad server task panicked")?
2342 .context("tokeirad server task returned an error")?;
2343 }
2344 match self.snapshot_policy.take() {
2345 Some(policy) => finish_snapshot_policy(policy).await,
2346 None => Ok(()),
2347 }
2348 }
2349}
2350
2351impl Drop for TokeiradHandle {
2352 fn drop(&mut self) {
2353 if let Some(tx) = self.shutdown_tx.take() {
2354 let _ = tx.send(());
2355 }
2356 self.background_cancel.cancel();
2357 }
2360}
2361
2362fn dsql_auth_config(config: &TokeiraConfig) -> Result<DsqlAuthConfig> {
2363 let dsql = &config.infrastructure.dsql;
2364 let endpoint = dsql
2365 .endpoint
2366 .clone()
2367 .filter(|endpoint| !endpoint.trim().is_empty())
2368 .ok_or_else(|| {
2369 anyhow!("infrastructure.dsql.endpoint must be set when infrastructure.storage is dsql")
2370 })?;
2371 Ok(DsqlAuthConfig {
2372 endpoint,
2373 region: dsql.region.clone(),
2374 admin_role_arn: dsql.admin_role_arn.clone(),
2375 runtime_role_arn: dsql.runtime_role_arn.clone(),
2376 readonly_role_arn: dsql.readonly_role_arn.clone(),
2377 })
2378}
2379
2380fn dsql_pool_config_with_client(
2381 config: &TokeiraConfig,
2382 ddb_client: aws_sdk_dynamodb::Client,
2383) -> (DsqlPoolConfig, aws_sdk_dynamodb::Client) {
2384 let (rate_limiter_table, conn_lease_table) = dsql_coordination_table_names(config);
2385 let pool_config = DsqlPoolConfig {
2386 coordination: DsqlCoordinationConfig {
2387 rate_limiter_table,
2388 conn_lease_table,
2389 },
2390 shard_count: config.infrastructure.placement.shard_count,
2391 projection_partition_count: config.infrastructure.placement.partition_count,
2392 ..DsqlPoolConfig::default()
2393 };
2394 (pool_config, ddb_client)
2395}
2396
2397async fn dsql_pool_config(
2398 config: &TokeiraConfig,
2399 auth: &DsqlAuthConfig,
2400) -> Result<(DsqlPoolConfig, aws_sdk_dynamodb::Client)> {
2401 let region = auth
2402 .resolved_region()
2403 .ok_or_else(|| anyhow!("dsql region must be configured or derivable from endpoint"))?;
2404 let sdk_config = aws_config::defaults(aws_config::BehaviorVersion::latest())
2405 .region(aws_config::Region::new(region))
2406 .load()
2407 .await;
2408 Ok(dsql_pool_config_with_client(
2409 config,
2410 aws_sdk_dynamodb::Client::new(&sdk_config),
2411 ))
2412}
2413
2414fn dsql_coordination_table_names(config: &TokeiraConfig) -> (String, String) {
2415 let dsql = &config.infrastructure.dsql;
2416 let rate_limiter = dsql
2417 .rate_limiter_table
2418 .clone()
2419 .unwrap_or_else(|| format!("{}-dsql-rate-limiter", config.infrastructure.cluster_name));
2420 let conn_lease = dsql
2421 .conn_lease_table
2422 .clone()
2423 .unwrap_or_else(|| format!("{}-dsql-conn-lease", config.infrastructure.cluster_name));
2424 (rate_limiter, conn_lease)
2425}
2426
2427#[allow(clippy::print_stdout)]
2437pub async fn run_from_cli(cli: Cli) -> Result<()> {
2438 if cli.version {
2439 println!("{}", render_build_info(cli.verbose, cli.json));
2440 return Ok(());
2441 }
2442
2443 let (effective_config, config_source) = TokeiraConfig::resolve(cli.config.as_deref())?;
2444 if cli.dump_config {
2445 println!("# config source: {config_source}");
2447 println!("{}", effective_config.to_toml()?);
2448 return Ok(());
2449 }
2450 let observability = observability::ObservabilityConfig::from_tokeira_config(&effective_config)?;
2451 let effective_config = Arc::new(effective_config);
2452 let readiness = observability::TokeiradReadiness::new();
2453 let _observability_runtime = observability::install_process_observability(
2454 &observability,
2455 effective_config.clone(),
2456 readiness.registry.clone(),
2457 )
2458 .await?;
2459 log_build_info("tokeirad");
2460 for warning in effective_config.emergency_warnings() {
2461 tracing::warn!("{warning}");
2462 }
2463
2464 let addr: SocketAddr = effective_config
2465 .infrastructure
2466 .network
2467 .grpc_addr
2468 .parse()
2469 .with_context(|| {
2470 format!(
2471 "invalid infrastructure.network.grpc_addr value: {}",
2472 effective_config.infrastructure.network.grpc_addr
2473 )
2474 })?;
2475 info!(config_source = %config_source, "loaded tokeirad configuration");
2476
2477 let (
2478 (
2479 mut server_task,
2480 bound_addr,
2481 shutdown_tx,
2482 background_cancel,
2483 _log_broadcast,
2484 wire_recorder,
2485 ),
2486 snapshot_policy,
2487 ) = build_and_serve(addr, effective_config).await?;
2488 readiness.mark_started();
2489 info!("tokeirad gRPC server listening on {bound_addr}");
2490
2491 let serve_result = {
2499 let drain_signal = async {
2500 #[cfg(unix)]
2501 {
2502 let mut sigterm =
2503 tokio::signal::unix::signal(tokio::signal::unix::SignalKind::terminate())
2504 .context("failed to install the SIGTERM handler")?;
2505 tokio::select! {
2506 result = tokio::signal::ctrl_c() => result
2507 .context("failed to listen for ctrl-c")
2508 .map(|()| "SIGINT"),
2509 _ = sigterm.recv() => Ok("SIGTERM"),
2510 }
2511 }
2512 #[cfg(not(unix))]
2513 {
2514 tokio::signal::ctrl_c()
2515 .await
2516 .context("failed to listen for ctrl-c")
2517 .map(|()| "ctrl-c")
2518 }
2519 };
2520 tokio::select! {
2521 result = &mut server_task => result
2522 .context("tokeirad server task panicked")?
2523 .context("tokeirad server task returned an error"),
2524 signal = drain_signal => {
2525 match signal {
2526 Ok(signal) => info!(signal, "shutdown signal received; draining tokeirad"),
2527 Err(error) => {
2528 tracing::warn!(?error, "signal handling failed; shutting down");
2531 }
2532 }
2533 let _ = shutdown_tx.send(());
2534 (&mut server_task)
2535 .await
2536 .context("tokeirad server task panicked")?
2537 .context("tokeirad server task returned an error")
2538 }
2539 }
2540 };
2541 background_cancel.cancel();
2542 let snapshot_result = match snapshot_policy {
2547 Some(policy) => finish_snapshot_policy(policy).await,
2548 None => Ok(()),
2549 };
2550
2551 if let Some(recorder) = wire_recorder {
2560 let out_path = wire_coverage_out_path();
2561 if let Err(error) = export_wire_coverage(recorder.as_ref(), &out_path) {
2562 tracing::warn!(
2563 path = %out_path.display(),
2564 ?error,
2565 "failed to export conformance wire-coverage evidence"
2566 );
2567 }
2568 }
2569
2570 serve_result?;
2571 snapshot_result?;
2572 Ok(())
2573}
2574
2575fn export_wire_coverage(recorder: &WireCoverageRecorder, path: &Path) -> Result<()> {
2586 let record = recorder.snapshot();
2587 let row_count = record.rows.len();
2588 let json = serde_json::to_string_pretty(&record)
2589 .context("failed to serialize wire-coverage record to JSON")?;
2590 std::fs::write(path, json).with_context(|| {
2591 format!(
2592 "failed to write wire-coverage evidence to {}",
2593 path.display()
2594 )
2595 })?;
2596 info!(
2597 path = %path.display(),
2598 rows = row_count,
2599 "wrote conformance wire-coverage evidence"
2600 );
2601 Ok(())
2602}
2603
2604fn log_build_info(process: &'static str) {
2605 let info = tokeira_build_info::summary();
2606 tracing::info!(
2607 process,
2608 server_version = tokeira_build_info::SERVER_VERSION,
2609 tokeira_version = info.tokeira_version,
2610 tokeira_git_sha = info.tokeira_git_sha,
2611 tokeira_source_revision = info.tokeira_source_revision,
2612 temporal_proto_version = info.temporal_proto_version,
2613 temporal_server_compat = info.temporal_server_compat,
2614 rust_toolchain = info.rust_toolchain,
2615 source_tree_hash = info.source_tree_hash,
2616 feature_matrix_digest = info.feature_matrix_digest,
2617 sdk_matrix_digest = info.sdk_matrix_digest,
2618 build_mode = info.build_mode,
2619 "tokeira build provenance"
2620 );
2621}
2622
2623fn render_build_info(verbose: bool, json: bool) -> String {
2624 let info = tokeira_build_info::summary();
2625 if json {
2626 return serde_json::json!({
2627 "server_version": tokeira_build_info::SERVER_VERSION,
2628 "tokeira_version": info.tokeira_version,
2629 "tokeira_git_sha": info.tokeira_git_sha,
2630 "source_revision": info.tokeira_source_revision,
2631 "temporal_proto_version": info.temporal_proto_version,
2632 "temporal_server_compat": info.temporal_server_compat,
2633 "rust_toolchain": info.rust_toolchain,
2634 "source_tree_hash": info.source_tree_hash,
2635 "feature_matrix_digest": info.feature_matrix_digest,
2636 "sdk_matrix_digest": info.sdk_matrix_digest,
2637 "build_mode": info.build_mode,
2638 })
2639 .to_string();
2640 }
2641
2642 if verbose {
2643 return [
2644 format!("server_version: {}", tokeira_build_info::SERVER_VERSION),
2645 format!("tokeira_version: {}", info.tokeira_version),
2646 format!("tokeira_git_sha: {}", info.tokeira_git_sha),
2647 format!("tokeira_source_revision: {}", info.tokeira_source_revision),
2648 format!("temporal_proto_version: {}", info.temporal_proto_version),
2649 format!("temporal_server_compat: {}", info.temporal_server_compat),
2650 format!("rust_toolchain: {}", info.rust_toolchain),
2651 format!("source_tree_hash: {}", info.source_tree_hash),
2652 format!("feature_matrix_digest: {}", info.feature_matrix_digest),
2653 format!("sdk_matrix_digest: {}", info.sdk_matrix_digest),
2654 format!("build_mode: {}", info.build_mode),
2655 ]
2656 .join("\n");
2657 }
2658
2659 format!(
2660 "tokeira {}\ngit {}\ntemporal_proto {}\ntemporal_server {}",
2661 tokeira_build_info::SERVER_VERSION,
2662 info.tokeira_git_sha,
2663 info.temporal_proto_version,
2664 info.temporal_server_compat
2665 )
2666}
2667
2668type ServerStack = (
2678 JoinHandle<Result<()>>,
2679 SocketAddr,
2680 oneshot::Sender<()>,
2681 CancellationToken,
2682 broadcast::Sender<LogEvent>,
2683 Option<Arc<WireCoverageRecorder>>,
2684);
2685
2686#[derive(Clone, Copy, Debug)]
2687enum StackTransport {
2688 Network(SocketAddr),
2689 Embedded,
2690}
2691
2692#[derive(Debug)]
2693enum ConstructedStack {
2694 Network(ServerStack),
2695 Embedded(EmbeddedStack),
2696}
2697
2698struct EmbeddedStack {
2699 component_visibility_store: chasm_extensions::VisibilityQueryStore,
2700 chasm_engine: Arc<tokeira_runtime::chasm::ChasmEngine>,
2701 registry: Arc<tokeira_chasm::Registry>,
2702 service: InProcessGrpcService,
2703 background_cancel: CancellationToken,
2704 log_broadcast: broadcast::Sender<LogEvent>,
2705 recovery_task: Option<JoinHandle<Result<()>>>,
2706 runtime_tasks: RuntimeShutdownHandle,
2707 engine_tasks: RuntimeShutdownHandle,
2708 shard_cleanup: Option<CleanupAction>,
2709}
2710
2711struct StackStartupGuard {
2712 runtime_tasks: RuntimeShutdownHandle,
2713 engine_tasks: RuntimeShutdownHandle,
2714 armed: bool,
2715}
2716
2717impl StackStartupGuard {
2718 fn new(runtime_tasks: RuntimeShutdownHandle, engine_tasks: RuntimeShutdownHandle) -> Self {
2719 Self {
2720 runtime_tasks,
2721 engine_tasks,
2722 armed: true,
2723 }
2724 }
2725
2726 fn disarm(&mut self) {
2727 self.armed = false;
2728 }
2729}
2730
2731impl Drop for StackStartupGuard {
2732 fn drop(&mut self) {
2733 if self.armed {
2734 self.engine_tasks.begin_shutdown();
2738 self.runtime_tasks.begin_shutdown();
2739 }
2740 }
2741}
2742
2743#[derive(Clone, Debug)]
2744struct SelfAssignedShardLease {
2745 shard_id: ShardId,
2746 owner: String,
2747 epoch: ShardEpoch,
2748}
2749
2750impl std::fmt::Debug for EmbeddedStack {
2751 fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
2752 formatter
2753 .debug_struct("EmbeddedStack")
2754 .field("service", &self.service)
2755 .field("background_cancel", &self.background_cancel)
2756 .field("runtime_tasks", &self.runtime_tasks)
2757 .field("engine_tasks", &self.engine_tasks)
2758 .field("has_shard_cleanup", &self.shard_cleanup.is_some())
2759 .finish_non_exhaustive()
2760 }
2761}
2762
2763const DEFAULT_MAX_ID_LENGTH: usize = 1000;
2767
2768fn refuse_distributed_migration_policy(config: &TokeiraConfig) -> Result<()> {
2777 match config.infrastructure.dsql.migration_policy {
2778 None | Some(DsqlMigrationPolicy::ValidateOnly) => Ok(()),
2779 Some(DsqlMigrationPolicy::Automatic) => Err(anyhow!(
2780 "infrastructure.dsql.migration_policy = \"automatic\" is not supported by \
2781 listener-backed distributed DSQL: a distributed tokeirad never migrates the \
2782 schema it shares with other nodes. Apply migrations with `tkr schema setup`, \
2783 then set the policy to \"validate_only\" or remove it"
2784 )),
2785 }
2786}
2787
2788async fn check_distributed_schema(dsql_store: &DsqlStore) -> Result<()> {
2799 let contract = MigrationRunner::compatibility_contract();
2800 let decision = {
2801 let mut permit = dsql_store
2802 .connection_director()
2803 .acquire(DbClass::Control)
2804 .await?;
2805 MigrationRunner::embedded()
2806 .assess_connection(
2807 permit.connection()?,
2808 &contract,
2809 SchemaMigrationPolicy::ValidateOnly,
2810 )
2811 .await
2812 .context("failed to read the DSQL schema's migration ledger")?
2813 };
2814 let current = distributed_schema_admission(decision)?;
2815 info!(
2816 schema_version = current,
2817 target_version = contract.target_version,
2818 maximum_readable_version = contract.maximum_readable_version,
2819 "DSQL schema check passed"
2820 );
2821 Ok(())
2822}
2823
2824fn distributed_schema_admission(decision: SchemaDecision) -> Result<u32> {
2829 match decision {
2830 SchemaDecision::Compatible { current, .. } => Ok(current),
2831 SchemaDecision::MigrationRequired { current: 0, target } => Err(anyhow!(
2832 "tokeirad refuses to start: the DSQL database has no Tokeira schema. Apply this \
2833 release's migrations, through V{target}, with `tkr schema setup`, then start \
2834 tokeirad"
2835 )),
2836 SchemaDecision::MigrationRequired { current, target } => Err(anyhow!(
2837 "tokeirad refuses to start: the DSQL schema is at V{current}, behind this \
2838 release's target V{target}. Apply the missing migrations with `tkr schema \
2839 setup`, then start tokeirad"
2840 )),
2841 SchemaDecision::Reject(incompatibility) => {
2842 let advice = match incompatibility.category {
2843 SchemaIncompatibilityCategory::FutureSchema => {
2844 "a later release migrated it; run that release or a later one"
2845 }
2846 SchemaIncompatibilityCategory::BelowMinimum => {
2847 "it is older than this release migrates from; upgrade it through an \
2848 earlier release first"
2849 }
2850 _ => {
2851 "its migration ledger doesn't match this release's migrations; find out \
2852 what changed it before any node runs on it"
2853 }
2854 };
2855 Err(
2856 anyhow::Error::new(SchemaCompatibilityError::Incompatible(incompatibility))
2857 .context(format!(
2858 "tokeirad refuses to start: this release can't run on the DSQL \
2859 schema, because {advice}"
2860 )),
2861 )
2862 }
2863 decision @ (SchemaDecision::Initialize { .. } | SchemaDecision::Migrate { .. }) => {
2866 Err(anyhow!(
2867 "tokeirad refuses to start: a validate-only schema assessment decided \
2868 {decision:?}"
2869 ))
2870 }
2871 }
2872}
2873
2874async fn build_and_serve(
2885 addr: SocketAddr,
2886 effective_config: Arc<TokeiraConfig>,
2887) -> Result<(ServerStack, Option<EngineSnapshotPolicy>)> {
2888 let mut snapshot_inputs = None;
2889 let stack = match effective_config.infrastructure.storage {
2890 ConfigStorageKind::InMemory => {
2891 let (store, restored) = restore_snapshot_store(&effective_config).await?;
2896 snapshot_inputs = effective_config
2897 .policy
2898 .snapshot
2899 .clone()
2900 .map(|config| (store.clone(), config));
2901 build_in_memory_stack(
2902 StackTransport::Network(addr),
2903 effective_config,
2904 store,
2905 restored,
2906 ChasmExtensions::default(),
2907 )
2908 .await
2909 }
2910 ConfigStorageKind::Dsql => {
2911 if effective_config.policy.snapshot.is_some() {
2912 return Err(anyhow!(
2913 "policy.snapshot is configured but infrastructure.storage is \"dsql\": \
2914 snapshots are the in-memory store's persistence mechanism, and DSQL \
2915 storage is already durable — remove policy.snapshot or switch \
2916 infrastructure.storage to \"memory\""
2917 ));
2918 }
2919 refuse_distributed_migration_policy(&effective_config)?;
2920 let auth = dsql_auth_config(&effective_config)?;
2921 let endpoint = auth.endpoint.clone();
2922 let (pool_config, ddb_client) = dsql_pool_config(&effective_config, &auth).await?;
2923 let dsql_store = DsqlStore::connect(auth, pool_config, ddb_client)
2924 .await
2925 .context("failed to connect DSQL storage backend")?;
2926 if let Err(refusal) = check_distributed_schema(&dsql_store).await {
2931 if let Err(error) = dsql_store.shutdown().await {
2932 tracing::warn!(
2933 error = %format!("{error:#}"),
2934 "failed to shut DSQL storage down after refusing its schema"
2935 );
2936 }
2937 return Err(refusal);
2938 }
2939 build_dsql_stack(
2940 StackTransport::Network(addr),
2941 effective_config,
2942 dsql_store,
2943 endpoint,
2944 ChasmExtensions::default(),
2945 )
2946 .await
2947 }
2948 }?;
2949 match stack {
2950 ConstructedStack::Network(stack) => {
2951 let snapshot_policy = snapshot_inputs
2955 .map(|(store, config)| EngineSnapshotPolicy::start(store, config, stack.3.clone()));
2956 Ok((stack, snapshot_policy))
2957 }
2958 ConstructedStack::Embedded(_) => Err(anyhow!(
2959 "network service construction returned an embedded stack"
2960 )),
2961 }
2962}
2963
2964async fn build_dsql_stack(
2965 transport: StackTransport,
2966 effective_config: Arc<TokeiraConfig>,
2967 dsql_store: DsqlStore,
2968 endpoint: String,
2969 extensions: ChasmExtensions,
2970) -> Result<ConstructedStack> {
2971 let (director, run_repository, projection_log, worker_deployment_repository, _migration_runner) =
2972 dsql_store.into_parts();
2973 let budget_applier: Arc<dyn ConnectionBudgetApplier> = Arc::new(DsqlConnectionBudgetApplier {
2977 director: director.clone(),
2978 });
2979 let chasm_node_repo = Arc::new(tokeira_storage::dsql::DsqlChasmNodeRepository::new(
2980 director.clone(),
2981 ));
2982 let task_queue_config_repository: Arc<dyn TaskQueueConfigRepository> = Arc::new(
2983 tokeira_storage::dsql::DsqlTaskQueueConfigRepository::new(director.clone()),
2984 );
2985 let worker_compute_repository = effective_config.policy.worker_compute.enabled.then(|| {
2986 Arc::new(tokeira_storage::dsql::DsqlWorkerComputeRepository::new(
2987 director.clone(),
2988 )) as Arc<dyn WorkerComputeRepository>
2989 });
2990 let worker_task_provenance: Arc<dyn WorkerTaskProvenanceStore> = Arc::new(
2991 tokeira_storage::dsql::DsqlWorkerTaskProvenanceStore::new(director.clone()),
2992 );
2993 let visibility_store = DsqlVisibilityStore::new(director);
2994 let worker_deployment_repository: Arc<dyn WorkerDeploymentRepository> =
2995 Arc::new(worker_deployment_repository);
2996 build_service_stack_with_storage(
2997 transport,
2998 effective_config,
2999 Arc::new(run_repository),
3000 worker_deployment_repository,
3001 worker_compute_repository,
3002 task_queue_config_repository,
3003 worker_task_provenance,
3004 projection_log,
3005 visibility_store.clone(),
3006 {
3007 let visibility_store = visibility_store.clone();
3008 move || visibility_store.clone()
3009 },
3010 Some(endpoint),
3011 chasm_node_repo,
3012 false,
3013 budget_applier,
3014 extensions,
3015 )
3016 .await
3017}
3018
3019async fn build_embedded(
3020 effective_config: Arc<TokeiraConfig>,
3021 store: InMemoryStore,
3022 recover_self_assigned_shard: bool,
3023 extensions: ChasmExtensions,
3024) -> Result<EmbeddedStack> {
3025 match build_in_memory_stack(
3026 StackTransport::Embedded,
3027 effective_config,
3028 store,
3029 recover_self_assigned_shard,
3030 extensions,
3031 )
3032 .await?
3033 {
3034 ConstructedStack::Embedded(stack) => Ok(stack),
3035 ConstructedStack::Network(_) => Err(anyhow!(
3036 "embedded service construction returned a network stack"
3037 )),
3038 }
3039}
3040
3041async fn build_in_memory_stack(
3042 transport: StackTransport,
3043 effective_config: Arc<TokeiraConfig>,
3044 store: InMemoryStore,
3045 recover_self_assigned_shard: bool,
3046 extensions: ChasmExtensions,
3047) -> Result<ConstructedStack> {
3048 let visibility_store = InMemoryVisibilityStore::default();
3049 let chasm_node_repo: Arc<dyn tokeira_storage::ChasmNodeRepository> =
3050 Arc::new(tokeira_storage::InMemoryChasmNodeStore::new());
3051 #[cfg(test)]
3052 let (chasm_node_repo, visibility_store) = extensions
3053 .test_storage
3054 .clone()
3055 .unwrap_or((chasm_node_repo, visibility_store));
3056 let worker_deployment_repository: Arc<dyn WorkerDeploymentRepository> = Arc::new(store.clone());
3057 let worker_task_provenance: Arc<dyn WorkerTaskProvenanceStore> = Arc::new(store.clone());
3058 let worker_compute_repository = effective_config.policy.worker_compute.enabled.then(|| {
3059 Arc::new(InMemoryWorkerComputeRepository::default()) as Arc<dyn WorkerComputeRepository>
3060 });
3061 build_service_stack_with_storage(
3062 transport,
3063 effective_config,
3064 Arc::new(store.clone()),
3065 worker_deployment_repository,
3066 worker_compute_repository,
3067 Arc::new(store.clone()),
3068 worker_task_provenance,
3069 store,
3070 visibility_store.clone(),
3071 {
3072 let visibility_store = visibility_store.clone();
3073 move || VisibilitySink::new(visibility_store.clone())
3074 },
3075 None,
3076 chasm_node_repo,
3077 recover_self_assigned_shard,
3078 Arc::new(NoopConnectionBudgetApplier),
3080 extensions,
3081 )
3082 .await
3083}
3084
3085const VISIBILITY_REPAIR_INTERVAL: std::time::Duration = std::time::Duration::from_secs(30);
3088
3089const CHASM_REBUILD_INTERVAL: std::time::Duration = std::time::Duration::from_secs(30);
3091
3092const WORKER_TASK_PROVENANCE_CLEANUP_INTERVAL: std::time::Duration =
3096 std::time::Duration::from_secs(30);
3097const WORKER_TASK_PROVENANCE_CLEANUP_BATCH: usize = 256;
3098
3099fn spawn_worker_task_provenance_cleanup(
3100 tasks: &RuntimeShutdownHandle,
3101 store: Arc<dyn WorkerTaskProvenanceStore>,
3102 cancel: CancellationToken,
3103) -> JoinHandle<()> {
3104 tasks.spawn(async move {
3105 let mut interval = tokio::time::interval(WORKER_TASK_PROVENANCE_CLEANUP_INTERVAL);
3106 interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
3107 loop {
3108 tokio::select! {
3109 _ = cancel.cancelled() => break,
3110 _ = interval.tick() => {
3111 if let Err(error) = store
3112 .delete_expired(
3113 time::OffsetDateTime::now_utc(),
3114 WORKER_TASK_PROVENANCE_CLEANUP_BATCH,
3115 )
3116 .await
3117 {
3118 tracing::warn!(
3119 error_kind = worker_task_provenance_cleanup_error_kind(&error),
3120 "Worker task authorization-evidence cleanup pass failed"
3121 );
3122 }
3123 }
3124 }
3125 }
3126 })
3127}
3128
3129fn worker_task_provenance_cleanup_error_kind(
3130 error: &tokeira_storage::WorkerTaskProvenanceError,
3131) -> &'static str {
3132 match error {
3133 tokeira_storage::WorkerTaskProvenanceError::Unavailable { .. } => "unavailable",
3134 tokeira_storage::WorkerTaskProvenanceError::DigestConflict => "conflict",
3135 tokeira_storage::WorkerTaskProvenanceError::Corrupt { .. } => "corrupt",
3136 }
3137}
3138
3139fn spawn_visibility_repair(
3145 tasks: &RuntimeShutdownHandle,
3146 nodes: Arc<dyn tokeira_storage::ChasmNodeRepository>,
3147 sink: Arc<dyn tokeira_projection::ProjectionSink>,
3148 registry: Arc<tokeira_chasm::Registry>,
3149 partition_count: u32,
3150 cancel: CancellationToken,
3151) {
3152 let rebuild: tokeira_runtime::chasm::SnapshotRebuilder =
3153 Arc::new(move |archetype_id, bytes| {
3154 match registry.visibility_snapshot(archetype_id, bytes) {
3155 Ok(snapshot) => snapshot,
3156 Err(error) => {
3157 tracing::warn!(
3158 ?error,
3159 archetype_id,
3160 "CHASM visibility rebuild decode failed"
3161 );
3162 None
3163 }
3164 }
3165 });
3166 let scanner =
3167 tokeira_runtime::chasm::VisibilityRepairScanner::new(nodes, sink, rebuild, partition_count);
3168 let _repair = tasks.spawn(async move {
3169 let mut interval = tokio::time::interval(VISIBILITY_REPAIR_INTERVAL);
3170 interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
3171 loop {
3172 tokio::select! {
3173 _ = cancel.cancelled() => break,
3174 _ = interval.tick() => {
3175 if let Err(error) = scanner.repair_once().await {
3176 tracing::warn!(?error, "visibility repair pass failed");
3177 }
3178 }
3179 }
3180 }
3181 });
3182}
3183
3184fn spawn_outbox_rebuild(
3187 tasks: &RuntimeShutdownHandle,
3188 scanner: tokeira_runtime::chasm::OutboxRebuildScanner,
3189 cancel: CancellationToken,
3190) {
3191 let _rebuild = tasks.spawn(async move {
3192 let mut interval = tokio::time::interval(CHASM_REBUILD_INTERVAL);
3193 interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
3194 interval.tick().await;
3196 loop {
3197 tokio::select! {
3198 _ = cancel.cancelled() => break,
3199 _ = interval.tick() => {
3200 tokio::select! {
3201 _ = cancel.cancelled() => break,
3202 result = scanner.rebuild_once() => {
3203 match result {
3204 Ok(stats) if stats.unserviceable > 0 => tracing::error!(unserviceable = stats.unserviceable, first = ?stats.first_unserviceable, "CHASM rebuild isolated executions; healthy executions were rebuilt"),
3205 Ok(_) => {},
3206 Err(error) => tracing::warn!(?error, "CHASM outbox rebuild pass failed"),
3207 }
3208 }
3209 }
3210 }
3211 }
3212 }
3213 });
3214}
3215
3216const CHASM_TIMER_SWEEP_INTERVAL: std::time::Duration = std::time::Duration::from_millis(200);
3221
3222fn spawn_chasm_timer_sweeper(
3231 tasks: &RuntimeShutdownHandle,
3232 engine: Arc<tokeira_runtime::chasm::ChasmEngine>,
3233 evaluator: Arc<dyn tokeira_runtime::chasm::TimeoutEvaluator>,
3234 archetype_id: u32,
3235 cancel: CancellationToken,
3236) {
3237 let sweeper = tokeira_runtime::chasm::ChasmTimerSweeper::new(engine)
3238 .with_evaluator(archetype_id, evaluator);
3239 let _sweeper = tasks.spawn(async move {
3240 let mut interval = tokio::time::interval(CHASM_TIMER_SWEEP_INTERVAL);
3241 interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
3242 loop {
3243 tokio::select! {
3244 _ = cancel.cancelled() => break,
3245 _ = interval.tick() => {
3246 sweeper.sweep_once().await;
3247 }
3248 }
3249 }
3250 });
3251}
3252
3253async fn build_service_stack_with_storage<R, L, S, V, F>(
3254 transport: StackTransport,
3255 effective_config: Arc<TokeiraConfig>,
3256 run_repository: Arc<R>,
3257 worker_deployment_repository: Arc<dyn WorkerDeploymentRepository>,
3258 worker_compute_repository: Option<Arc<dyn WorkerComputeRepository>>,
3259 task_queue_config_repository: Arc<dyn TaskQueueConfigRepository>,
3260 worker_task_provenance: Arc<dyn WorkerTaskProvenanceStore>,
3261 projection_log: L,
3262 visibility_query_store: V,
3263 projection_sink: F,
3264 dsql_endpoint: Option<String>,
3265 chasm_node_repo: Arc<dyn tokeira_storage::ChasmNodeRepository>,
3266 recover_self_assigned_shard: bool,
3267 budget_applier: Arc<dyn ConnectionBudgetApplier>,
3268 extensions: ChasmExtensions,
3269) -> Result<ConstructedStack>
3270where
3271 R: LeaseRepository + RunRepository + 'static,
3272 L: ProjectionLog + Clone + 'static,
3273 S: ProjectionSink + VisibilityStore + 'static,
3274 V: VisibilityStore + Clone + 'static,
3275 F: Fn() -> S + Clone + Send + Sync + 'static,
3276{
3277 tokeira_storage::prepare_execution_placement(run_repository.as_ref()).await?;
3280 let has_extensions = !extensions.libraries.is_empty();
3284 let mut registry_builder = tokeira_chasm::Registry::builder();
3285 tokeira_chasm_activity::ActivityLibrary::register(&mut registry_builder)
3286 .map_err(registry_start_error)?;
3287 registry_builder.seal_built_ins();
3288 for library in extensions.libraries {
3289 library(&mut registry_builder).map_err(registry_start_error)?;
3290 }
3291 let registry = Arc::new(registry_builder.build());
3292 let declared = declared_search_attributes(®istry);
3293 let node_id = IncarnationId::new();
3296 let advertised_addr = match transport {
3297 StackTransport::Network(addr) => addr,
3298 StackTransport::Embedded => SocketAddr::from(([127, 0, 0, 1], 0)),
3299 };
3300 let node_endpoint = configured_node_endpoint(&effective_config, advertised_addr);
3301 let history_waits = HistoryWaitRegistry::default();
3302 let repo = Arc::new(HistoryNotifyingRepository::new(
3303 run_repository,
3304 history_waits.clone(),
3305 ));
3306 let default_namespace = ResolvedNamespace::active("default");
3307 let default_namespace_id = namespace_id_for("default");
3308
3309 let namespaces = Arc::new(InMemoryNamespaceCache::new());
3311 namespaces.insert(default_namespace).await?;
3312 let nexus_store = build_nexus_endpoint_store(
3316 namespaces.as_ref(),
3317 HashMap::<String, BootstrapNexusEndpointConfig>::new(),
3318 )
3319 .await?;
3320 let nexus_registry = NexusEndpointRegistry::new(nexus_store.clone());
3321 let nexus_http_client: Arc<dyn tokeira_runtime::NexusHttpClient> =
3322 Arc::new(HttpNexusClient::new());
3323
3324 let schedule_store = Arc::new(ScheduleStore::default());
3327 let task_queue_config_store = Arc::new(RepositoryBackedTaskQueueConfigStore::new(
3328 task_queue_config_repository,
3329 ));
3330 task_queue_config_store
3333 .hydrate()
3334 .await
3335 .context("failed to hydrate task-queue policy")?;
3336 let initial_routing_cache = if let Some(endpoint) = &effective_config
3340 .infrastructure
3341 .placement
3342 .controller_endpoint
3343 {
3344 let cache = Arc::new(RoutingCache::new(placement_config(&effective_config)));
3345 tokio::time::timeout(
3346 std::time::Duration::from_secs(10),
3347 tokeira_edge::subscribe_routing_once(cache.clone(), endpoint.clone()),
3348 )
3349 .await
3350 .context("timed out waiting for controller placement snapshot")??;
3351 Some(cache)
3352 } else {
3353 None
3354 };
3355 let mut runtime_config = RuntimeConfig::default();
3356 runtime_config.lane.controller_managed_placement = effective_config
3357 .infrastructure
3358 .placement
3359 .controller_endpoint
3360 .is_some();
3361 let seed_default_shard = !recover_self_assigned_shard
3362 && dsql_endpoint.is_none()
3363 && effective_config
3364 .infrastructure
3365 .placement
3366 .controller_endpoint
3367 .is_none();
3368
3369 let nexus_completion_cfg = effective_config.policy.nexus_completion.clone();
3370 let (
3371 nexus_callback_listener,
3372 in_process_nexus_client,
3373 nexus_completion_client,
3374 resolved_system_callback_url,
3375 ): (
3376 Option<TcpListener>,
3377 Option<Arc<InProcessNexusCompletionClient>>,
3378 Arc<dyn NexusCompletionClient>,
3379 String,
3380 ) = match transport {
3381 StackTransport::Network(_) => {
3382 let listener = TcpListener::bind(&nexus_completion_cfg.http_addr)
3385 .await
3386 .with_context(|| {
3387 format!(
3388 "failed to bind nexus completion listener on {}",
3389 nexus_completion_cfg.http_addr
3390 )
3391 })?;
3392 let callback_addr = listener
3393 .local_addr()
3394 .context("failed to resolve bound nexus completion listener address")?;
3395 let callback_url = with_loopback_port(
3396 &nexus_completion_cfg.system_callback_url,
3397 callback_addr.port(),
3398 );
3399 info!(
3400 bind = %callback_addr,
3401 loopback = %callback_url,
3402 "nexus completion callback listener bound"
3403 );
3404 (
3405 Some(listener),
3406 None,
3407 Arc::new(HttpNexusCompletionClient::new()),
3408 callback_url,
3409 )
3410 }
3411 StackTransport::Embedded => {
3412 let client = Arc::new(InProcessNexusCompletionClient::new());
3415 (
3416 None,
3417 Some(client.clone()),
3418 client,
3419 EMBEDDED_NEXUS_CALLBACK_BASE.to_owned(),
3420 )
3421 }
3422 };
3423 let nexus_completion_deps = NexusCompletionDeps {
3424 client: nexus_completion_client,
3425 config: NexusCompletionRuntimeConfig {
3426 system_callback_url: resolved_system_callback_url,
3427 retry_initial_interval: time::Duration::milliseconds(
3428 nexus_completion_cfg.retry_initial_interval_ms as i64,
3429 ),
3430 retry_max_interval: time::Duration::milliseconds(
3431 nexus_completion_cfg.retry_max_interval_ms as i64,
3432 ),
3433 retry_backoff_coefficient: nexus_completion_cfg.retry_backoff_coefficient,
3434 retry_max_attempts: nexus_completion_cfg.retry_max_attempts,
3435 },
3436 scanner: CompletionCallbackScannerConfig::default(),
3437 };
3438
3439 let chasm_nexus_client = nexus_completion_deps.client.clone();
3440 let chasm_nexus_config = nexus_completion_deps.config.clone();
3441
3442 let worker_compute_deployment_repository = worker_deployment_repository.clone();
3443 let runtime = TokeiraRuntime::new_with_nexus_and_shards_and_endpoint(
3444 repo.clone(),
3445 runtime_config.lane_count,
3446 runtime_config.lane,
3447 runtime_config.timer_scanner,
3448 runtime_config.workflow_timeout_scanner,
3449 runtime_config.backlog,
3450 runtime_config.activity_timeout_scanner,
3451 runtime_config.nexus_timeout_scanner,
3452 nexus_registry.clone(),
3453 nexus_http_client.clone(),
3454 nexus_completion_deps,
3457 effective_config.infrastructure.placement.shard_count,
3458 node_id.to_string(),
3459 node_endpoint.as_authority(),
3460 seed_default_shard,
3461 Some(Arc::new(CacheNexusNamespaceResolver {
3465 namespaces: namespaces.clone(),
3466 }) as Arc<dyn NexusNamespaceResolver>),
3467 )
3468 .with_queue_placement(
3472 effective_config.infrastructure.placement.partition_count,
3473 effective_config.infrastructure.placement.bundle_count,
3474 )
3475 .with_worker_deployment_repository(worker_deployment_repository)
3476 .with_task_queue_config_store(task_queue_config_store.clone())
3477 .with_delivery_mode_provider(Arc::new(
3478 tokeira_runtime::ConfiguredDeliveryModeProvider::new(
3479 tokeira_runtime::StaticDeliveryPolicy {
3480 enable_fairness: effective_config.policy.task_queues.enable_fairness,
3481 },
3482 ),
3483 ));
3484 runtime.ready().await?;
3485 let runtime = Arc::new(runtime);
3486 let runtime_tasks = runtime.shutdown_handle();
3487 let engine_tasks = RuntimeShutdownHandle::new();
3488 let mut startup_guard = StackStartupGuard::new(runtime_tasks.clone(), engine_tasks.clone());
3489 let background_cancel = engine_tasks.cancellation_token();
3490
3491 let recovered_lease = if recover_self_assigned_shard {
3492 let shard_id = ShardId(0);
3493 let epoch = runtime
3494 .acquire_shard(shard_id)
3495 .await
3496 .context("failed to recover embedded snapshot shard")?;
3497 Some((shard_id, node_id.to_string(), epoch))
3498 } else {
3499 None
3500 };
3501
3502 if matches!(transport, StackTransport::Network(_))
3503 && dsql_endpoint.is_some()
3504 && effective_config
3505 .infrastructure
3506 .placement
3507 .controller_endpoint
3508 .is_none()
3509 {
3510 let _leases = self_assign_dsql_shards(
3511 runtime.as_ref(),
3512 repo.as_ref(),
3513 effective_config.infrastructure.placement.shard_count,
3514 &node_id,
3515 &node_endpoint,
3516 false,
3517 )
3518 .await?;
3519 }
3520
3521 let recovery_task = recovered_lease.map(|(shard_id, owner, epoch)| {
3522 let runtime = runtime.clone();
3523 let repo = repo.clone();
3524 let cancel = background_cancel.clone();
3525 tokio::spawn(async move {
3526 cancel.cancelled().await;
3527 runtime.relinquish_shard(shard_id).await;
3532 match repo
3533 .relinquish_bundle(shard_id, owner, epoch)
3534 .await
3535 .context("failed to release embedded snapshot recovery lease")?
3536 {
3537 LeaseOutcome::Acquired { .. } => Ok(()),
3538 LeaseOutcome::Rejected {
3539 current_owner,
3540 current_epoch,
3541 } => Err(anyhow!(
3542 "embedded snapshot recovery lease was fenced by owner {current_owner} at epoch {}",
3543 current_epoch.0
3544 )),
3545 LeaseOutcome::Renewed { epoch } => Err(anyhow!(
3546 "embedded snapshot recovery lease unexpectedly renewed at epoch {}",
3547 epoch.0
3548 )),
3549 }
3550 })
3551 });
3552 let task_queue_config_refresh = task_queue_config_store
3553 .clone()
3554 .spawn_refresh(background_cancel.clone());
3555 let _task_queue_config_refresh = engine_tasks.track_join(task_queue_config_refresh);
3556 let _worker_task_provenance_cleanup = spawn_worker_task_provenance_cleanup(
3557 &engine_tasks,
3558 worker_task_provenance.clone(),
3559 background_cancel.clone(),
3560 );
3561
3562 let _worker_compute_service = if effective_config.policy.worker_compute.enabled {
3563 let controller_repository = worker_compute_repository
3564 .clone()
3565 .context("worker-compute enabled without a controller repository")?;
3566 let (observation_sender, observation_receiver) =
3567 mpsc::channel(OBSERVATION_CHANNEL_CAPACITY);
3568 let observation_sink = Arc::new(ChannelDemandObservationSink::new(observation_sender));
3569 runtime
3570 .broker()
3571 .set_demand_observation_sink(observation_sink.clone());
3572 runtime
3573 .activity_broker()
3574 .set_demand_observation_sink(observation_sink.clone());
3575 runtime
3576 .nexus_task_broker()
3577 .set_demand_observation_sink(observation_sink);
3578
3579 let (reconcile_sender, reconcile_receiver) = mpsc::channel(RECONCILE_CHANNEL_CAPACITY);
3580 runtime
3581 .deployment_registry()
3582 .expect("Worker Deployment repository was installed above")
3583 .set_worker_compute_reconcile_sink(Arc::new(ChannelWorkerComputeReconcileSink::new(
3584 reconcile_sender,
3585 )));
3586
3587 let clock = Arc::new(SystemWorkerComputeClock);
3588 let catalog = Arc::new(WorkerComputeNamespaceCatalogAdapter::new(
3589 namespaces.clone(),
3590 ));
3591 let reconciler = WorkerComputeReconciler::new(
3592 worker_compute_deployment_repository,
3593 controller_repository.clone(),
3594 node_id,
3595 );
3596 let sampler = runtime.worker_compute_queue_sampler(controller_repository.clone(), node_id);
3597 let provider = Arc::new(NexusWorkerComputeProvider::new(
3598 nexus_registry,
3599 nexus_http_client,
3600 runtime.nexus_task_broker(),
3601 ));
3602 let outbox = WorkerComputeOutbox::new(
3603 controller_repository.clone(),
3604 provider,
3605 clock.clone(),
3606 node_id,
3607 );
3608 let shard_runtime = runtime.clone();
3609 let active_shards: tokeira_runtime::WorkerComputeActiveShards =
3610 Arc::new(move || shard_runtime.active_shards());
3611 let service = WorkerComputeControllerService::new(
3612 catalog,
3613 controller_repository,
3614 reconciler,
3615 sampler,
3616 outbox,
3617 clock,
3618 active_shards,
3619 observation_receiver,
3620 reconcile_receiver,
3621 );
3622 let service_cancel = background_cancel.clone();
3623 Some(engine_tasks.spawn(async move {
3624 if let Err(error) = service.run(service_cancel).await {
3625 tracing::warn!(?error, "worker-compute controller service exited");
3626 }
3627 }))
3628 } else {
3629 None
3630 };
3631
3632 let membership_client = effective_config
3633 .infrastructure
3634 .placement
3635 .controller_endpoint
3636 .clone()
3637 .map(|controller_endpoint| {
3638 runtime.spawn_membership_client(
3639 membership_config(
3640 &effective_config,
3641 node_id,
3642 node_endpoint.clone(),
3643 controller_endpoint,
3644 ),
3645 budget_applier,
3646 background_cancel.clone(),
3647 )
3648 });
3649 if let Some(membership_client) = membership_client {
3650 let _membership_client = engine_tasks.track_join(membership_client);
3651 }
3652 let schedule_engine = run_schedule_engine(
3653 schedule_store.clone(),
3654 runtime.clone(),
3655 ScheduleEngineConfig::default(),
3656 background_cancel.clone(),
3657 );
3658 let _schedule_engine = engine_tasks.track_join(schedule_engine);
3659
3660 let authorization = build_authorization_stack(&effective_config).await?;
3661 let interceptors = Arc::new(EdgeInterceptors::configured(
3662 namespaces.clone(),
3663 authorization.grpc,
3664 authorization.principal_attribution,
3665 ));
3666 let router: Arc<dyn tokeira_edge::EdgeRouter> = if let Some(controller_endpoint) =
3667 effective_config
3668 .infrastructure
3669 .placement
3670 .controller_endpoint
3671 .clone()
3672 {
3673 let routing_cache = initial_routing_cache
3674 .expect("controller snapshot validated before runtime construction");
3675 let routing_config = EdgeRoutingConfig {
3676 controller_endpoint,
3677 max_retries: effective_config
3678 .infrastructure
3679 .placement
3680 .routing_max_retries,
3681 };
3682 let subscription_cache = routing_cache.clone();
3683 let subscription_cancel = background_cancel.clone();
3684 let routing_shutdown = runtime.shutdown_handle();
3685 let routing_cancel = background_cancel.clone();
3686 let _routing_subscription = engine_tasks.spawn(async move {
3687 if let Err(error) =
3688 run_routing_subscription(subscription_cache, routing_config, subscription_cancel)
3689 .await
3690 {
3691 tracing::error!(
3692 ?error,
3693 "routing placement became incompatible; stopping node"
3694 );
3695 routing_shutdown.begin_shutdown();
3696 routing_cancel.cancel();
3697 }
3698 });
3699 runtime.set_workflow_task_starter(Arc::new(
3700 tokeira_edge::grpc::workflow_dispatch::RemoteWorkflowTaskStarter::new(
3701 routing_cache.clone(),
3702 ),
3703 ));
3704 Arc::new(CacheBackedRouter::new(routing_cache, node_id))
3705 } else {
3706 Arc::new(LocalOnlyRouter)
3707 };
3708 let workflow_broker = runtime.broker();
3709 let buffered_queries = runtime.buffered_queries();
3710 let worker_registry = runtime.worker_registry();
3711 let nexus_task_broker = runtime.nexus_task_broker();
3712 let runtime_adapter = Arc::new(RuntimeAdapter::new(runtime.clone()));
3713 let callback_runtime = runtime_adapter.clone() as Arc<dyn WorkflowRuntimeApi>;
3714 if let Some(client) = in_process_nexus_client {
3715 client.attach(&callback_runtime)?;
3716 }
3717 if let Some(listener) = nexus_callback_listener {
3718 spawn_nexus_callback_server(listener, callback_runtime, background_cancel.clone());
3721 }
3722 let resolver = Arc::new(StoreExecutionResolver::new(repo.clone()));
3723 tokeira_projection::seed_predefined_search_attributes(
3724 &visibility_query_store,
3725 default_namespace_id,
3726 )
3727 .await
3728 .context("failed to seed Temporal predefined search attributes")?;
3729 let operator_visibility_store = visibility_query_store.clone();
3730 let visibility = Arc::new(VisibilityQueryService::new(visibility_query_store.clone()));
3731 let long_polls = LongPollGate::new(LongPollConfig::default());
3732 let operator_api = Arc::new(
3733 VisibilityRegistryOperatorApi::new(
3734 InMemoryOperatorApi::new("tokeira-local", tokeira_build_info::SERVER_VERSION),
3735 operator_visibility_store,
3736 )
3737 .with_declared_search_attributes(declared.clone()),
3738 );
3739
3740 for partition_id in 0..effective_config.infrastructure.placement.partition_count {
3741 let projection_worker = ProjectionWorker {
3742 log: projection_log.clone(),
3743 sink: projection_sink(),
3744 batch_size: 256,
3745 };
3746 let projection_cancel = background_cancel.clone();
3747 let _projection_worker = engine_tasks.spawn(async move {
3748 if let Err(error) = projection_worker
3749 .run_from_cursor(
3750 projection_cancel,
3751 ProjectionCursor::beginning(partition_id, 1),
3752 )
3753 .await
3754 {
3755 tracing::warn!(?error, partition_id, "projection worker exited");
3756 }
3757 });
3758 }
3759
3760 let nexus_http_waiters = tokeira_edge::nexus_http::NexusHttpWaiterRegistry::default();
3761 let workflow_service =
3762 WorkflowService::new_with_stores_and_buffered_queries_and_history_wait_registry(
3763 runtime_adapter.clone(),
3764 resolver,
3765 visibility,
3766 repo.clone(),
3767 operator_api.clone(),
3768 namespaces.clone(),
3769 interceptors.clone(),
3770 PollerRegistry::default(),
3771 PendingQueryStore::default(),
3772 buffered_queries,
3773 workflow_broker,
3774 nexus_task_broker.clone(),
3775 long_polls,
3776 router,
3777 history_waits,
3778 worker_registry,
3779 runtime.heartbeat_store(),
3780 schedule_store,
3781 task_queue_config_store,
3782 Arc::new(tokeira_runtime::BatchOperationStore::default()),
3783 )
3784 .with_nexus_http_waiters(nexus_http_waiters.clone())
3785 .with_worker_task_provenance(worker_task_provenance)
3786 .with_scoped_worker_sessions(ScopedWorkerSessionRegistry::default())
3787 .with_worker_deployment_runtime(runtime_adapter);
3788 let nexus_endpoint_limits = {
3793 let cfg = &effective_config.policy.nexus_endpoint_limits;
3794 tokeira_edge::nexus_endpoint::NexusEndpointLimits {
3795 name_max_length: cfg.name_max_length,
3796 external_url_max_length: cfg.external_url_max_length,
3797 description_max_size: cfg.description_max_size,
3798 task_queue_max_length: cfg.task_queue_max_length,
3799 list_default_page_size: cfg.list_default_page_size,
3800 list_max_page_size: cfg.list_max_page_size,
3801 }
3802 };
3803 let nexus_endpoint_admin = Arc::new(tokeira_edge::nexus_endpoint::NexusEndpointAdmin::new(
3804 nexus_store.clone(),
3805 Arc::new(tokeira_edge::nexus_endpoint::CacheNamespaceResolver::new(
3806 namespaces.clone(),
3807 )),
3808 nexus_endpoint_limits,
3809 ));
3810 let dispatch_grpc = tokeira_edge::grpc::workflow_dispatch::WorkflowTaskDispatchGrpc::new(
3811 runtime.clone(),
3812 interceptors.clone(),
3813 workflow_service.clone(),
3814 );
3815 let operator_service = OperatorService::new(operator_api, interceptors)
3816 .with_nexus_endpoints(nexus_endpoint_admin)
3817 .with_namespace_deletion(namespaces.clone(), Arc::new(workflow_service.clone()));
3818 let production_nexus_authorizer = authorization.nexus;
3819 let nexus_http_authorizer: Arc<dyn tokeira_edge::nexus_http::NexusHttpAuthorizer> =
3820 match harness::installed().and_then(|hooks| hooks.wrap_nexus_http_authorizer.as_ref()) {
3821 Some(wrap) => wrap(production_nexus_authorizer),
3822 None => production_nexus_authorizer,
3823 };
3824 let nexus_http_layer =
3825 NexusHttpLayer::new(Arc::new(tokeira_edge::nexus_http::NexusHttpHandler::new(
3826 namespaces.clone(),
3827 nexus_store,
3828 nexus_task_broker,
3829 nexus_http_waiters,
3830 nexus_http_authorizer,
3831 )));
3832 let http_api_layer = HttpApiLayer::new(
3833 HttpApiCatalog::pinned().context("failed to build Temporal HTTP API route catalog")?,
3834 HttpApiPolicy::new(
3835 effective_config.policy.http_api.allowed_hosts.clone(),
3836 effective_config
3837 .policy
3838 .http_api
3839 .additional_forwarded_headers
3840 .clone(),
3841 )
3842 .context("failed to compile Temporal HTTP API policy")?,
3843 );
3844
3845 let (workflow_grpc, chasm_engine) = {
3852 let repair_archetype = registry.archetype_id(
3856 <tokeira_chasm_activity::ActivityExecution as tokeira_chasm::Component>::FQN,
3857 );
3858 tokeira_storage::run_current_execution_backfill(
3861 chasm_node_repo.as_ref(),
3862 repair_archetype.context("activity CHASM archetype is not registered")?,
3863 500,
3864 )
3865 .await
3866 .context("failed to backfill CHASM current executions")?;
3867 check_registered_archetypes(&chasm_node_repo.distinct_archetypes().await?, ®istry)
3868 .map_err(registry_start_error)?;
3869 let repair_nodes = chasm_node_repo.clone();
3870 let dispatch_queue = Arc::new(tokeira_edge::chasm_activity::ActivityDispatchQueue::new());
3871 let multiplexer = Arc::new(tokeira_runtime::chasm::DispatchMultiplexer::default());
3872 let chasm_visibility_sink =
3877 Arc::new(tokeira_runtime::chasm::ProjectionVisibilitySink::new(
3878 Arc::new(projection_sink()),
3879 effective_config.infrastructure.placement.partition_count,
3880 ));
3881 let mut chasm_engine = tokeira_runtime::chasm::ChasmEngine::new(
3882 chasm_node_repo.clone(),
3883 registry.clone(),
3884 multiplexer.clone(),
3885 chasm_visibility_sink,
3886 );
3887 if let Some(clock) = extensions.clock {
3888 chasm_engine = chasm_engine.with_clock(clock);
3889 }
3890 let chasm_engine = Arc::new(chasm_engine);
3891 let activity_config = tokeira_chasm_activity::ActivityConfig {
3892 enable_standalone: effective_config
3893 .policy
3894 .compatibility
3895 .enable_standalone_activities,
3896 enable_callbacks: effective_config
3897 .policy
3898 .compatibility
3899 .enable_standalone_activity_callbacks,
3900 ..tokeira_chasm_activity::ActivityConfig::default()
3901 };
3902 let standalone_enabled = activity_config.enable_standalone;
3903 let sweeper_engine = chasm_engine.clone();
3905 let dispatch_executor = Arc::new(
3906 tokeira_edge::chasm_executors::ActivityDispatchExecutor::new(
3907 Arc::downgrade(&chasm_engine),
3908 dispatch_queue,
3909 ),
3910 );
3911 let activity_bridge = Arc::new(
3912 tokeira_edge::chasm_activity::ActivityBridge::new(
3913 chasm_engine.clone(),
3914 activity_config.clone(),
3915 DEFAULT_MAX_ID_LENGTH,
3916 )
3917 .with_dispatch_executor(dispatch_executor.clone()),
3918 );
3919 multiplexer
3922 .register(dispatch_executor)
3923 .map_err(registry_start_error)?;
3924 multiplexer
3925 .register(Arc::new(
3926 tokeira_edge::chasm_executors::StartActivityExecutor::new(
3927 Arc::downgrade(&chasm_engine),
3928 activity_config,
3929 DEFAULT_MAX_ID_LENGTH,
3930 ),
3931 ))
3932 .map_err(registry_start_error)?;
3933 multiplexer
3934 .register(Arc::new(
3935 tokeira_edge::chasm_executors::DeliverCallbackExecutor::new(
3936 Arc::downgrade(&chasm_engine),
3937 chasm_nexus_client,
3938 chasm_nexus_config,
3939 namespaces.clone(),
3940 ),
3941 ))
3942 .map_err(registry_start_error)?;
3943 for executor in extensions.executors {
3944 multiplexer
3945 .register(executor)
3946 .map_err(registry_start_error)?;
3947 }
3948 for namespace in namespaces.list_all().await? {
3951 seed_declared_search_attributes(
3952 &visibility_query_store,
3953 namespace_id_for(&namespace.name),
3954 &declared,
3955 )
3956 .await
3957 .map_err(seed_start_error)?;
3958 }
3959 let rebuild = tokeira_runtime::chasm::OutboxRebuildScanner::new(
3960 chasm_node_repo,
3961 chasm_engine.clone(),
3962 multiplexer.clone(),
3963 );
3964 let rebuild_stats = rebuild
3965 .rebuild_once()
3966 .await
3967 .context("failed to rebuild CHASM outboxes before serving")?;
3968 check_rebuilt_outboxes(rebuild_stats)?;
3969 spawn_outbox_rebuild(&engine_tasks, rebuild, background_cancel.clone());
3970 spawn_visibility_repair(
3976 &engine_tasks,
3977 repair_nodes,
3978 Arc::new(projection_sink()),
3979 registry.clone(),
3980 effective_config.infrastructure.placement.partition_count,
3981 background_cancel.clone(),
3982 );
3983 let sweeper_forced =
3989 harness::installed().is_some_and(|hooks| hooks.force_chasm_timer_sweeper);
3990 if standalone_enabled || sweeper_forced || has_extensions {
3991 spawn_chasm_timer_sweeper(
3992 &engine_tasks,
3993 sweeper_engine,
3994 activity_bridge.clone(),
3995 repair_archetype.context("activity CHASM archetype is not registered")?,
3996 background_cancel.clone(),
3997 );
3998 }
3999 (
4000 WorkflowServiceGrpc::new(workflow_service.clone()).with_chasm_activity(activity_bridge),
4001 chasm_engine,
4002 )
4003 };
4004 let admin_grpc = AdminServiceGrpc::new(workflow_service);
4007 let operator_grpc = OperatorServiceGrpc::new(operator_service);
4008
4009 let shard_cleanup = if matches!(transport, StackTransport::Embedded)
4014 && dsql_endpoint.is_some()
4015 && effective_config
4016 .infrastructure
4017 .placement
4018 .controller_endpoint
4019 .is_none()
4020 {
4021 let leases = self_assign_dsql_shards(
4022 runtime.as_ref(),
4023 repo.as_ref(),
4024 effective_config.infrastructure.placement.shard_count,
4025 &node_id,
4026 &node_endpoint,
4027 true,
4028 )
4029 .await?;
4030 Some(self_assigned_shard_cleanup(
4031 runtime.clone(),
4032 repo.clone(),
4033 leases,
4034 ))
4035 } else {
4036 None
4037 };
4038
4039 match dsql_endpoint {
4040 Some(endpoint) => info!(%endpoint, "storage backend: dsql"),
4041 None => info!("storage backend: in-memory"),
4042 }
4043
4044 let log_broadcast = broadcast::Sender::<LogEvent>::new(LOG_BROADCAST_CAPACITY);
4045 let addr = match transport {
4046 StackTransport::Embedded => {
4047 startup_guard.disarm();
4048 return Ok(ConstructedStack::Embedded(EmbeddedStack {
4049 component_visibility_store: chasm_extensions::VisibilityQueryStore(Arc::new(
4050 visibility_query_store,
4051 )),
4052 chasm_engine,
4053 registry,
4054 service: InProcessGrpcService::new(workflow_grpc, operator_grpc, admin_grpc),
4055 background_cancel,
4056 log_broadcast,
4057 recovery_task,
4058 runtime_tasks,
4059 engine_tasks,
4060 shard_cleanup,
4061 }));
4062 }
4063 StackTransport::Network(addr) => addr,
4064 };
4065
4066 let reflection_v1 = tonic_reflection::server::Builder::configure()
4070 .register_encoded_file_descriptor_set(tokeira_proto::public::FILE_DESCRIPTOR_SET)
4071 .build_v1()
4072 .context("failed to build gRPC reflection service")?;
4073 let reflection_v1alpha = tonic_reflection::server::Builder::configure()
4074 .register_encoded_file_descriptor_set(tokeira_proto::public::FILE_DESCRIPTOR_SET)
4075 .build_v1alpha()
4076 .context("failed to build gRPC reflection service")?;
4077
4078 let listener = TcpListener::bind(addr)
4079 .await
4080 .with_context(|| format!("failed to bind tokeirad gRPC listener on {addr}"))?;
4081 let bound_addr = listener
4082 .local_addr()
4083 .context("failed to resolve bound local address for tokeirad gRPC listener")?;
4084
4085 let (shutdown_tx, shutdown_rx) = oneshot::channel();
4086
4087 let wire_coverage_recorder = wire_coverage_enabled().then(|| {
4100 info!("conformance wire-coverage recorder enabled");
4101 Arc::new(WireCoverageRecorder::new())
4102 });
4103 let server_recorder = wire_coverage_recorder.clone();
4104
4105 let server_cancel = background_cancel.clone();
4106 let server_task = tokio::spawn(async move {
4107 let shutdown_signal = async move {
4108 tokio::select! {
4109 _ = shutdown_rx => {},
4110 _ = server_cancel.cancelled() => {},
4111 }
4112 };
4113
4114 match server_recorder {
4118 Some(recorder) => {
4119 Server::builder()
4120 .accept_http1(true)
4121 .layer(nexus_http_layer.clone())
4122 .layer(http_api_layer.clone())
4123 .layer(CorsLayer::permissive())
4124 .layer(GrpcWebLayer::new())
4125 .layer(WireCoverageLayer::new(recorder))
4126 .add_service(workflow_grpc.into_service())
4127 .add_service(operator_grpc.into_service())
4128 .add_service(admin_grpc.into_service())
4129 .add_service(dispatch_grpc.into_service())
4130 .add_service(reflection_v1)
4131 .add_service(reflection_v1alpha)
4132 .serve_with_incoming_shutdown(TcpListenerStream::new(listener), shutdown_signal)
4133 .await
4134 }
4135 None => {
4136 Server::builder()
4137 .accept_http1(true)
4138 .layer(nexus_http_layer)
4139 .layer(http_api_layer)
4140 .layer(CorsLayer::permissive())
4141 .layer(GrpcWebLayer::new())
4142 .add_service(workflow_grpc.into_service())
4143 .add_service(operator_grpc.into_service())
4144 .add_service(admin_grpc.into_service())
4145 .add_service(dispatch_grpc.into_service())
4146 .add_service(reflection_v1)
4147 .add_service(reflection_v1alpha)
4148 .serve_with_incoming_shutdown(TcpListenerStream::new(listener), shutdown_signal)
4149 .await
4150 }
4151 }
4152 .with_context(|| format!("failed to serve gRPC transport on {bound_addr}"))?;
4153 Ok::<(), anyhow::Error>(())
4154 });
4155
4156 if let Some(task) = harness::installed().and_then(|hooks| hooks.background_task.as_ref()) {
4162 task(background_cancel.clone());
4163 }
4164
4165 startup_guard.disarm();
4166 Ok(ConstructedStack::Network((
4167 server_task,
4168 bound_addr,
4169 shutdown_tx,
4170 background_cancel,
4171 log_broadcast,
4172 wire_coverage_recorder,
4173 )))
4174}
4175
4176fn with_loopback_port(base: &str, port: u16) -> String {
4180 let trimmed = base.trim_end_matches('/');
4181 match trimmed.rsplit_once(':') {
4182 Some((prefix, suffix))
4185 if !suffix.is_empty() && suffix.bytes().all(|b| b.is_ascii_digit()) =>
4186 {
4187 format!("{prefix}:{port}")
4188 }
4189 _ => format!("{trimmed}:{port}"),
4190 }
4191}
4192
4193fn spawn_nexus_callback_server(
4198 listener: TcpListener,
4199 runtime: Arc<dyn WorkflowRuntimeApi>,
4200 cancel: CancellationToken,
4201) {
4202 tokio::spawn(async move {
4203 loop {
4204 let stream = tokio::select! {
4205 _ = cancel.cancelled() => break,
4206 accept = listener.accept() => match accept {
4207 Ok((stream, _peer)) => stream,
4208 Err(error) => {
4209 tracing::warn!(?error, "nexus callback listener accept failed");
4210 continue;
4211 }
4212 },
4213 };
4214 let io = TokioIo::new(stream);
4215 let runtime = runtime.clone();
4216 let conn_cancel = cancel.clone();
4217 tokio::spawn(async move {
4218 let service = service_fn(move |request| {
4219 let runtime = runtime.clone();
4220 async move {
4221 Ok::<_, Infallible>(
4222 nexus_callback_response(runtime.as_ref(), request).await,
4223 )
4224 }
4225 });
4226 let conn = hyper::server::conn::http1::Builder::new().serve_connection(io, service);
4227 tokio::pin!(conn);
4228 tokio::select! {
4229 result = conn.as_mut() => {
4230 if let Err(error) = result {
4231 tracing::debug!(?error, "nexus callback connection closed with error");
4232 }
4233 }
4234 _ = conn_cancel.cancelled() => {
4235 conn.as_mut().graceful_shutdown();
4236 let _ = conn.await;
4237 }
4238 }
4239 });
4240 }
4241 });
4242}
4243
4244#[derive(Debug)]
4246enum BoundedBody {
4247 Collected(Bytes),
4249 TooLarge,
4251 ReadFailed(Box<dyn std::error::Error + Send + Sync>),
4253}
4254
4255async fn collect_bounded_body<B>(body: B, limit: usize) -> BoundedBody
4262where
4263 B: hyper::body::Body<Data = Bytes>,
4264 B::Error: Into<Box<dyn std::error::Error + Send + Sync>>,
4265{
4266 match Limited::new(body, limit).collect().await {
4267 Ok(collected) => BoundedBody::Collected(collected.to_bytes()),
4268 Err(error) if error.downcast_ref::<LengthLimitError>().is_some() => BoundedBody::TooLarge,
4269 Err(error) => BoundedBody::ReadFailed(error),
4270 }
4271}
4272
4273async fn nexus_callback_response(
4277 runtime: &dyn WorkflowRuntimeApi,
4278 request: Request<Incoming>,
4279) -> Response<Full<Bytes>> {
4280 let (parts, body) = request.into_parts();
4281
4282 if parts.method != Method::POST || parts.uri.path() != NEXUS_CALLBACK_PATH {
4283 return nexus_response(StatusCode::NOT_FOUND, None);
4284 }
4285
4286 let header = |name: &str| {
4287 parts
4288 .headers
4289 .get(name)
4290 .and_then(|value| value.to_str().ok())
4291 .map(str::to_owned)
4292 };
4293 let token = header(TEMPORAL_CALLBACK_TOKEN_HEADER);
4294 let operation_token = header(NEXUS_OPERATION_TOKEN_HEADER);
4295 let state = header(NEXUS_OPERATION_STATE_HEADER);
4296 let content_type = header(hyper::header::CONTENT_TYPE.as_str());
4297
4298 let body_bytes = match collect_bounded_body(body, MAX_NEXUS_PAYLOAD_BYTES).await {
4305 BoundedBody::Collected(bytes) => bytes,
4306 BoundedBody::TooLarge => {
4307 tracing::debug!(
4308 limit = MAX_NEXUS_PAYLOAD_BYTES,
4309 "nexus callback body too large"
4310 );
4311 return nexus_response(
4312 StatusCode::PAYLOAD_TOO_LARGE,
4313 Some(b"request body too large".to_vec()),
4314 );
4315 }
4316 BoundedBody::ReadFailed(error) => {
4317 tracing::debug!(?error, "failed to read nexus callback request body");
4318 return nexus_response(
4319 StatusCode::BAD_REQUEST,
4320 Some(b"failed to read request body".to_vec()),
4321 );
4322 }
4323 };
4324
4325 let CallbackResponse { status, body } = handle_nexus_callback(
4326 runtime,
4327 token.as_deref(),
4328 operation_token.as_deref(),
4329 state.as_deref(),
4330 content_type.as_deref(),
4331 &body_bytes,
4332 )
4333 .await;
4334
4335 let status = StatusCode::from_u16(status).unwrap_or(StatusCode::INTERNAL_SERVER_ERROR);
4336 nexus_response(status, body)
4337}
4338
4339fn nexus_response(status: StatusCode, body: Option<Vec<u8>>) -> Response<Full<Bytes>> {
4340 let mut response = Response::new(Full::new(Bytes::from(body.unwrap_or_default())));
4341 *response.status_mut() = status;
4342 response
4343}
4344
4345const LOG_BROADCAST_CAPACITY: usize = 256;
4350
4351const WIRE_COVERAGE_ENV: &str = "TOKEIRA_CONFORMANCE_WIRE_COVERAGE";
4359
4360const WIRE_COVERAGE_OUT_ENV: &str = "TOKEIRA_CONFORMANCE_WIRE_COVERAGE_OUT";
4369
4370const WIRE_COVERAGE_DEFAULT_OUT: &str = "./wire-coverage.json";
4377
4378fn wire_coverage_enabled() -> bool {
4384 match std::env::var(WIRE_COVERAGE_ENV) {
4385 Ok(value) => {
4386 let trimmed = value.trim();
4387 !trimmed.is_empty()
4388 && !trimmed.eq_ignore_ascii_case("0")
4389 && !trimmed.eq_ignore_ascii_case("false")
4390 }
4391 Err(_) => false,
4392 }
4393}
4394
4395fn wire_coverage_out_path() -> PathBuf {
4403 match std::env::var(WIRE_COVERAGE_OUT_ENV) {
4404 Ok(value) if !value.trim().is_empty() => PathBuf::from(value.trim()),
4405 _ => PathBuf::from(WIRE_COVERAGE_DEFAULT_OUT),
4406 }
4407}
4408
4409async fn build_nexus_endpoint_store(
4417 namespaces: &dyn NamespaceCache,
4418 configs: HashMap<String, BootstrapNexusEndpointConfig>,
4419) -> Result<Arc<InMemoryNexusEndpointStore>> {
4420 let store = Arc::new(InMemoryNexusEndpointStore::new());
4421 let now = std::time::SystemTime::now()
4422 .duration_since(std::time::UNIX_EPOCH)
4423 .map(|d| d.as_nanos() as i64)
4424 .unwrap_or(0);
4425 for (endpoint_name, config) in configs {
4426 let target = match config.target {
4427 BootstrapNexusEndpointTarget::External { address } => {
4428 NexusEndpointSpecTarget::External { url: address }
4429 }
4430 BootstrapNexusEndpointTarget::Worker {
4431 namespace_name,
4432 task_queue,
4433 } => {
4434 let namespace = namespaces.get(&namespace_name).await?.ok_or_else(|| {
4435 anyhow!(
4436 "failed to register nexus worker endpoint `{endpoint_name}`: namespace `{namespace_name}` not found"
4437 )
4438 })?;
4439 NexusEndpointSpecTarget::Worker {
4440 namespace_name: namespace.name.clone(),
4441 namespace_id: namespace_id_for(&namespace.name).0.to_string(),
4442 task_queue,
4443 }
4444 }
4445 };
4446 store
4447 .create(
4448 NexusEndpointSpec {
4449 name: endpoint_name.clone(),
4450 description: Vec::new(),
4451 target,
4452 },
4453 now,
4454 )
4455 .map_err(|e| anyhow!("failed to seed nexus endpoint `{endpoint_name}`: {e}"))?;
4456 }
4457 Ok(store)
4458}
4459
4460async fn self_assign_dsql_shards<R>(
4461 runtime: &TokeiraRuntime<R>,
4462 lease_repository: &R,
4463 shard_count: u32,
4464 node_id: &IncarnationId,
4465 node_endpoint: &NodeEndpoint,
4466 require_all: bool,
4467) -> Result<Vec<SelfAssignedShardLease>>
4468where
4469 R: LeaseRepository + RunRepository + 'static,
4470{
4471 let existing_leases = lease_repository.list_bundle_leases().await?;
4475 for lease in &existing_leases {
4476 if let Some(owner) = &lease.owner_node_id {
4477 let _ = lease_repository
4478 .relinquish_bundle(lease.bundle_id, owner.clone(), lease.epoch)
4479 .await;
4480 }
4481 }
4482
4483 let mut acquired = Vec::new();
4484 let mut recovered = SweepResult::default();
4485 let owner = node_id.to_string();
4486 for shard_index in 0..shard_count {
4487 let shard_id = ShardId(shard_index);
4488 match lease_repository
4489 .try_acquire_bundle(shard_id, owner.clone(), node_endpoint.as_authority())
4490 .await
4491 {
4492 Ok(LeaseOutcome::Acquired { epoch } | LeaseOutcome::Renewed { epoch }) => {
4493 match runtime.recover_self_assigned_shard(shard_id, epoch).await {
4499 Ok(swept) => {
4500 recovered.workflow_tasks_republished += swept.workflow_tasks_republished;
4501 recovered.activity_tasks_republished += swept.activity_tasks_republished;
4502 recovered.due_timers_injected += swept.due_timers_injected;
4503 recovered.activity_tracking_entries_reconstructed +=
4504 swept.activity_tracking_entries_reconstructed;
4505 acquired.push(SelfAssignedShardLease {
4506 shard_id,
4507 owner: owner.clone(),
4508 epoch,
4509 });
4510 }
4511 Err(error) => {
4512 let _ = lease_repository
4514 .relinquish_bundle(shard_id, owner.clone(), epoch)
4515 .await;
4516 if require_all {
4517 rollback_self_assigned_shards(runtime, lease_repository, &acquired)
4518 .await;
4519 return Err(error).context(format!(
4520 "failed to recover embedded DSQL shard {shard_index}"
4521 ));
4522 }
4523 tracing::warn!(
4524 shard_index,
4525 ?error,
4526 "failed to recover self-assigned shard; leaving it unowned"
4527 );
4528 }
4529 }
4530 }
4531 Ok(LeaseOutcome::Rejected {
4532 current_owner,
4533 current_epoch,
4534 }) => {
4535 if require_all {
4536 rollback_self_assigned_shards(runtime, lease_repository, &acquired).await;
4537 return Err(anyhow!(
4538 "embedded DSQL shard {shard_index} is held by owner {current_owner} at epoch {}",
4539 current_epoch.0
4540 ));
4541 }
4542 tracing::warn!(
4543 shard_index,
4544 %current_owner,
4545 current_epoch = current_epoch.0,
4546 "failed to self-assign shard: lease is held by another owner"
4547 );
4548 }
4549 Err(error) => {
4550 if require_all {
4551 rollback_self_assigned_shards(runtime, lease_repository, &acquired).await;
4552 return Err(error).context(format!(
4553 "failed to self-assign embedded DSQL shard {shard_index}"
4554 ));
4555 }
4556 tracing::warn!(shard_index, ?error, "failed to self-assign shard");
4557 }
4558 }
4559 }
4560 info!(
4561 acquired = acquired.len(),
4562 shard_count,
4563 workflow_tasks_republished = recovered.workflow_tasks_republished,
4564 activity_tasks_republished = recovered.activity_tasks_republished,
4565 due_timers_injected = recovered.due_timers_injected,
4566 activity_tracking_entries_reconstructed = recovered.activity_tracking_entries_reconstructed,
4567 "self-assigned and recovered DSQL shards (no controller)"
4568 );
4569 Ok(acquired)
4570}
4571
4572async fn rollback_self_assigned_shards<R>(
4573 runtime: &TokeiraRuntime<R>,
4574 lease_repository: &R,
4575 leases: &[SelfAssignedShardLease],
4576) where
4577 R: LeaseRepository + RunRepository + 'static,
4578{
4579 for lease in leases.iter().rev() {
4580 runtime.relinquish_shard(lease.shard_id).await;
4581 let _ = lease_repository
4582 .relinquish_bundle(lease.shard_id, lease.owner.clone(), lease.epoch)
4583 .await;
4584 }
4585}
4586
4587fn self_assigned_shard_cleanup<R>(
4588 runtime: Arc<TokeiraRuntime<R>>,
4589 lease_repository: Arc<R>,
4590 leases: Vec<SelfAssignedShardLease>,
4591) -> CleanupAction
4592where
4593 R: LeaseRepository + RunRepository + 'static,
4594{
4595 Box::new(move |deadline| {
4596 Box::pin(async move {
4597 let mut first_error = None;
4598 for lease in leases.into_iter().rev() {
4599 runtime.relinquish_shard(lease.shard_id).await;
4600 let now = Instant::now();
4601 if now >= deadline {
4602 first_error.get_or_insert_with(|| {
4603 anyhow!("deadline elapsed while relinquishing embedded DSQL shards")
4604 });
4605 continue;
4606 }
4607 let release = tokio::time::timeout(
4608 deadline.saturating_duration_since(now),
4609 lease_repository.relinquish_bundle(lease.shard_id, lease.owner, lease.epoch),
4610 )
4611 .await;
4612 match release {
4613 Ok(Ok(LeaseOutcome::Acquired { .. })) => {}
4614 Ok(Ok(outcome)) => {
4615 first_error.get_or_insert_with(|| {
4616 anyhow!("embedded DSQL shard release was fenced: {outcome:?}")
4617 });
4618 }
4619 Ok(Err(error)) => {
4620 first_error.get_or_insert(error);
4621 }
4622 Err(_) => {
4623 first_error.get_or_insert_with(|| {
4624 anyhow!("deadline elapsed while relinquishing embedded DSQL shard")
4625 });
4626 }
4627 }
4628 }
4629 match first_error {
4630 Some(error) => Err(error),
4631 None => Ok(()),
4632 }
4633 })
4634 })
4635}
4636
4637fn configured_node_endpoint(config: &TokeiraConfig, listen_addr: SocketAddr) -> NodeEndpoint {
4638 NodeEndpoint {
4639 host: config.infrastructure.placement.node_host.clone(),
4640 port: config
4641 .infrastructure
4642 .placement
4643 .node_port
4644 .unwrap_or_else(|| listen_addr.port()),
4645 }
4646}
4647
4648fn placement_config(config: &TokeiraConfig) -> PlacementConfig {
4649 let placement = &config.infrastructure.placement;
4650 PlacementConfig {
4651 shard_count: placement.shard_count,
4652 bundle_count: placement.bundle_count,
4653 partition_count: placement.partition_count,
4654 hash_version: placement.hash_version,
4655 }
4656}
4657
4658fn membership_config(
4659 config: &TokeiraConfig,
4660 node_id: IncarnationId,
4661 node_endpoint: NodeEndpoint,
4662 controller_endpoint: String,
4663) -> MembershipConfig {
4664 let placement = &config.infrastructure.placement;
4665 MembershipConfig {
4666 controller_endpoint,
4667 heartbeat_interval: std::time::Duration::from_millis(placement.heartbeat_interval_ms),
4668 reconnect_base_delay: std::time::Duration::from_millis(placement.reconnect_base_delay_ms),
4669 reconnect_max_delay: std::time::Duration::from_millis(placement.reconnect_max_delay_ms),
4670 node_id,
4671 node_endpoint,
4672 zone: None,
4673 version: tokeira_build_info::TOKEIRA_VERSION.to_string(),
4674 build_id: tokeira_build_info::TOKEIRA_GIT_SHA.to_string(),
4675 }
4676}
4677
4678#[derive(Debug)]
4681struct NoopConnectionBudgetApplier;
4682
4683impl ConnectionBudgetApplier for NoopConnectionBudgetApplier {
4684 fn apply_budget(
4685 &self,
4686 _rate_per_second: f64,
4687 _capacity: u64,
4688 _max_reservoir_size: u32,
4689 ) -> Result<()> {
4690 Ok(())
4691 }
4692
4693 fn available_connections(&self) -> u32 {
4694 0
4695 }
4696}
4697
4698#[derive(Debug)]
4709struct DsqlConnectionBudgetApplier {
4710 director: Arc<DsqlConnectionDirector>,
4711}
4712
4713impl ConnectionBudgetApplier for DsqlConnectionBudgetApplier {
4714 fn apply_budget(
4715 &self,
4716 rate_per_second: f64,
4717 capacity: u64,
4718 max_reservoir_size: u32,
4719 ) -> Result<()> {
4720 let retired = self.director.apply_connection_budget(max_reservoir_size)?;
4721 tracing::debug!(
4722 rate_per_second,
4723 capacity,
4724 max_reservoir_size,
4725 reservoir_target = self.director.reservoir_target(),
4726 retired,
4727 "applied controller connection budget to the DSQL reservoir"
4728 );
4729 Ok(())
4730 }
4731
4732 fn available_connections(&self) -> u32 {
4733 u32::try_from(self.director.ready_connections()).unwrap_or(u32::MAX)
4734 }
4735}
4736
4737struct StoreExecutionResolver<R> {
4743 repo: Arc<R>,
4744}
4745
4746impl<R> StoreExecutionResolver<R> {
4747 fn new(repo: Arc<R>) -> Self {
4748 Self { repo }
4749 }
4750}
4751
4752fn apply_reported_problem_search_attribute(
4753 search_attributes: &mut tokeira_types::SearchAttributes,
4754 problem: Option<WorkflowTaskReportedProblem>,
4755) {
4756 let Some(problem) = problem else {
4757 return;
4758 };
4759 let entries = match &problem.problem {
4767 tokeira_kernel::WorkflowTaskProblem::Failed(cause) => vec![
4768 "category=WorkflowTaskFailed".to_string(),
4769 format!("cause=WorkflowTaskFailedCause{}", cause.as_str()),
4770 ],
4771 tokeira_kernel::WorkflowTaskProblem::TimedOutStartToClose => vec![
4772 "category=WorkflowTaskTimedOut".to_string(),
4773 "cause=WorkflowTaskTimedOutCauseStartToClose".to_string(),
4774 ],
4775 };
4776 search_attributes.0.insert(
4777 "TemporalReportedProblems".to_string(),
4778 SearchAttrValue::KeywordList(entries),
4779 );
4780}
4781
4782#[async_trait]
4783impl<R> ExecutionResolver for StoreExecutionResolver<R>
4784where
4785 R: RunRepository + 'static,
4786{
4787 async fn current_run_key(
4788 &self,
4789 namespace: &str,
4790 workflow_id: &str,
4791 ) -> Result<Option<tokeira_types::RunKey>> {
4792 let namespace_id = namespace_id_for(namespace);
4793 let result = self
4794 .repo
4795 .resolve_execution(&ExecutionRef {
4796 namespace_id,
4797 workflow_id: WorkflowId(workflow_id.to_string()),
4798 run_id: None,
4799 })
4800 .await?;
4801 if result.is_some() {
4802 return Ok(result);
4803 }
4804 self.repo
4805 .find_latest_run(namespace_id, &WorkflowId(workflow_id.to_string()))
4806 .await
4807 }
4808
4809 async fn describe_execution(
4810 &self,
4811 namespace: &str,
4812 workflow_id: &str,
4813 run_id: Option<tokeira_types::RunId>,
4814 ) -> Result<Option<WorkflowExecutionDescription>> {
4815 let namespace_id = namespace_id_for(namespace);
4816 let run_key = match self
4817 .repo
4818 .resolve_execution(&ExecutionRef {
4819 namespace_id,
4820 workflow_id: WorkflowId(workflow_id.to_string()),
4821 run_id,
4822 })
4823 .await?
4824 {
4825 Some(rk) => rk,
4826 None if run_id.is_none() => {
4827 match self
4828 .repo
4829 .find_latest_run(namespace_id, &WorkflowId(workflow_id.to_string()))
4830 .await?
4831 {
4832 Some(rk) => rk,
4833 None => return Ok(None),
4834 }
4835 }
4836 None => return Ok(None),
4837 };
4838
4839 let (loaded, stats) = self.repo.load_run_with_stats(run_key).await?;
4840 match loaded {
4841 LoadedRun::Existing(state) => {
4842 let history_size_bytes = stats.history_size_bytes;
4850 let external_payload_count = state.external_payload_count;
4851 let external_payload_size_bytes = state.external_payload_size_bytes;
4852 let reported_problem = reported_problem_from_state(&state);
4856 let versioning_info = state.versioning_info.clone();
4857 let most_recent_worker_version_stamp = versioning_info
4858 .as_ref()
4859 .and_then(|info| info.most_recent_worker_version_stamp.clone());
4860 let pending_nexus_operations =
4861 tokeira_edge::translate::describe_pending_nexus_operations(&state);
4862 let mut search_attributes = state.search_attributes;
4863 apply_reported_problem_search_attribute(&mut search_attributes, reported_problem);
4864 if let Some(build_ids) = versioning_info
4865 .as_ref()
4866 .map(|info| &info.build_id_search_attributes)
4867 .filter(|build_ids| !build_ids.is_empty())
4868 {
4869 search_attributes.0.insert(
4870 "BuildIds".to_string(),
4871 tokeira_types::SearchAttrValue::KeywordList(build_ids.clone()),
4872 );
4873 }
4874 Ok(Some(WorkflowExecutionDescription {
4875 namespace: namespace.to_string(),
4876 workflow_id: state.workflow_id.0,
4877 run_key: state.run_key,
4878 run_id: state.run_id,
4879 workflow_type: state.workflow_type.0,
4880 task_queue: state.task_queue.0.clone(),
4881 status: state.status,
4882 start_time: Some(state.started_at),
4883 close_time: state.closed_at,
4884 execution_time: state.started_at
4889 + state.workflow_start_delay.unwrap_or_default(),
4890 execution_config: tokeira_edge::translate::ExecutionConfigDescription {
4891 task_queue: state.task_queue.0.clone(),
4892 workflow_execution_timeout: state.workflow_execution_timeout,
4893 workflow_run_timeout: state.workflow_run_timeout,
4894 default_workflow_task_timeout: state.workflow_task_timeout,
4895 user_metadata: state.user_metadata.map(|metadata| {
4899 tokeira_edge::translate::UserMetadata {
4900 summary: metadata.summary,
4901 details: metadata.details,
4902 }
4903 }),
4904 },
4905 history_length: state.last_event_id,
4906 history_size_bytes,
4907 state_transition_count: state.transition_seq.0 as i64,
4908 parent_namespace_id: state
4909 .parent_namespace_id
4910 .map(|namespace_id| namespace_id.0.to_string()),
4911 parent_workflow_id: state.parent_workflow_id.clone(),
4912 parent_run_id: state.parent_run_id,
4913 root_workflow_id: state.root_workflow_id.clone(),
4914 root_run_id: state.root_run_id,
4915 first_run_id: state.first_execution_run_id,
4916 memo: state.memo,
4917 search_attributes,
4918 pending_activities: state
4919 .activities
4920 .values()
4921 .map(
4922 |activity| tokeira_edge::translate::PendingActivityDescription {
4923 activity_id: activity.activity_id.clone(),
4924 activity_type: activity.activity_type.clone(),
4925 is_started: activity.started_at.is_some(),
4926 cancel_requested: activity.cancel_requested,
4927 attempt: activity.attempt,
4928 maximum_attempts: activity
4929 .retry_policy
4930 .as_ref()
4931 .map(|policy| policy.maximum_attempts)
4932 .unwrap_or_default(),
4933 scheduled_at: activity.scheduled_at,
4934 started_at: activity.started_at,
4935 last_failure: activity.last_failure.clone(),
4936 heartbeat_details: activity.heartbeat_details.clone(),
4937 last_heartbeat_at: activity.last_heartbeat_at,
4938 last_worker_identity: {
4939 let started = activity
4944 .started_identity
4945 .clone()
4946 .map(|identity| identity.0)
4947 .unwrap_or_default();
4948 if started.is_empty() && activity.retry_policy.is_some() {
4949 activity
4950 .retry_last_worker_identity
4951 .clone()
4952 .map(|identity| identity.0)
4953 .unwrap_or_default()
4954 } else {
4955 started
4956 }
4957 },
4958 paused: activity.pause_info.is_some(),
4959 pause_info: activity.pause_info.as_ref().map(|info| {
4960 tokeira_edge::translate::PauseInfoDescription {
4961 identity: info.identity.clone(),
4962 paused_time: info.pause_time,
4963 reason: info.reason.clone(),
4964 rule_id: info.rule_id.clone(),
4965 }
4966 }),
4967 activity_options: tokeira_edge::translate::ActivityOptions {
4968 task_queue: Some(activity.task_queue.0.clone()),
4969 schedule_to_close_timeout: activity.schedule_to_close_timeout,
4970 schedule_to_start_timeout: activity.schedule_to_start_timeout,
4971 start_to_close_timeout: activity.start_to_close_timeout,
4972 heartbeat_timeout: activity.heartbeat_timeout,
4973 retry_policy: activity.retry_policy.clone(),
4974 priority: activity.priority.as_ref().map(|priority| {
4975 tokeira_edge::translate::Priority {
4976 priority_key: priority.priority_key,
4977 fairness_key: priority.fairness_key.clone(),
4978 fairness_weight: priority.fairness_weight,
4979 }
4980 }),
4981 },
4982 },
4983 )
4984 .collect(),
4985 pending_children: state
4986 .children
4987 .values()
4988 .map(|child| tokeira_edge::translate::PendingChildDescription {
4989 workflow_id: child.child_workflow_id.0.clone(),
4990 run_id: child
4991 .child_run_id
4992 .as_ref()
4993 .map(|run_id| run_id.0.to_string()),
4994 workflow_type: String::new(),
4995 initiated_event_id: child.initiated_event_id,
4996 parent_close_policy: child.parent_close_policy,
4997 })
4998 .collect(),
4999 pending_workflow_task: state.pending_workflow_task.as_ref().map(|task| {
5000 tokeira_edge::translate::PendingWorkflowTaskDescription {
5001 is_started: task.started_event_id.is_some(),
5002 scheduled_at: task.scheduled_at,
5003 started_at: task.started_at,
5004 attempt: task.attempt,
5005 }
5006 }),
5007 callbacks: state.completion_callbacks.clone(),
5008 pending_nexus_operations,
5009 pause_info: state.pause_info.as_ref().map(|info| {
5010 tokeira_edge::translate::PauseInfoDescription {
5011 identity: info.identity.clone(),
5012 paused_time: info.pause_time,
5013 reason: info.reason.clone(),
5014 rule_id: None,
5015 }
5016 }),
5017 execution_expiration_time: state.workflow_execution_timeout.map(|timeout| {
5018 state.first_run_started_at.unwrap_or(state.started_at) + timeout
5019 }),
5020 run_expiration_time: state
5021 .workflow_run_timeout
5022 .map(|timeout| state.started_at + timeout),
5023 cancel_requested: state.cancel_requested,
5024 original_start_time: state.first_run_started_at.unwrap_or(state.started_at),
5025 versioning_info: versioning_info
5026 .filter(|info| info.has_execution_versioning_info()),
5027 worker_deployment_name: state.worker_deployment_name.clone(),
5028 priority: state.priority.as_ref().map(|priority| {
5029 tokeira_edge::translate::Priority {
5030 priority_key: priority.priority_key,
5031 fairness_key: priority.fairness_key.clone(),
5032 fairness_weight: priority.fairness_weight,
5033 }
5034 }),
5035 auto_reset_points: state.auto_reset_points.clone(),
5036 most_recent_worker_version_stamp,
5037 request_id_infos: state.request_id_infos.clone(),
5038 external_payload_count,
5039 external_payload_size_bytes,
5040 }))
5041 }
5042 LoadedRun::Absent => Err(anyhow!("resolved run missing from storage: {:?}", run_key)),
5043 }
5044 }
5045}
5046
5047#[doc(hidden)]
5048pub fn __cli_parse() -> Cli {
5049 Cli::parse()
5050}
5051
5052#[cfg(test)]
5053#[allow(unsafe_code)]
5055mod tests {
5056 use super::*;
5057 use metrics::with_local_recorder;
5058 use metrics_util::debugging::DebuggingRecorder;
5059 use opentelemetry::trace::TracerProvider as _;
5060 use opentelemetry_sdk::{
5061 error::{OTelSdkError, OTelSdkResult},
5062 trace::{SdkTracerProvider, SpanData, SpanExporter},
5063 };
5064 use proptest::prelude::*;
5065 use tracing_subscriber::layer::SubscriberExt as _;
5066 use tokeira_runtime::{EndpointTarget, NexusEndpointConfig};
5069
5070 #[derive(Clone, Copy, Debug, PartialEq, Eq)]
5071 enum StartupModelPhase {
5072 Cluster,
5073 Pool,
5074 Schema,
5075 Ownership,
5076 Runtime,
5077 }
5078
5079 #[derive(Clone, Copy, Debug, PartialEq, Eq)]
5080 enum StartupModelEvent {
5081 Attempt(StartupModelPhase),
5082 Complete(StartupModelPhase),
5083 ReleaseOwnership,
5084 ClosePool,
5085 ReturnHandle,
5086 }
5087
5088 const STARTUP_MODEL_PHASES: [StartupModelPhase; 5] = [
5089 StartupModelPhase::Cluster,
5090 StartupModelPhase::Pool,
5091 StartupModelPhase::Schema,
5092 StartupModelPhase::Ownership,
5093 StartupModelPhase::Runtime,
5094 ];
5095
5096 fn startup_model(failure_boundary: Option<usize>) -> Vec<StartupModelEvent> {
5097 let mut events = Vec::new();
5098 let mut pool_open = false;
5099 let mut ownership_held = false;
5100 for (index, phase) in STARTUP_MODEL_PHASES.into_iter().enumerate() {
5101 events.push(StartupModelEvent::Attempt(phase));
5102 if failure_boundary == Some(index) {
5103 if ownership_held {
5104 events.push(StartupModelEvent::ReleaseOwnership);
5105 }
5106 if pool_open {
5107 events.push(StartupModelEvent::ClosePool);
5108 }
5109 return events;
5110 }
5111 events.push(StartupModelEvent::Complete(phase));
5112 pool_open |= phase == StartupModelPhase::Pool;
5113 ownership_held |= phase == StartupModelPhase::Ownership;
5114 }
5115 events.push(StartupModelEvent::ReturnHandle);
5116 events
5117 }
5118
5119 #[derive(Clone, Copy, Debug, PartialEq, Eq)]
5120 enum ShutdownModelEvent {
5121 CloseAdmission,
5122 CancelTasks,
5123 CallsComplete,
5124 DrainComplete,
5125 TasksComplete,
5126 JoinComplete,
5127 FinishTelemetry,
5128 AttemptShardRelease,
5129 CloseOwnershipAdmission,
5130 AttemptOwnershipRelease,
5131 AttemptPoolClose,
5132 ReturnToHost,
5133 }
5134
5135 fn shutdown_model(calls_pending: bool, tasks_pending: bool) -> Vec<ShutdownModelEvent> {
5136 let mut events = vec![
5137 ShutdownModelEvent::CloseAdmission,
5138 ShutdownModelEvent::CancelTasks,
5139 ];
5140 if calls_pending {
5141 events.push(ShutdownModelEvent::CallsComplete);
5142 }
5143 events.push(ShutdownModelEvent::DrainComplete);
5144 if tasks_pending {
5145 events.push(ShutdownModelEvent::TasksComplete);
5146 }
5147 events.extend([
5148 ShutdownModelEvent::JoinComplete,
5149 ShutdownModelEvent::FinishTelemetry,
5150 ShutdownModelEvent::AttemptShardRelease,
5151 ShutdownModelEvent::CloseOwnershipAdmission,
5152 ShutdownModelEvent::AttemptOwnershipRelease,
5153 ShutdownModelEvent::AttemptPoolClose,
5154 ShutdownModelEvent::ReturnToHost,
5155 ]);
5156 events
5157 }
5158
5159 #[derive(Clone, Copy, Debug)]
5160 enum HostInstrumentationModel {
5161 None,
5162 LocalRecorder,
5163 LocalSubscriber,
5164 RecorderAndSubscriber,
5165 }
5166
5167 fn host_instrumentation_strategy() -> impl Strategy<Value = HostInstrumentationModel> {
5168 prop_oneof![
5169 Just(HostInstrumentationModel::None),
5170 Just(HostInstrumentationModel::LocalRecorder),
5171 Just(HostInstrumentationModel::LocalSubscriber),
5172 Just(HostInstrumentationModel::RecorderAndSubscriber),
5173 ]
5174 }
5175
5176 #[derive(Clone, Copy, Debug)]
5177 enum EmbeddedStorageModeModel {
5178 InMemory,
5179 ManagedDsql,
5180 ExistingDsql,
5181 }
5182
5183 fn storage_mode_strategy() -> impl Strategy<Value = EmbeddedStorageModeModel> {
5184 prop_oneof![
5185 Just(EmbeddedStorageModeModel::InMemory),
5186 Just(EmbeddedStorageModeModel::ManagedDsql),
5187 Just(EmbeddedStorageModeModel::ExistingDsql),
5188 ]
5189 }
5190
5191 #[derive(Clone, Copy, Debug, PartialEq, Eq)]
5192 struct EmbeddedConstructionEffects {
5193 listener_attempts: usize,
5194 global_install_attempts: usize,
5195 signal_handler_attempts: usize,
5196 local_emission_attempts: usize,
5197 }
5198
5199 fn embedded_construction_effects(
5200 _host: HostInstrumentationModel,
5201 _storage: EmbeddedStorageModeModel,
5202 ) -> EmbeddedConstructionEffects {
5203 EmbeddedConstructionEffects {
5206 listener_attempts: 0,
5207 global_install_attempts: 0,
5208 signal_handler_attempts: 0,
5209 local_emission_attempts: 1,
5210 }
5211 }
5212
5213 #[derive(Clone, Copy, Debug)]
5214 enum TelemetryFixture {
5215 NoSubscriber,
5216 LocalRecorder,
5217 FailingExporter,
5218 }
5219
5220 fn telemetry_fixture_strategy() -> impl Strategy<Value = TelemetryFixture> {
5221 prop_oneof![
5222 Just(TelemetryFixture::NoSubscriber),
5223 Just(TelemetryFixture::LocalRecorder),
5224 Just(TelemetryFixture::FailingExporter),
5225 ]
5226 }
5227
5228 #[derive(Clone, Debug)]
5229 struct FailingSpanExporter;
5230
5231 impl SpanExporter for FailingSpanExporter {
5232 async fn export(&self, _batch: Vec<SpanData>) -> OTelSdkResult {
5233 Err(OTelSdkError::InternalFailure(
5234 "fixture exporter rejected the batch".to_owned(),
5235 ))
5236 }
5237 }
5238
5239 fn apply_observed_sequence(ops: &[i16]) -> Vec<u8> {
5240 let mut state = 0_i64;
5241 let mut committed = Vec::with_capacity(ops.len() * std::mem::size_of::<i64>());
5242 let span = tracing::info_span!(
5243 "embedded.observational_fixture",
5244 tokeira.operation = "transition_sequence"
5245 );
5246 let _entered = span.enter();
5247 for op in ops {
5248 record_embedded_lifecycle(
5249 EmbeddedStorageModeLabel::InMemory,
5250 ClusterStatusLabel::NotApplicable,
5251 SchemaOutcomeLabel::NotApplicable,
5252 OwnershipOutcomeLabel::NotApplicable,
5253 DbClassLabel::Commit,
5254 EmbeddedOperationLabel::Startup,
5255 ErrorClassLabel::None,
5256 );
5257 state = state.wrapping_add(i64::from(*op));
5258 committed.extend_from_slice(&state.to_le_bytes());
5259 }
5260 committed
5261 }
5262
5263 fn observed_sequence_bytes(ops: &[i16], fixture: TelemetryFixture) -> Vec<u8> {
5264 match fixture {
5265 TelemetryFixture::NoSubscriber => apply_observed_sequence(ops),
5266 TelemetryFixture::LocalRecorder => {
5267 let recorder = DebuggingRecorder::new();
5268 with_local_recorder(&recorder, || apply_observed_sequence(ops))
5269 }
5270 TelemetryFixture::FailingExporter => {
5271 let provider = SdkTracerProvider::builder()
5272 .with_simple_exporter(FailingSpanExporter)
5273 .build();
5274 let tracer = provider.tracer("observational-property");
5275 let subscriber = tracing_subscriber::registry()
5276 .with(tracing_opentelemetry::layer().with_tracer(tracer));
5277 let dispatch = tracing::Dispatch::new(subscriber);
5278 let bytes =
5279 tracing::dispatcher::with_default(&dispatch, || apply_observed_sequence(ops));
5280 let _ = provider.force_flush();
5281 bytes
5282 }
5283 }
5284 }
5285
5286 proptest! {
5287 #![proptest_config(ProptestConfig::with_cases(128))]
5288
5289 #[test]
5291 fn startup_is_prefix_safe_and_failure_atomic(
5292 failure_boundary in prop::option::of(0usize..STARTUP_MODEL_PHASES.len()),
5293 ) {
5294 let events = startup_model(failure_boundary);
5295 let attempted = events.iter().filter_map(|event| match event {
5296 StartupModelEvent::Attempt(phase) => Some(*phase),
5297 _ => None,
5298 }).collect::<Vec<_>>();
5299 let completed = events.iter().filter_map(|event| match event {
5300 StartupModelEvent::Complete(phase) => Some(*phase),
5301 _ => None,
5302 }).collect::<Vec<_>>();
5303
5304 prop_assert_eq!(attempted.as_slice(), &STARTUP_MODEL_PHASES[..attempted.len()]);
5305 prop_assert_eq!(completed.as_slice(), &STARTUP_MODEL_PHASES[..completed.len()]);
5306 if let Some(boundary) = failure_boundary {
5307 prop_assert!(!events.contains(&StartupModelEvent::ReturnHandle));
5308 let rollback = events.iter().filter(|event| matches!(
5309 event,
5310 StartupModelEvent::ReleaseOwnership | StartupModelEvent::ClosePool
5311 )).copied().collect::<Vec<_>>();
5312 let expected = match boundary {
5313 0 | 1 => Vec::new(),
5314 2 | 3 => vec![StartupModelEvent::ClosePool],
5315 _ => vec![
5316 StartupModelEvent::ReleaseOwnership,
5317 StartupModelEvent::ClosePool,
5318 ],
5319 };
5320 prop_assert_eq!(rollback, expected);
5321 } else {
5322 prop_assert_eq!(completed.len(), STARTUP_MODEL_PHASES.len());
5323 prop_assert_eq!(events.last(), Some(&StartupModelEvent::ReturnHandle));
5324 }
5325 }
5326
5327 #[test]
5329 fn shutdown_establishes_the_host_flush_boundary(
5330 calls_pending in any::<bool>(),
5331 tasks_pending in any::<bool>(),
5332 independent_failure_mask in 0u8..8,
5333 ) {
5334 let events = shutdown_model(calls_pending, tasks_pending);
5335 prop_assert_eq!(events.first(), Some(&ShutdownModelEvent::CloseAdmission));
5336 prop_assert_eq!(events.get(1), Some(&ShutdownModelEvent::CancelTasks));
5337 prop_assert_eq!(events.last(), Some(&ShutdownModelEvent::ReturnToHost));
5338 for cleanup in [
5339 ShutdownModelEvent::AttemptShardRelease,
5340 ShutdownModelEvent::AttemptOwnershipRelease,
5341 ShutdownModelEvent::AttemptPoolClose,
5342 ] {
5343 prop_assert!(events.contains(&cleanup));
5344 }
5345 prop_assert!(
5346 events.iter().position(|event| *event == ShutdownModelEvent::AttemptShardRelease)
5347 < events.iter().position(|event| *event == ShutdownModelEvent::CloseOwnershipAdmission)
5348 );
5349 prop_assert!(
5350 events.iter().position(|event| *event == ShutdownModelEvent::CloseOwnershipAdmission)
5351 < events.iter().position(|event| *event == ShutdownModelEvent::AttemptOwnershipRelease)
5352 );
5353 let failures = (0..3)
5354 .filter(|bit| independent_failure_mask & (1 << bit) != 0)
5355 .count();
5356 prop_assert_eq!(failures, independent_failure_mask.count_ones() as usize);
5357 prop_assert!(
5358 events.iter().position(|event| *event == ShutdownModelEvent::FinishTelemetry)
5359 < events.iter().position(|event| *event == ShutdownModelEvent::ReturnToHost)
5360 );
5361 }
5362
5363 #[test]
5365 fn embedded_construction_is_transport_and_global_state_neutral(
5366 host in host_instrumentation_strategy(),
5367 storage in storage_mode_strategy(),
5368 ) {
5369 let effects = embedded_construction_effects(host, storage);
5370 prop_assert_eq!(effects.listener_attempts, 0);
5371 prop_assert_eq!(effects.global_install_attempts, 0);
5372 prop_assert_eq!(effects.signal_handler_attempts, 0);
5373 prop_assert_eq!(effects.local_emission_attempts, 1);
5374 }
5375
5376 #[test]
5378 fn startup_stays_zero_listener(
5379 host in host_instrumentation_strategy(),
5380 storage in storage_mode_strategy(),
5381 explicit_listens in 0usize..4,
5382 ) {
5383 let startup = embedded_construction_effects(host, storage);
5386 prop_assert_eq!(startup.listener_attempts, 0);
5387 prop_assert_eq!(
5388 startup.listener_attempts + explicit_listens,
5389 explicit_listens
5390 );
5391 }
5392
5393 #[test]
5395 fn telemetry_is_observational_only(
5396 operations in prop::collection::vec(any::<i16>(), 0..64),
5397 first in telemetry_fixture_strategy(),
5398 second in telemetry_fixture_strategy(),
5399 ) {
5400 let expected = observed_sequence_bytes(&operations, TelemetryFixture::NoSubscriber);
5401 prop_assert_eq!(observed_sequence_bytes(&operations, first), expected.clone());
5402 prop_assert_eq!(observed_sequence_bytes(&operations, second), expected);
5403 }
5404 }
5405
5406 #[tokio::test]
5407 async fn listen_after_shutdown_began_is_refused_without_binding() {
5408 let engine = Engine::start().await.expect("engine starts");
5409 engine.background_cancel.cancel();
5413 let error = engine
5414 .listen("127.0.0.1:0".parse().expect("static loopback address"))
5415 .await
5416 .expect_err("a cancelled engine must refuse to bind");
5417 assert!(matches!(error, EngineListenError::ShutDown));
5418 assert_eq!(engine.listeners.attached(), 0);
5419 }
5420
5421 #[test]
5422 fn every_startup_failure_boundary_rolls_back_without_returning_a_handle() {
5423 for boundary in 0..STARTUP_MODEL_PHASES.len() {
5424 let events = startup_model(Some(boundary));
5425 assert!(!events.contains(&StartupModelEvent::ReturnHandle));
5426 }
5427 }
5428
5429 #[test]
5430 fn durable_startup_report_exposes_only_approved_evidence() {
5431 let report = EngineStartupReport {
5432 storage_mode: EmbeddedStorageMode::ManagedDsql,
5433 cluster: Some(ClusterStartupReport {
5434 region: "eu-west-1".to_owned(),
5435 cluster_id: "cluster-1".to_owned(),
5436 cluster_arn: "arn:aws:dsql:eu-west-1:123456789012:cluster/cluster-1".to_owned(),
5437 endpoint: "cluster-1.dsql.eu-west-1.on.aws".to_owned(),
5438 action: ClusterAction::Recovered,
5439 }),
5440 schema: Some(SchemaStartupReport {
5441 observed_version: 1,
5442 minimum_supported_version: 1,
5443 target_version: 1,
5444 maximum_readable_version: 1,
5445 migration_set_digest: "release-digest".to_owned(),
5446 outcome: SchemaStartupOutcome::Compatible,
5447 }),
5448 ownership: Some(OwnershipStartupReport {
5449 outcome: ControlLeaseAcquireOutcome::Clean,
5450 fence_token: 7,
5451 }),
5452 };
5453 let diagnostic = format!("{report:?}");
5454 assert!(!diagnostic.contains("descriptor-path-secret"));
5455 assert!(!diagnostic.contains("create-token-secret"));
5456 assert!(!diagnostic.contains("credential-secret"));
5457 }
5458
5459 #[tokio::test]
5460 async fn explicit_in_memory_start_preserves_legacy_mode_and_shuts_down_cleanly() {
5461 let engine = Engine::start_with_embedded_config(EmbeddedEngineConfig::default())
5462 .await
5463 .expect("explicit in-memory startup succeeds");
5464 assert_eq!(engine.startup_report(), &EngineStartupReport::in_memory());
5465 engine.shutdown().await.expect("tracked shutdown succeeds");
5466 }
5467
5468 #[test]
5469 fn embedded_lifecycle_composes_with_a_host_local_recorder() {
5470 let recorder = DebuggingRecorder::new();
5471 let runtime = tokio::runtime::Builder::new_current_thread()
5472 .enable_all()
5473 .build()
5474 .expect("embedded fixture runtime");
5475
5476 with_local_recorder(&recorder, || {
5477 runtime.block_on(async {
5478 for _ in 0..2 {
5481 let engine =
5482 Engine::start_with_embedded_config(EmbeddedEngineConfig::default())
5483 .await
5484 .expect("embedded startup");
5485 engine.shutdown().await.expect("embedded shutdown");
5486 }
5487 });
5488 });
5489
5490 let lifecycle_observations = recorder
5491 .snapshotter()
5492 .snapshot()
5493 .into_vec()
5494 .into_iter()
5495 .filter(|(key, _, _, _)| {
5496 key.key().name() == tokeira_observability::EMBEDDED_LIFECYCLE_OPERATIONS_TOTAL
5497 })
5498 .count();
5499 assert_eq!(lifecycle_observations, 2);
5500 }
5501
5502 #[tokio::test]
5503 async fn invalid_explicit_config_fails_before_startup() {
5504 let config = EmbeddedEngineConfig {
5505 startup_timeout_ms: 0,
5506 ..EmbeddedEngineConfig::default()
5507 };
5508 assert!(matches!(
5509 Engine::start_with_embedded_config(config).await,
5510 Err(EmbeddedEngineStartError::InvalidConfiguration(_))
5511 ));
5512 }
5513
5514 #[tokio::test]
5515 async fn unequal_home_counts_fail_before_descriptor_or_listener_work() {
5516 let descriptor_path = std::env::temp_dir().join(format!(
5517 "tokeira-home-count-preflight-{}.json",
5518 IncarnationId::new()
5519 ));
5520 let mut config = EmbeddedEngineConfig::default();
5521 config.server.infrastructure.placement.shard_count = 8;
5522 config.server.infrastructure.placement.bundle_count = 4;
5523 config.storage =
5524 EmbeddedStorageConfig::ManagedDsql(tokeira_config::ManagedEmbeddedDsqlConfig {
5525 intent: tokeira_config::ManagedClusterIntent::CreateOrRecover,
5526 descriptor_path: descriptor_path.clone(),
5527 region: "test-region".to_owned(),
5528 migration_policy: None,
5529 limits: tokeira_config::EmbeddedDsqlLimits::default(),
5530 tags: std::collections::BTreeMap::new(),
5531 });
5532 let error = Engine::start_with_embedded_config(config.clone())
5533 .await
5534 .expect_err("unequal counts must fail before any AWS work");
5535 assert!(matches!(
5536 error,
5537 EmbeddedEngineStartError::InvalidConfiguration(_)
5538 ));
5539 assert!(
5540 error
5541 .to_string()
5542 .contains("shard_count (8) must equal bundle_count (4)")
5543 );
5544 assert!(!descriptor_path.exists());
5545
5546 let address = "127.0.0.1:0".parse().expect("test address");
5547 let error = TokeiradHandle::start_in_memory_with_config(address, config.server)
5548 .await
5549 .expect_err("unequal counts must fail before binding a listener");
5550 assert!(format!("{error:#}").contains("shard_count (8) must equal bundle_count (4)"));
5551 }
5552
5553 #[tokio::test]
5554 async fn durable_snapshot_conflict_fails_before_descriptor_or_aws_work() {
5555 let descriptor_path = std::env::temp_dir().join(format!(
5556 "tokeira-managed-dsql-preflight-{}.json",
5557 IncarnationId::new()
5558 ));
5559 let mut config = EmbeddedEngineConfig::default();
5560 config.server.policy.snapshot = Some(SnapshotPolicyConfig {
5561 location: descriptor_path.with_extension("snapshot"),
5562 interval_ms: 1_000,
5563 });
5564 config.storage =
5565 EmbeddedStorageConfig::ManagedDsql(tokeira_config::ManagedEmbeddedDsqlConfig {
5566 intent: tokeira_config::ManagedClusterIntent::CreateOrRecover,
5567 descriptor_path: descriptor_path.clone(),
5568 region: "eu-west-2".to_owned(),
5569 migration_policy: None,
5570 limits: tokeira_config::EmbeddedDsqlLimits::default(),
5571 tags: std::collections::BTreeMap::new(),
5572 });
5573
5574 assert!(matches!(
5575 Engine::start_with_embedded_config(config).await,
5576 Err(EmbeddedEngineStartError::Phase {
5577 phase: EmbeddedStartupPhase::Configuration
5578 })
5579 ));
5580 assert!(!descriptor_path.exists());
5581 }
5582
5583 #[tokio::test]
5584 async fn drop_synchronously_closes_endpoint_admission() {
5585 let engine = Engine::start().await.expect("engine starts");
5586 let endpoint = engine.endpoint();
5587 drop(engine);
5588
5589 let error = endpoint
5590 .call(InProcessGrpcRequest {
5591 service: "temporal.api.workflowservice.v1.WorkflowService".to_owned(),
5592 rpc: "GetSystemInfo".to_owned(),
5593 headers: http::HeaderMap::new(),
5594 proto: Bytes::new(),
5595 })
5596 .await
5597 .expect_err("dropped engine rejects endpoint clones");
5598 assert_eq!(error.code(), tonic::Code::Unavailable);
5599 }
5600
5601 #[tokio::test]
5602 async fn snapshot_policy_over_dsql_storage_is_rejected() {
5603 let mut config = TokeiraConfig::default();
5604 config.infrastructure.storage = ConfigStorageKind::Dsql;
5605 config.policy.snapshot = Some(SnapshotPolicyConfig {
5606 location: std::path::PathBuf::from("unused.snapshot"),
5607 interval_ms: 60_000,
5608 });
5609 let error = build_and_serve(
5610 "127.0.0.1:0".parse().expect("loopback addr"),
5611 Arc::new(config),
5612 )
5613 .await
5614 .expect_err("snapshot policy over DSQL must be rejected before any I/O");
5615 let message = format!("{error:#}");
5616 assert!(
5617 message.contains("in-memory store's persistence mechanism"),
5618 "the rejection must name the mechanism: {message}"
5619 );
5620 }
5621
5622 #[tokio::test]
5625 async fn property_the_policy_setting_selects_the_check() {
5626 for policy in [
5627 None,
5628 Some(DsqlMigrationPolicy::ValidateOnly),
5629 Some(DsqlMigrationPolicy::Automatic),
5630 ] {
5631 let mut config = TokeiraConfig::default();
5632 config.infrastructure.storage = ConfigStorageKind::Dsql;
5633 config.infrastructure.dsql.migration_policy = policy;
5634 let error = build_and_serve(
5635 "127.0.0.1:0".parse().expect("loopback addr"),
5636 Arc::new(config),
5637 )
5638 .await
5639 .expect_err("no endpoint is configured");
5640 let message = format!("{error:#}");
5641 let refused_by_policy = message.contains("migration_policy");
5644 assert_eq!(
5645 refused_by_policy,
5646 policy == Some(DsqlMigrationPolicy::Automatic),
5647 "{policy:?}: {message}"
5648 );
5649 assert!(
5650 refused_by_policy || message.contains("infrastructure.dsql.endpoint must be set"),
5651 "{policy:?}: {message}"
5652 );
5653 }
5654 }
5655
5656 #[test]
5657 fn each_schema_refusal_says_what_was_found_and_what_to_do() {
5658 let refusal = |decision| {
5659 format!(
5660 "{:#}",
5661 distributed_schema_admission(decision).expect_err("the node refuses")
5662 )
5663 };
5664 let reject = |category| {
5665 SchemaDecision::Reject(tokeira_storage::dsql::SchemaIncompatibility {
5666 observed_version: Some(83),
5667 minimum_supported_version: 1,
5668 target_version: 82,
5669 maximum_readable_version: 82,
5670 category,
5671 })
5672 };
5673 assert_eq!(
5674 distributed_schema_admission(SchemaDecision::Compatible {
5675 current: 82,
5676 legacy_backfill: false,
5677 })
5678 .expect("a compatible schema starts"),
5679 82
5680 );
5681 let message = refusal(SchemaDecision::MigrationRequired {
5682 current: 0,
5683 target: 82,
5684 });
5685 assert!(
5686 message.contains("no Tokeira schema")
5687 && message.contains("V82")
5688 && message.contains("tkr schema setup"),
5689 "{message}"
5690 );
5691 let message = refusal(SchemaDecision::MigrationRequired {
5692 current: 81,
5693 target: 82,
5694 });
5695 assert!(
5696 message.contains("V81")
5697 && message.contains("V82")
5698 && message.contains("tkr schema setup"),
5699 "{message}"
5700 );
5701 let message = refusal(reject(SchemaIncompatibilityCategory::FutureSchema));
5702 assert!(
5703 message.contains("a later release migrated it") && message.contains("FutureSchema"),
5704 "{message}"
5705 );
5706 let message = refusal(reject(SchemaIncompatibilityCategory::BelowMinimum));
5707 assert!(
5708 message.contains("older than this release migrates from"),
5709 "{message}"
5710 );
5711 let message = refusal(reject(SchemaIncompatibilityCategory::ChecksumMismatch {
5712 version: 7,
5713 expected: "expected".to_owned(),
5714 observed: "observed".to_owned(),
5715 }));
5716 assert!(
5717 message.contains("migration ledger doesn't match")
5718 && message.contains("ChecksumMismatch"),
5719 "{message}"
5720 );
5721 for decision in [
5722 SchemaDecision::Initialize { target: 82 },
5723 SchemaDecision::Migrate { from: 81, to: 82 },
5724 ] {
5725 let message = refusal(decision);
5726 assert!(
5727 message.contains("validate-only schema assessment decided"),
5728 "{message}"
5729 );
5730 }
5731 }
5732
5733 proptest! {
5734 #![proptest_config(ProptestConfig::with_cases(256))]
5735
5736 #[test]
5739 fn property_a_node_starts_only_on_a_schema_its_release_runs_on(
5740 known in 2_u32..30,
5741 maximum_back in 0_u32..3,
5742 target_back in 0_u32..3,
5743 minimum_seed in 1_u32..30,
5744 current in 0_u32..34,
5745 tampered in proptest::option::of(0_u32..30),
5746 ) {
5747 use tokeira_storage::{
5748 dsql::{
5749 AppliedMigration, SchemaCompatibilityContract, SchemaObservation,
5750 assess_schema_compatibility,
5751 },
5752 schema_contract::{MigrationIdentity, migration_set_digest, sha256_hex},
5753 };
5754
5755 let maximum = known.saturating_sub(maximum_back).max(1);
5756 let target = maximum.saturating_sub(target_back).max(1);
5757 let minimum = minimum_seed.clamp(1, target);
5758 let recognized: Vec<MigrationIdentity> = (1..=known)
5759 .map(|version| MigrationIdentity {
5760 version,
5761 name: format!("migration_{version}"),
5762 checksum: sha256_hex(format!("sql-{version}").as_bytes()),
5763 })
5764 .collect();
5765 let contract = SchemaCompatibilityContract {
5766 tokeira_release: "this".to_owned(),
5767 minimum_supported_version: minimum,
5768 target_version: target,
5769 maximum_readable_version: maximum,
5770 migration_set_digest: migration_set_digest(&recognized, maximum)
5771 .expect("the readable prefix exists"),
5772 };
5773 let tampered = tampered.filter(|version| *version >= 1 && *version <= current.min(known));
5776 let applied_migrations = (1..=current)
5777 .map(|version| match recognized.get(version as usize - 1) {
5778 Some(identity) => AppliedMigration {
5779 version,
5780 name: identity.name.clone(),
5781 checksum: if tampered == Some(version) {
5782 sha256_hex(b"changed")
5783 } else {
5784 identity.checksum.clone()
5785 },
5786 },
5787 None => AppliedMigration {
5788 version,
5789 name: "later_release".to_owned(),
5790 checksum: sha256_hex(format!("later-{version}").as_bytes()),
5791 },
5792 })
5793 .collect();
5794 let decision = assess_schema_compatibility(
5795 &contract,
5796 &recognized,
5797 &SchemaObservation { applied_migrations, compatibility: None },
5798 SchemaMigrationPolicy::ValidateOnly,
5799 );
5800
5801 let runs = tampered.is_none() && target <= current && current <= maximum;
5802 match distributed_schema_admission(decision) {
5803 Ok(version) => {
5804 prop_assert!(runs, "admitted V{current}");
5805 prop_assert_eq!(version, current);
5806 }
5807 Err(error) => {
5808 prop_assert!(!runs, "refused V{current}");
5809 let message = format!("{error:#}");
5810 if tampered.is_none() && current < target && (current == 0 || current >= minimum) {
5811 prop_assert!(
5812 message.contains(&format!("V{target}")) && message.contains("tkr schema setup"),
5813 "{message}"
5814 );
5815 } else {
5816 prop_assert!(message.contains("schema is incompatible"), "{message}");
5817 }
5818 }
5819 }
5820 }
5821 }
5822
5823 #[tokio::test]
5827 async fn automatic_migration_policy_over_distributed_dsql_is_refused() {
5828 let mut config = TokeiraConfig::default();
5829 config.infrastructure.storage = ConfigStorageKind::Dsql;
5830 config.infrastructure.dsql.migration_policy = Some(DsqlMigrationPolicy::Automatic);
5831 let error = build_and_serve(
5832 "127.0.0.1:0".parse().expect("loopback addr"),
5833 Arc::new(config),
5834 )
5835 .await
5836 .expect_err("automatic must be refused before any I/O");
5837 let message = format!("{error:#}");
5838 assert!(
5839 message.contains("\"automatic\"") && message.contains("tkr schema setup"),
5840 "the refusal must name the policy and the way to migrate: {message}"
5841 );
5842 }
5843
5844 #[tokio::test(flavor = "multi_thread")]
5851 async fn sigterm_drains_persists_final_snapshot_and_next_boot_restores() {
5852 use tokeira_proto::workflowservice::{
5853 DescribeWorkflowExecutionRequest, StartWorkflowExecutionRequest,
5854 workflow_service_client::WorkflowServiceClient,
5855 };
5856
5857 let scratch =
5858 std::env::temp_dir().join(format!("tokeira-engine-sigterm-{}", std::process::id()));
5859 std::fs::create_dir_all(&scratch).expect("create scratch dir");
5860 let snapshot_location = scratch.join("tokeirad.snapshot");
5861
5862 let reserved = std::net::TcpListener::bind("127.0.0.1:0").expect("reserve port");
5866 let addr = reserved.local_addr().expect("reserved addr");
5867 drop(reserved);
5868
5869 let mut config = TokeiraConfig::default();
5870 config.infrastructure.network.grpc_addr = addr.to_string();
5871 config.policy.nexus_completion.http_addr = "127.0.0.1:0".to_string();
5874 config.policy.nexus_completion.system_callback_url = "http://127.0.0.1:0".to_string();
5875 config.policy.snapshot = Some(SnapshotPolicyConfig {
5876 location: snapshot_location.clone(),
5877 interval_ms: 3_600_000,
5880 });
5881 let config_path = scratch.join("tokeirad.toml");
5882 std::fs::write(&config_path, config.to_toml().expect("render config"))
5883 .expect("write config");
5884
5885 let mut daemon = tokio::spawn(run_from_cli(Cli {
5886 config: Some(config_path.display().to_string()),
5887 dump_config: false,
5888 version: false,
5889 verbose: false,
5890 json: false,
5891 }));
5892
5893 let target = format!("http://{addr}");
5898 let deadline = tokio::time::Instant::now() + std::time::Duration::from_secs(20);
5899 let mut client = loop {
5900 assert!(
5901 tokio::time::Instant::now() < deadline,
5902 "daemon must start serving"
5903 );
5904 tokio::select! {
5905 result = &mut daemon => {
5906 panic!("daemon exited before serving: {result:?}");
5907 }
5908 connected = WorkflowServiceClient::connect(target.clone()) => {
5909 match connected {
5910 Ok(client) => break client,
5911 Err(_) => {
5912 tokio::time::sleep(std::time::Duration::from_millis(50)).await;
5913 }
5914 }
5915 }
5916 }
5917 };
5918 client
5919 .start_workflow_execution(StartWorkflowExecutionRequest {
5920 namespace: "default".to_owned(),
5921 workflow_id: "sigterm-canary".to_owned(),
5922 workflow_type: Some(tokeira_proto::common::WorkflowType {
5923 name: "sigterm-workflow".to_owned(),
5924 }),
5925 task_queue: Some(tokeira_proto::taskqueue::TaskQueue {
5926 name: "sigterm-queue".to_owned(),
5927 ..Default::default()
5928 }),
5929 request_id: "sigterm-start".to_owned(),
5930 ..Default::default()
5931 })
5932 .await
5933 .expect("start canary workflow");
5934
5935 let delivered = std::process::Command::new("kill")
5938 .args(["-TERM", &std::process::id().to_string()])
5939 .status()
5940 .expect("send SIGTERM");
5941 assert!(delivered.success(), "kill -TERM must be deliverable");
5942
5943 daemon
5944 .await
5945 .expect("daemon task must not panic")
5946 .expect("SIGTERM drain must exit cleanly");
5947 assert!(
5948 snapshot_location.is_file(),
5949 "the drain must persist the final snapshot"
5950 );
5951
5952 let restored =
5954 TokeiradHandle::start_in_memory_with_config("127.0.0.1:0".parse().expect("addr"), {
5955 let mut config = TokeiraConfig::default();
5956 config.policy.snapshot = Some(SnapshotPolicyConfig {
5957 location: snapshot_location.clone(),
5958 interval_ms: 3_600_000,
5959 });
5960 config
5961 })
5962 .await
5963 .expect("restart over the final snapshot");
5964 let mut client =
5965 WorkflowServiceClient::connect(format!("http://{}", restored.bound_addr()))
5966 .await
5967 .expect("connect to restarted server");
5968 let described = client
5969 .describe_workflow_execution(DescribeWorkflowExecutionRequest {
5970 namespace: "default".to_owned(),
5971 execution: Some(tokeira_proto::common::WorkflowExecution {
5972 workflow_id: "sigterm-canary".to_owned(),
5973 run_id: String::new(),
5974 }),
5975 })
5976 .await
5977 .expect("restored workflow must be describable")
5978 .into_inner();
5979 assert!(
5980 described.workflow_execution_info.is_some(),
5981 "state must survive the SIGTERM drain via snapshot restore"
5982 );
5983 restored.shutdown().await.expect("clean restart shutdown");
5984 let _ = std::fs::remove_dir_all(&scratch);
5985 }
5986
5987 #[test]
5988 fn embedded_config_removes_daemon_storage_and_placement() {
5989 let mut config = TokeiraConfig::default();
5990 config.infrastructure.storage = ConfigStorageKind::Dsql;
5991 config.infrastructure.placement.controller_endpoint =
5992 Some("http://controller.internal:8080".to_owned());
5993
5994 let config = embedded_config(config).expect("embedded defaults are valid");
5995
5996 assert_eq!(config.infrastructure.storage, ConfigStorageKind::InMemory);
5997 assert!(
5998 config
5999 .infrastructure
6000 .placement
6001 .controller_endpoint
6002 .is_none()
6003 );
6004 }
6005
6006 #[tokio::test]
6007 async fn embedded_system_nexus_callback_never_falls_through_to_http() {
6008 let client = InProcessNexusCompletionClient::new();
6009 let target = system_callback_post_url(EMBEDDED_NEXUS_CALLBACK_BASE);
6010 let outcome = client
6011 .complete_operation(
6012 &target,
6013 "not-yet-attached",
6014 "operation-token",
6015 NexusCompletion::Succeeded(tokeira_types::Payloads::default()),
6016 &[],
6017 )
6018 .await
6019 .expect("the in-process preflight outcome is classifiable");
6020
6021 assert!(matches!(
6022 outcome,
6023 CompletionDeliveryOutcome::RetryableError { detail }
6024 if detail == "embedded Nexus completion runtime is unavailable"
6025 ));
6026 }
6027
6028 #[tokio::test]
6032 async fn bounded_body_caps_oversized_payload() {
6033 match collect_bounded_body(Full::new(Bytes::from(vec![0u8; 8])), 16).await {
6035 BoundedBody::Collected(bytes) => assert_eq!(bytes.len(), 8),
6036 other => panic!("expected Collected, got {other:?}"),
6037 }
6038 assert!(matches!(
6040 collect_bounded_body(Full::new(Bytes::from(vec![0u8; 17])), 16).await,
6041 BoundedBody::TooLarge
6042 ));
6043 }
6044
6045 #[test]
6046 fn with_loopback_port_rewrites_or_appends_port() {
6047 assert_eq!(
6049 with_loopback_port("http://127.0.0.1:7253", 51_000),
6050 "http://127.0.0.1:51000"
6051 );
6052 assert_eq!(
6054 with_loopback_port("http://127.0.0.1:7253/", 51_000),
6055 "http://127.0.0.1:51000"
6056 );
6057 assert_eq!(
6060 with_loopback_port("http://localhost", 51_000),
6061 "http://localhost:51000"
6062 );
6063 }
6064
6065 #[test]
6066 fn version_renderer_is_deterministic() {
6067 let short = render_build_info(false, false);
6068 let verbose = render_build_info(true, false);
6069 let json = render_build_info(false, true);
6070
6071 assert_eq!(short, render_build_info(false, false));
6072 assert_eq!(verbose, render_build_info(true, false));
6073 assert_eq!(json, render_build_info(false, true));
6074 assert!(short.contains(tokeira_build_info::SERVER_VERSION));
6075 assert!(short.contains(tokeira_build_info::TEMPORAL_PROTO_VERSION));
6076 assert!(short.contains(tokeira_build_info::TEMPORAL_SERVER_COMPAT));
6077 assert!(json.contains("server_version"));
6078 assert!(json.contains("temporal_proto_version"));
6079 }
6080
6081 #[test]
6082 fn reported_problem_search_attribute_has_exact_v131_keyword_list() {
6083 let mut search_attributes = tokeira_types::SearchAttributes::default();
6084 apply_reported_problem_search_attribute(
6085 &mut search_attributes,
6086 Some(WorkflowTaskReportedProblem {
6087 attempts_since_last_success: 5,
6088 problem: tokeira_kernel::WorkflowTaskProblem::Failed(
6089 tokeira_kernel::WorkflowTaskFailedCause::WorkflowWorkerUnhandledFailure,
6090 ),
6091 }),
6092 );
6093
6094 assert_eq!(
6095 search_attributes.0.get("TemporalReportedProblems"),
6096 Some(&SearchAttrValue::KeywordList(vec![
6097 "category=WorkflowTaskFailed".to_string(),
6098 "cause=WorkflowTaskFailedCauseWorkflowWorkerUnhandledFailure".to_string(),
6099 ]))
6100 );
6101 }
6102
6103 #[test]
6104 fn reported_problem_search_attribute_renders_timeout_category() {
6105 let mut search_attributes = tokeira_types::SearchAttributes::default();
6109 apply_reported_problem_search_attribute(
6110 &mut search_attributes,
6111 Some(WorkflowTaskReportedProblem {
6112 attempts_since_last_success: 5,
6113 problem: tokeira_kernel::WorkflowTaskProblem::TimedOutStartToClose,
6114 }),
6115 );
6116
6117 assert_eq!(
6118 search_attributes.0.get("TemporalReportedProblems"),
6119 Some(&SearchAttrValue::KeywordList(vec![
6120 "category=WorkflowTaskTimedOut".to_string(),
6121 "cause=WorkflowTaskTimedOutCauseStartToClose".to_string(),
6122 ]))
6123 );
6124 }
6125
6126 fn self_assigning_runtime(
6127 store: Arc<InMemoryStore>,
6128 shard_count: u32,
6129 ) -> Arc<TokeiraRuntime<InMemoryStore>> {
6130 Arc::new(TokeiraRuntime::new_with_nexus_and_shards_and_endpoint(
6131 store,
6132 2,
6133 tokeira_runtime::LaneConfig::default(),
6134 tokeira_runtime::TimerScannerConfig::default(),
6135 tokeira_runtime::WorkflowTimeoutScannerConfig::default(),
6136 tokeira_runtime::BacklogConfig::default(),
6137 tokeira_runtime::ActivityTimeoutScannerConfig::default(),
6138 tokeira_runtime::NexusTimeoutScannerConfig::default(),
6139 NexusEndpointRegistry::default(),
6140 Arc::new(tokeira_runtime::NoopNexusHttpClient),
6141 NexusCompletionDeps::default(),
6142 shard_count,
6143 IncarnationId::new().to_string(),
6144 "127.0.0.1:7233".to_owned(),
6145 false,
6146 None,
6147 ))
6148 }
6149
6150 fn boot_start_request(
6151 namespace_id: tokeira_types::NamespaceId,
6152 index: usize,
6153 ) -> tokeira_kernel::StartRequest {
6154 let run_id = tokeira_types::RunId::new();
6155 let workflow_id = tokeira_types::WorkflowId(format!("boot-{index}"));
6156 let now = time::OffsetDateTime::now_utc();
6157 tokeira_kernel::StartRequest {
6158 advice_policy: tokeira_kernel::ContinueAsNewAdvicePolicy::V1_31_0,
6159 initiator: None,
6160 run_key: tokeira_types::RunKey::derive(namespace_id, &workflow_id, run_id),
6161 namespace_id,
6162 workflow_id,
6163 run_id,
6164 workflow_type: tokeira_types::WorkflowType("boot".to_owned()),
6165 task_queue: tokeira_types::TaskQueueName("boot-queue".to_owned()),
6166 input: tokeira_types::Payloads::default(),
6167 header: None,
6168 memo: tokeira_types::Memo::default(),
6169 search_attributes: tokeira_types::SearchAttributes::default(),
6170 workflow_execution_timeout: None,
6171 workflow_run_timeout: None,
6172 workflow_task_timeout: time::Duration::seconds(10),
6173 retry_policy: None,
6174 conflict_policy: tokeira_kernel::WorkflowIdConflictPolicy::Fail,
6175 reuse_policy: tokeira_kernel::WorkflowIdReusePolicy::AllowDuplicate,
6176 deployment: None,
6177 build_id: None,
6178 versioning_override: None,
6179 workflow_start_delay: None,
6180 completion_callbacks: Vec::new(),
6181 user_metadata: None,
6182 links: Vec::new(),
6183 on_conflict_options: None,
6184 priority: None,
6185 attempt: 1,
6186 continued_execution_run_id: None,
6187 first_execution_run_id: None,
6188 parent_run_key: None,
6189 parent_workflow_id: None,
6190 parent_run_id: None,
6191 parent_namespace_id: None,
6192 parent_namespace_name: None,
6193 parent_initiated_event_id: 0,
6194 root_workflow_id: None,
6195 root_run_id: None,
6196 original_execution_run_id: Some(run_id),
6197 continued_failure: None,
6198 last_completion_result: None,
6199 first_run_started_at: None,
6200 request: tokeira_types::RequestContext {
6201 request_id: tokeira_types::RequestId(format!("boot-start-{index}")),
6202 caller_identity: None,
6203 principal: None,
6204 received_at: now,
6205 },
6206 now,
6207 client_cron_schedule: None,
6208 cron_schedule: None,
6209 eager_execution_accepted: false,
6210 reserved_poller_identity: None,
6211 inherited_versioning_info: None,
6212 }
6213 }
6214
6215 #[tokio::test]
6216 async fn self_assigned_dsql_boot_recovers_every_shard_before_admission() -> Result<()> {
6217 let shard_count = 2;
6222 let store = Arc::new(InMemoryStore::with_shard_count(shard_count));
6223 let endpoint = NodeEndpoint {
6224 host: "127.0.0.1".to_owned(),
6225 port: 7233,
6226 };
6227 let namespace_id = tokeira_types::NamespaceId::new();
6228
6229 let previous = self_assigning_runtime(Arc::clone(&store), shard_count);
6230 self_assign_dsql_shards(
6231 previous.as_ref(),
6232 store.as_ref(),
6233 shard_count,
6234 &IncarnationId::new(),
6235 &endpoint,
6236 true,
6237 )
6238 .await?;
6239 let mut started = Vec::new();
6240 let mut shards = std::collections::BTreeSet::new();
6241 for index in 0..8 {
6242 let request = boot_start_request(namespace_id, index);
6243 shards.insert(tokeira_types::execution_home_bundle(
6244 namespace_id.0.as_bytes(),
6245 request.workflow_id.0.as_bytes(),
6246 shard_count,
6247 ));
6248 started.push(request.run_key);
6249 let result = previous.start_workflow(request).await?;
6250 assert!(matches!(
6251 result,
6252 tokeira_storage::CommitResult::Applied { .. }
6253 ));
6254 }
6255 assert_eq!(shards.len(), 2, "the seeded runs span both shards");
6256
6257 let restarted = self_assigning_runtime(Arc::clone(&store), shard_count);
6258 let leases = self_assign_dsql_shards(
6259 restarted.as_ref(),
6260 store.as_ref(),
6261 shard_count,
6262 &IncarnationId::new(),
6263 &endpoint,
6264 true,
6265 )
6266 .await?;
6267 assert_eq!(leases.len(), 2);
6268 let mut active = restarted.active_shards();
6269 active.sort();
6270 assert_eq!(active, vec![ShardId(0), ShardId(1)]);
6271
6272 let queue = tokeira_types::QueueKey {
6273 namespace_id,
6274 task_queue: tokeira_types::TaskQueueName("boot-queue".to_owned()),
6275 task_kind: tokeira_types::TaskKind::Workflow,
6276 deployment: None,
6277 build_id: None,
6278 };
6279 let mut offered = Vec::new();
6280 while offered.len() < started.len() {
6281 let task = restarted
6282 .poll_workflow_task(
6283 queue.clone(),
6284 tokeira_types::WorkerIdentity("worker".to_owned()),
6285 tokio::time::Duration::from_secs(1),
6286 )
6287 .await?
6288 .expect("every pending first workflow task is offered again");
6289 offered.push(task.run_key);
6290 }
6291 offered.sort();
6292 started.sort();
6293 assert_eq!(offered, started);
6294 Ok(())
6295 }
6296
6297 #[tokio::test]
6298 async fn build_nexus_endpoint_store_resolves_worker_namespace_names() {
6299 let cache = InMemoryNamespaceCache::new();
6300 let namespace = ResolvedNamespace::active("payments");
6301 let namespace_id = namespace_id_for("payments");
6302 cache.insert(namespace).await.expect("namespace insert");
6303
6304 let store = build_nexus_endpoint_store(
6305 &cache,
6306 HashMap::from([(
6307 "payments-endpoint".to_string(),
6308 BootstrapNexusEndpointConfig {
6309 target: BootstrapNexusEndpointTarget::Worker {
6310 namespace_name: "payments".to_string(),
6311 task_queue: "nexus-q".to_string(),
6312 },
6313 },
6314 )]),
6315 )
6316 .await
6317 .expect("store should build");
6318
6319 let registry = NexusEndpointRegistry::new(store);
6320 let config = registry.resolve("payments-endpoint").expect("endpoint");
6321 assert_eq!(
6322 config,
6323 NexusEndpointConfig {
6324 target: EndpointTarget::Worker {
6325 namespace_id,
6326 task_queue: tokeira_types::TaskQueueName("nexus-q".to_string()),
6327 },
6328 }
6329 );
6330 }
6331
6332 #[tokio::test]
6333 async fn build_nexus_endpoint_store_rejects_unknown_worker_namespace() {
6334 let cache = InMemoryNamespaceCache::new();
6335
6336 let result = build_nexus_endpoint_store(
6337 &cache,
6338 HashMap::from([(
6339 "payments-endpoint".to_string(),
6340 BootstrapNexusEndpointConfig {
6341 target: BootstrapNexusEndpointTarget::Worker {
6342 namespace_name: "missing".to_string(),
6343 task_queue: "nexus-q".to_string(),
6344 },
6345 },
6346 )]),
6347 )
6348 .await;
6349
6350 assert!(result.is_err(), "missing namespace should fail");
6351 let error = result.expect_err("error");
6352
6353 assert!(error.to_string().contains("namespace `missing` not found"));
6354 }
6355
6356 #[test]
6357 fn placement_helpers_build_runtime_and_edge_config_from_server_config() {
6358 let mut config = TokeiraConfig::default();
6359 config.infrastructure.placement.controller_endpoint =
6360 Some("http://127.0.0.1:7240".to_string());
6361 config.infrastructure.placement.node_host = "10.0.0.9".to_string();
6362 config.infrastructure.placement.node_port = Some(8123);
6363 config.infrastructure.placement.shard_count = 8;
6364 config.infrastructure.placement.bundle_count = 4;
6365 config.infrastructure.placement.partition_count = 64;
6366 config.infrastructure.placement.hash_version = 2;
6367 let listen_addr = "127.0.0.1:7233".parse().unwrap();
6368 let node_id = IncarnationId::new();
6369 let endpoint = configured_node_endpoint(&config, listen_addr);
6370 let membership = membership_config(
6371 &config,
6372 node_id,
6373 endpoint.clone(),
6374 "http://127.0.0.1:7240".into(),
6375 );
6376
6377 assert_eq!(endpoint.as_authority(), "10.0.0.9:8123");
6378 assert_eq!(membership.node_id, node_id);
6379 assert_eq!(membership.node_endpoint, endpoint);
6380 assert_eq!(
6381 placement_config(&config),
6382 PlacementConfig {
6383 shard_count: 8,
6384 bundle_count: 4,
6385 partition_count: 64,
6386 hash_version: 2,
6387 }
6388 );
6389 }
6390
6391 #[test]
6392 fn dsql_pool_config_uses_effective_placement_values() {
6393 let mut config = TokeiraConfig::default();
6394 config.infrastructure.placement.shard_count = 32;
6395 config.infrastructure.placement.partition_count = 4;
6396
6397 let (pool_config, _ddb_client) =
6398 dsql_pool_config_with_client(&config, tokeira_storage::dsql::offline_ddb_client());
6399
6400 assert_eq!(pool_config.shard_count, 32);
6401 assert_eq!(pool_config.projection_partition_count, 4);
6402 }
6403
6404 #[test]
6405 fn wire_coverage_out_path_prefers_env_override_then_falls_back_to_default() {
6406 unsafe { std::env::remove_var(WIRE_COVERAGE_OUT_ENV) };
6412 assert_eq!(
6413 wire_coverage_out_path(),
6414 PathBuf::from(WIRE_COVERAGE_DEFAULT_OUT)
6415 );
6416
6417 unsafe { std::env::set_var(WIRE_COVERAGE_OUT_ENV, " /tmp/cov.json ") };
6420 assert_eq!(wire_coverage_out_path(), PathBuf::from("/tmp/cov.json"));
6421
6422 unsafe { std::env::set_var(WIRE_COVERAGE_OUT_ENV, " ") };
6425 assert_eq!(
6426 wire_coverage_out_path(),
6427 PathBuf::from(WIRE_COVERAGE_DEFAULT_OUT)
6428 );
6429
6430 unsafe { std::env::remove_var(WIRE_COVERAGE_OUT_ENV) };
6432 }
6433
6434 #[test]
6435 fn export_wire_coverage_writes_pretty_json_snapshot() {
6436 let recorder = WireCoverageRecorder::new();
6437 recorder.record(
6438 "/temporal.api.workflowservice.v1.WorkflowService/StartWorkflowExecution",
6439 0,
6440 );
6441
6442 let dir = std::env::temp_dir().join(format!("tokeirad-cov-{}", std::process::id()));
6443 std::fs::create_dir_all(&dir).expect("temp dir");
6444 let path = dir.join("wire-coverage.json");
6445
6446 export_wire_coverage(&recorder, &path).expect("export succeeds");
6447
6448 let written = std::fs::read_to_string(&path).expect("evidence file readable");
6449 assert!(written.contains("StartWorkflowExecution"));
6450 assert!(written.contains('\n'));
6452
6453 std::fs::remove_dir_all(&dir).ok();
6454 }
6455}