1use std::collections::{BTreeSet, HashMap, HashSet};
20use std::future::Future;
21use std::pin::Pin;
22use std::sync::Arc;
23use std::sync::RwLock as StdRwLock;
24#[cfg(not(target_arch = "wasm32"))]
25use std::sync::{Mutex as StdMutex, OnceLock, Weak};
26
27use meerkat_core::lifecycle::{InputId, RunId};
28use meerkat_core::time_compat::Instant;
29use meerkat_core::tool_scope::ToolScopeTurnOverlay;
30use meerkat_core::types::SessionId;
31use meerkat_core::{BlobId, BlobPayload, BlobRef, BlobStore, BlobStoreError};
32use meerkat_core::{
33 DeferredToolLoadAuthority, SessionToolVisibilityState, ToolFilter, ToolScopeApplyError,
34 ToolScopeRevision, ToolScopeStageError, ToolVisibilityOwner, ToolVisibilityWitness,
35};
36
37use crate::accept::AcceptOutcome;
38use crate::driver::ephemeral::EphemeralRuntimeDriver;
39use crate::driver::persistent::PersistentRuntimeDriver;
40use crate::identifiers::LogicalRuntimeId;
41use crate::input::Input;
42use crate::input_state::{
43 InputAbandonReason, InputLifecycleState, InputStateSeed, InputTerminalOutcome,
44};
45use crate::meerkat_machine_types::{
46 HydratedSessionLlmState, MeerkatAdmittedInputSnapshot, MeerkatArchiveSnapshot,
47 MeerkatBindingSnapshot, MeerkatCompletionWaiterSnapshot, MeerkatCompletionWaitersSnapshot,
48 MeerkatControlSnapshot, MeerkatCursorSnapshot, MeerkatDrainSnapshot, MeerkatDriverKind,
49 MeerkatFormalStateProjection, MeerkatInputsSnapshot, MeerkatLedgerSnapshot,
50 MeerkatMachineCommand, MeerkatMachineCommandError, MeerkatMachineCommandResult,
51 MeerkatMachineRunFailure, MeerkatMachineSpineSnapshot, MeerkatOpsSnapshot,
52 MemberResidencyExpectation, SessionLlmCapabilityDelta, SessionLlmCapabilitySurface,
53 SessionLlmReconfigureHost, SessionLlmReconfigureReport, SessionLlmReconfigureRequest,
54 SessionToolVisibilityDelta,
55};
56use crate::runtime_state::RuntimeState;
57use crate::service_ext::SessionServiceRuntimeExt;
58use crate::store::RuntimeStore;
59use crate::tokio;
60use crate::tokio::sync::{Mutex, RwLock, mpsc};
61#[cfg(test)]
62use crate::traits::RuntimeDriver;
63use crate::traits::{
64 DestroyReport, RecoveryReport, RecycleReport, ResetReport, RetireReport,
65 RuntimeControlPlaneError, RuntimeDriverError,
66};
67
68#[allow(clippy::expect_used)]
69#[cfg(test)]
70pub(crate) fn recover_projected_authority(
71 state: dsl::MeerkatMachineState,
72 context: &'static str,
73) -> dsl::MeerkatMachineAuthority {
74 dsl::MeerkatMachineAuthority::recover_from_state(state).expect(context)
75}
76
77struct ToolVisibilityOwnerGeneratedAuthorityBridgeToken;
78
79static TOOL_VISIBILITY_OWNER_GENERATED_AUTHORITY_BRIDGE_TOKEN:
80 ToolVisibilityOwnerGeneratedAuthorityBridgeToken =
81 ToolVisibilityOwnerGeneratedAuthorityBridgeToken;
82
83fn tool_visibility_owner_generated_authority_bridge_token()
84-> &'static (dyn std::any::Any + Send + Sync) {
85 &TOOL_VISIBILITY_OWNER_GENERATED_AUTHORITY_BRIDGE_TOKEN
86}
87
88#[doc(hidden)]
89#[allow(improper_ctypes_definitions, unsafe_code)]
90#[unsafe(export_name = concat!(
91 "__meerkat_runtime_generated_authority_bridge_token_is_valid_v1_tool_visibility_owner_",
92 env!("MEERKAT_GENERATED_AUTHORITY_BRIDGE_SYMBOL_SUFFIX")
93))]
94pub extern "Rust" fn tool_visibility_owner_generated_authority_bridge_token_is_valid(
95 token: &(dyn std::any::Any + Send + Sync),
96) -> bool {
97 token.is::<ToolVisibilityOwnerGeneratedAuthorityBridgeToken>()
98}
99
100fn generated_tool_visibility_owner(
101 owner: Arc<dyn ToolVisibilityOwner>,
102) -> Result<meerkat_core::GeneratedToolVisibilityOwner, String> {
103 #[allow(improper_ctypes_definitions, unsafe_code)]
104 unsafe extern "Rust" {
105 #[link_name = concat!(
106 "__meerkat_core_runtime_generated_tool_visibility_owner_build_v1_",
107 env!("MEERKAT_GENERATED_AUTHORITY_BRIDGE_SYMBOL_SUFFIX")
108 )]
109 fn core_runtime_generated_tool_visibility_owner_build(
110 token: &'static (dyn std::any::Any + Send + Sync),
111 owner: Arc<dyn ToolVisibilityOwner>,
112 ) -> Result<meerkat_core::GeneratedToolVisibilityOwner, String>;
113 }
114 #[allow(unsafe_code)]
115 unsafe {
116 core_runtime_generated_tool_visibility_owner_build(
117 tool_visibility_owner_generated_authority_bridge_token(),
118 owner,
119 )
120 }
121}
122
123#[derive(Clone)]
131pub struct StandaloneSessionRuntimeAuthorities {
132 tool_visibility_owner: meerkat_core::GeneratedToolVisibilityOwner,
133 turn_state: Arc<dyn meerkat_core::TurnStateHandle>,
134 model_routing: Arc<dyn meerkat_core::handles::ModelRoutingHandle>,
135 #[cfg(test)]
136 model_routing_test: Arc<crate::handles::RuntimeModelRoutingHandle>,
137}
138
139impl StandaloneSessionRuntimeAuthorities {
140 pub fn tool_visibility_owner(&self) -> &meerkat_core::GeneratedToolVisibilityOwner {
141 &self.tool_visibility_owner
142 }
143
144 pub fn turn_state(&self) -> &Arc<dyn meerkat_core::TurnStateHandle> {
145 &self.turn_state
146 }
147
148 pub fn model_routing(&self) -> &Arc<dyn meerkat_core::handles::ModelRoutingHandle> {
149 &self.model_routing
150 }
151
152 #[cfg(test)]
153 pub(crate) fn commit_sticky_model_fallback_for_test(
154 &self,
155 previous_identity: &meerkat_core::SessionLlmIdentity,
156 target_identity: &meerkat_core::SessionLlmIdentity,
157 target_profile: &meerkat_core::ModelProfileWitness,
158 visibility_plan: &meerkat_core::handles::StickyModelFallbackVisibilityPlan,
159 retry_attempt: u32,
160 ) -> Result<(), meerkat_core::handles::DslTransitionError> {
161 self.model_routing_test
162 .commit_sticky_model_fallback_for_test(
163 previous_identity,
164 target_identity,
165 target_profile,
166 visibility_plan,
167 retry_attempt,
168 )
169 }
170}
171
172pub fn standalone_session_runtime_authorities(
178 session_id: &SessionId,
179 current_identity: &meerkat_core::SessionLlmIdentity,
180 model_profile: Option<&meerkat_core::model_profile::ModelProfile>,
181 capability_base_filter: &ToolFilter,
182) -> Result<StandaloneSessionRuntimeAuthorities, String> {
183 let mut authority = dsl_authority::new_registered_authority(session_id)
184 .map_err(|err| dsl_authority::map_error(err, "standalone visibility authority"))?;
185 let (current_capability_surface, current_capability_surface_status) = match model_profile {
186 Some(profile) => (
187 Some(dsl::SessionLlmCapabilitySurface {
188 supports_temperature: profile.supports_temperature,
189 supports_thinking: profile.supports_thinking,
190 supports_reasoning: profile.supports_reasoning,
191 inline_video: profile.inline_video,
192 vision: profile.vision,
193 image_input: profile.image_input,
194 image_tool_results: profile.image_tool_results,
195 supports_web_search: profile.supports_web_search,
196 image_generation: profile.image_generation,
197 realtime: profile.realtime,
198 call_timeout_secs: profile.call_timeout_secs,
199 }),
200 dsl::SessionLlmCapabilitySurfaceStatus::Resolved,
201 ),
202 None => (None, dsl::SessionLlmCapabilitySurfaceStatus::Unresolved),
203 };
204 dsl::MeerkatMachineMutator::apply(
205 &mut authority,
206 dsl::MeerkatMachineInput::HydrateSessionLlmState {
207 current_identity: dsl::SessionLlmIdentity::from_domain(current_identity),
208 current_capability_surface,
209 current_capability_surface_status,
210 current_capability_base_filter: dsl::ToolFilter::from_domain(capability_base_filter),
211 },
212 )
213 .map_err(|err| dsl_authority::map_error(err, "standalone visibility hydration"))?;
214 dsl::MeerkatMachineMutator::apply(
215 &mut authority,
216 dsl::MeerkatMachineInput::SetModelRoutingBaseline {
217 baseline_model: current_identity.model.clone(),
218 realtime_capable: model_profile.is_some_and(|profile| profile.realtime),
219 },
220 )
221 .map_err(|err| dsl_authority::map_error(err, "standalone model routing baseline"))?;
222 let authority = Arc::new(std::sync::Mutex::new(authority));
223 let owner = Arc::new(MachineToolVisibilityOwner::new());
224 owner.bind_dsl_authority(Arc::clone(&authority));
225 let shared_handle_authority = Arc::new(crate::handles::HandleDslAuthority::from_shared(
226 Arc::clone(&authority),
227 ));
228 let tool_visibility_owner =
229 generated_tool_visibility_owner(Arc::clone(&owner) as Arc<dyn ToolVisibilityOwner>)?;
230 let turn_state = Arc::new(crate::handles::RuntimeTurnStateHandle::standalone(
231 Arc::clone(&shared_handle_authority),
232 session_id.clone(),
233 )) as Arc<dyn meerkat_core::TurnStateHandle>;
234 let runtime_model_routing = Arc::new(
235 crate::handles::RuntimeModelRoutingHandle::new_with_visibility_owner(
236 shared_handle_authority,
237 owner,
238 ),
239 );
240 let model_routing =
241 Arc::clone(&runtime_model_routing) as Arc<dyn meerkat_core::handles::ModelRoutingHandle>;
242 Ok(StandaloneSessionRuntimeAuthorities {
243 tool_visibility_owner,
244 turn_state,
245 model_routing,
246 #[cfg(test)]
247 model_routing_test: runtime_model_routing,
248 })
249}
250
251pub fn standalone_tool_visibility_owner(
255 session_id: &SessionId,
256 current_identity: &meerkat_core::SessionLlmIdentity,
257 model_profile: Option<&meerkat_core::model_profile::ModelProfile>,
258 capability_base_filter: &ToolFilter,
259) -> Result<meerkat_core::GeneratedToolVisibilityOwner, String> {
260 standalone_session_runtime_authorities(
261 session_id,
262 current_identity,
263 model_profile,
264 capability_base_filter,
265 )
266 .map(|authorities| authorities.tool_visibility_owner)
267}
268
269#[derive(Debug, thiserror::Error)]
271pub enum RuntimeBindingsError {
272 #[error("session {0} not found in runtime adapter after registration")]
274 SessionNotFound(SessionId),
275 #[error("failed to prepare runtime bindings for session {0}: {1}")]
277 PrepareFailed(SessionId, String),
278}
279
280#[derive(Debug, Clone, Copy, PartialEq, Eq)]
282pub struct InputPublicStateProjection {
283 pub lifecycle_state: dsl::InputPublicLifecycleState,
284 pub terminal_outcome: Option<dsl::InputPublicTerminalOutcome>,
285}
286
287#[derive(Debug, Clone, Copy, PartialEq, Eq)]
290pub struct RuntimeLifecycleFacts {
291 pub terminality: dsl::RuntimeLifecycleTerminality,
292 pub input_admission: dsl::RuntimeInputAdmission,
293 pub queue_admission: dsl::RuntimeQueueAdmission,
294 pub prepare_admission: dsl::RuntimePrepareAdmission,
295 pub ingress_admission: dsl::RuntimeIngressAdmission,
296}
297
298impl RuntimeLifecycleFacts {
299 #[must_use]
300 pub fn can_accept_input(self) -> bool {
301 self.input_admission == dsl::RuntimeInputAdmission::AcceptsInput
302 }
303
304 #[must_use]
305 pub fn can_process_queue(self) -> bool {
306 self.queue_admission == dsl::RuntimeQueueAdmission::ProcessesQueue
307 }
308
309 #[must_use]
310 pub fn can_prepare_run(self) -> bool {
311 self.prepare_admission == dsl::RuntimePrepareAdmission::Ready
312 }
313
314 #[must_use]
315 pub fn is_terminal(self) -> bool {
316 self.terminality == dsl::RuntimeLifecycleTerminality::Terminal
317 }
318}
319
320#[derive(Debug, Clone, Copy, PartialEq, Eq)]
323pub struct RuntimeLoopQueueAdmissionPlan {
324 pub queue_admission: dsl::RuntimeQueueAdmission,
325 pub run_binding: dsl::RuntimeLoopRunBinding,
326}
327
328impl RuntimeLoopQueueAdmissionPlan {
329 #[must_use]
330 pub fn can_process_queue(self) -> bool {
331 self.queue_admission == dsl::RuntimeQueueAdmission::ProcessesQueue
332 }
333
334 #[must_use]
335 pub fn uses_prebound_run(self) -> bool {
336 self.run_binding == dsl::RuntimeLoopRunBinding::UsePrebound
337 }
338}
339
340pub fn classify_runtime_lifecycle_state(
344 state: RuntimeState,
345) -> Result<RuntimeLifecycleFacts, String> {
346 let observed_state = dsl_authority::observed_runtime_lifecycle_state(state);
347 let mut authority = projection_authority();
348 let transition = dsl::MeerkatMachineMutator::apply(
349 &mut authority,
350 dsl::MeerkatMachineInput::ClassifyRuntimeLifecycleState {
351 state: observed_state,
352 },
353 )
354 .map_err(|err| {
355 format!("MeerkatMachine rejected runtime lifecycle classification for {state}: {err}")
356 })?;
357
358 transition
359 .into_effects()
360 .into_iter()
361 .find_map(|effect| match effect {
362 dsl::MeerkatMachineEffect::RuntimeLifecycleStateClassified {
363 state,
364 terminality,
365 input_admission,
366 queue_admission,
367 prepare_admission,
368 ingress_admission,
369 } if state == observed_state => Some(RuntimeLifecycleFacts {
370 terminality,
371 input_admission,
372 queue_admission,
373 prepare_admission,
374 ingress_admission,
375 }),
376 _ => None,
377 })
378 .ok_or_else(|| {
379 format!("MeerkatMachine emitted no runtime lifecycle classification for {state}")
380 })
381}
382
383pub fn classify_runtime_lifecycle_durable_state(
387 state: RuntimeState,
388) -> Result<RuntimeState, String> {
389 classify_runtime_lifecycle_durable_state_with_pre_run_phase(state, None)
390}
391
392pub(crate) fn classify_runtime_lifecycle_durable_state_with_pre_run_phase(
396 state: RuntimeState,
397 pre_run_phase: Option<RuntimeState>,
398) -> Result<RuntimeState, String> {
399 let observed_state = dsl_authority::observed_runtime_lifecycle_state(state);
400 let pre_run_phase = pre_run_phase.and_then(dsl_authority::pre_run_phase_from_runtime_state);
401 let mut authority = projection_authority();
402 let transition = dsl::MeerkatMachineMutator::apply(
403 &mut authority,
404 dsl::MeerkatMachineInput::ClassifyRuntimeLifecycleDurability {
405 state: observed_state,
406 pre_run_phase,
407 },
408 )
409 .map_err(|err| {
410 format!(
411 "MeerkatMachine rejected runtime lifecycle durability classification for {state}: {err}"
412 )
413 })?;
414
415 transition
416 .into_effects()
417 .into_iter()
418 .find_map(|effect| match effect {
419 dsl::MeerkatMachineEffect::RuntimeLifecycleDurabilityClassified {
420 state,
421 durable_state,
422 } if state == observed_state => Some(
423 dsl_authority::runtime_state_from_observed_lifecycle_state(durable_state),
424 ),
425 _ => None,
426 })
427 .ok_or_else(|| {
428 format!(
429 "MeerkatMachine emitted no runtime lifecycle durability classification for {state}"
430 )
431 })
432}
433
434pub fn classify_runtime_loop_queue_admission(
439 state: RuntimeState,
440 current_run_bound: bool,
441) -> Result<RuntimeLoopQueueAdmissionPlan, String> {
442 let observed_state = dsl_authority::observed_runtime_lifecycle_state(state);
443 let mut authority = projection_authority();
444 let transition = dsl::MeerkatMachineMutator::apply(
445 &mut authority,
446 dsl::MeerkatMachineInput::ClassifyRuntimeLoopQueueAdmission {
447 state: observed_state,
448 current_run_bound,
449 },
450 )
451 .map_err(|err| {
452 format!(
453 "MeerkatMachine rejected runtime-loop queue admission for {state} with current_run_bound={current_run_bound}: {err}"
454 )
455 })?;
456
457 transition
458 .into_effects()
459 .into_iter()
460 .find_map(|effect| match effect {
461 dsl::MeerkatMachineEffect::RuntimeLoopQueueAdmissionClassified {
462 state,
463 current_run_bound: observed_current_run_bound,
464 queue_admission,
465 run_binding,
466 } if state == observed_state && observed_current_run_bound == current_run_bound => {
467 Some(RuntimeLoopQueueAdmissionPlan {
468 queue_admission,
469 run_binding,
470 })
471 }
472 _ => None,
473 })
474 .ok_or_else(|| {
475 format!(
476 "MeerkatMachine emitted no runtime-loop queue admission for {state} with current_run_bound={current_run_bound}"
477 )
478 })
479}
480
481#[derive(Debug, Clone, Copy, PartialEq, Eq)]
489pub struct VisibleRuntimePhasePlan {
490 pub publish_control: bool,
491 pub selected_raw_phase: RuntimeState,
492 pub visible_phase: RuntimeState,
493}
494
495pub fn resolve_visible_runtime_phase(
503 dsl_phase: RuntimeState,
504 dsl_pre_run_phase: Option<RuntimeState>,
505 control_phase: RuntimeState,
506 control_pre_run_phase: Option<RuntimeState>,
507 has_runtime_persistence: bool,
508) -> Result<VisibleRuntimePhasePlan, String> {
509 let observed_dsl = dsl_authority::observed_runtime_lifecycle_state(dsl_phase);
510 let observed_control = dsl_authority::observed_runtime_lifecycle_state(control_phase);
511 let observed_dsl_pre_run =
512 dsl_pre_run_phase.map(dsl_authority::observed_runtime_lifecycle_state);
513 let observed_control_pre_run =
514 control_pre_run_phase.map(dsl_authority::observed_runtime_lifecycle_state);
515 let mut authority = projection_authority();
516 let transition = dsl::MeerkatMachineMutator::apply(
517 &mut authority,
518 dsl::MeerkatMachineInput::ResolveVisibleRuntimePhase {
519 dsl_phase: observed_dsl,
520 dsl_pre_run_phase: observed_dsl_pre_run,
521 control_phase: observed_control,
522 control_pre_run_phase: observed_control_pre_run,
523 has_runtime_persistence,
524 },
525 )
526 .map_err(|err| {
527 format!(
528 "MeerkatMachine rejected visible runtime phase resolution \
529 (dsl={dsl_phase}, control={control_phase}, persistence={has_runtime_persistence}): {err}"
530 )
531 })?;
532
533 transition
534 .into_effects()
535 .into_iter()
536 .find_map(|effect| match effect {
537 dsl::MeerkatMachineEffect::VisibleRuntimePhaseResolved {
538 publish_control,
539 selected_raw_phase,
540 visible_phase,
541 } => Some(VisibleRuntimePhasePlan {
542 publish_control,
543 selected_raw_phase: dsl_authority::runtime_state_from_observed_lifecycle_state(
544 selected_raw_phase,
545 ),
546 visible_phase: dsl_authority::runtime_state_from_observed_lifecycle_state(
547 visible_phase,
548 ),
549 }),
550 _ => None,
551 })
552 .ok_or_else(|| {
553 format!(
554 "MeerkatMachine emitted no visible runtime phase resolution \
555 (dsl={dsl_phase}, control={control_phase}, persistence={has_runtime_persistence})"
556 )
557 })
558}
559
560pub fn resolve_input_public_lifecycle_projection(
563 input_id: &InputId,
564 phase: InputLifecycleState,
565) -> Result<dsl::InputPublicLifecycleState, String> {
566 let input_key = input_id.to_string();
567 let mut authority = projection_authority();
568 let transition = dsl::MeerkatMachineMutator::apply(
569 &mut authority,
570 dsl::MeerkatMachineInput::ResolveInputPublicLifecycle {
571 input_id: input_key.clone(),
572 phase: observed_input_phase(phase),
573 },
574 )
575 .map_err(|err| {
576 format!("MeerkatMachine rejected public lifecycle projection for '{input_id}': {err}")
577 })?;
578
579 transition
580 .into_effects()
581 .into_iter()
582 .find_map(|effect| match effect {
583 dsl::MeerkatMachineEffect::InputPublicLifecycleResolved { input_id, phase }
584 if input_id == input_key =>
585 {
586 Some(phase)
587 }
588 _ => None,
589 })
590 .ok_or_else(|| {
591 format!("MeerkatMachine emitted no public lifecycle projection for '{input_id}'")
592 })
593}
594
595pub fn resolve_input_public_state_projection(
598 input_id: &InputId,
599 seed: &InputStateSeed,
600) -> Result<InputPublicStateProjection, String> {
601 let lifecycle_state = resolve_input_public_lifecycle_projection(input_id, seed.phase)?;
602 let terminal_outcome = resolve_input_public_terminal_projection(input_id, seed)?;
603 Ok(InputPublicStateProjection {
604 lifecycle_state,
605 terminal_outcome,
606 })
607}
608
609pub(crate) fn input_seed_behavioral_terminality_via_authority(
610 input_id: &InputId,
611 seed: &InputStateSeed,
612) -> Result<bool, String> {
613 classify_input_behavioral_terminality(input_id, seed.phase, seed.terminal_outcome.as_ref())
614}
615
616pub(crate) fn input_phase_behavioral_terminality_via_authority(
617 input_id: &InputId,
618 phase: InputLifecycleState,
619 terminal_outcome: Option<InputTerminalOutcome>,
620) -> Result<bool, String> {
621 classify_input_behavioral_terminality(input_id, phase, terminal_outcome.as_ref())
622}
623
624pub(crate) fn authorize_stored_input_state_seed(
627 input_id: &InputId,
628 seed: &InputStateSeed,
629) -> Result<(), String> {
630 let input_key = input_id.to_string();
631 let (terminal_kind, superseded_by, aggregate_id, abandon_reason, abandon_attempt_count) =
632 input_seed_terminal_parts(seed)?;
633 let mut authority = projection_authority();
634 let transition = dsl::MeerkatMachineMutator::apply(
635 &mut authority,
636 dsl::MeerkatMachineInput::AuthorizeStoredInputStateSeed {
637 input_id: input_key.clone(),
638 phase: observed_input_phase(seed.phase),
639 terminal_kind,
640 superseded_by,
641 aggregate_id,
642 abandon_reason,
643 abandon_attempt_count,
644 attempt_count: u64::from(seed.attempt_count),
645 run_id: seed.last_run_id.as_ref().map(dsl::RunId::from_domain),
646 boundary_sequence: seed.last_boundary_sequence,
647 admission_sequence: seed.admission_sequence,
648 recovery_lane: seed.recovery_lane.map(dsl::InputLane::from),
649 },
650 )
651 .map_err(|err| {
652 format!("MeerkatMachine rejected stored input-state seed for '{input_id}': {err}")
653 })?;
654
655 transition
656 .into_effects()
657 .into_iter()
658 .find_map(|effect| match effect {
659 dsl::MeerkatMachineEffect::StoredInputStateSeedAuthorized { input_id }
660 if input_id == input_key =>
661 {
662 Some(())
663 }
664 _ => None,
665 })
666 .ok_or_else(|| {
667 format!("MeerkatMachine emitted no stored input-state seed authority for '{input_id}'")
668 })
669}
670
671fn classify_input_behavioral_terminality(
672 input_id: &InputId,
673 phase: InputLifecycleState,
674 terminal_outcome: Option<&InputTerminalOutcome>,
675) -> Result<bool, String> {
676 let input_key = input_id.to_string();
677 let (terminal_kind, abandon_reason) = input_terminality_parts(terminal_outcome);
678 let mut authority = projection_authority();
679 let transition = dsl::MeerkatMachineMutator::apply(
680 &mut authority,
681 dsl::MeerkatMachineInput::ClassifyInputTerminality {
682 input_id: input_key.clone(),
683 phase: observed_input_phase(phase),
684 terminal_kind,
685 abandon_reason,
686 },
687 )
688 .map_err(|err| {
689 format!("MeerkatMachine rejected behavioral input terminality for '{input_id}': {err}")
690 })?;
691
692 let mut terminality = None;
693 for effect in transition.into_effects() {
694 match effect {
695 dsl::MeerkatMachineEffect::InputBehavioralTerminalityResolved {
696 input_id,
697 terminal,
698 } if input_id == input_key => terminality = Some(terminal),
699 other => {
700 return Err(format!(
701 "MeerkatMachine emitted unexpected behavioral input terminality effect for '{input_id}': {other:?}"
702 ));
703 }
704 }
705 }
706 terminality.ok_or_else(|| {
707 format!("MeerkatMachine emitted no behavioral input terminality for '{input_id}'")
708 })
709}
710
711fn resolve_input_public_terminal_projection(
712 input_id: &InputId,
713 seed: &InputStateSeed,
714) -> Result<Option<dsl::InputPublicTerminalOutcome>, String> {
715 let input_key = input_id.to_string();
716 let (terminal_kind, abandon_reason) = input_terminality_parts(seed.terminal_outcome.as_ref());
717 let mut authority = projection_authority();
718 let transition = dsl::MeerkatMachineMutator::apply(
719 &mut authority,
720 dsl::MeerkatMachineInput::ResolveInputPublicTerminalOutcome {
721 input_id: input_key.clone(),
722 phase: observed_input_phase(seed.phase),
723 terminal_kind,
724 abandon_reason,
725 },
726 )
727 .map_err(|err| {
728 format!("MeerkatMachine rejected public terminal projection for '{input_id}': {err}")
729 })?;
730
731 transition
732 .into_effects()
733 .into_iter()
734 .find_map(|effect| match effect {
735 dsl::MeerkatMachineEffect::InputPublicTerminalOutcomeResolved {
736 input_id,
737 terminal_outcome,
738 } if input_id == input_key => Some(terminal_outcome),
739 _ => None,
740 })
741 .ok_or_else(|| {
742 format!("MeerkatMachine emitted no public terminal projection for '{input_id}'")
743 })
744}
745
746fn projection_authority() -> dsl::MeerkatMachineAuthority {
747 dsl_authority::new_initialized_authority("projection authority must initialize")
748}
749
750#[cfg(feature = "live")]
751fn live_unbound_rejection_authority() -> crate::driver::ephemeral::SharedIngressDslAuthority {
752 Arc::new(std::sync::Mutex::new(
753 dsl_authority::new_initialized_authority(
754 "live unbound rejection authority must initialize",
755 ),
756 ))
757}
758
759fn observed_input_phase(phase: InputLifecycleState) -> dsl::RecoveredInputObservedPhase {
760 match phase {
761 InputLifecycleState::Accepted => dsl::RecoveredInputObservedPhase::Accepted,
762 InputLifecycleState::Queued => dsl::RecoveredInputObservedPhase::Queued,
763 InputLifecycleState::Staged => dsl::RecoveredInputObservedPhase::Staged,
764 InputLifecycleState::Applied => dsl::RecoveredInputObservedPhase::Applied,
765 InputLifecycleState::AppliedPendingConsumption => {
766 dsl::RecoveredInputObservedPhase::AppliedPendingConsumption
767 }
768 InputLifecycleState::Consumed => dsl::RecoveredInputObservedPhase::Consumed,
769 InputLifecycleState::Superseded => dsl::RecoveredInputObservedPhase::Superseded,
770 InputLifecycleState::Coalesced => dsl::RecoveredInputObservedPhase::Coalesced,
771 InputLifecycleState::Abandoned => dsl::RecoveredInputObservedPhase::Abandoned,
772 }
773}
774
775type InputSeedTerminalParts = (
776 Option<dsl::InputTerminalKind>,
777 Option<String>,
778 Option<String>,
779 Option<dsl::InputAbandonReason>,
780 u64,
781);
782
783fn input_seed_terminal_parts(seed: &InputStateSeed) -> Result<InputSeedTerminalParts, String> {
784 match seed.terminal_outcome.as_ref() {
785 None => Ok((None, None, None, None, 0)),
786 Some(InputTerminalOutcome::Consumed) => {
787 Ok((Some(dsl::InputTerminalKind::Consumed), None, None, None, 0))
788 }
789 Some(InputTerminalOutcome::Superseded { superseded_by }) => Ok((
790 Some(dsl::InputTerminalKind::Superseded),
791 Some(superseded_by.to_string()),
792 None,
793 None,
794 0,
795 )),
796 Some(InputTerminalOutcome::Coalesced { aggregate_id }) => Ok((
797 Some(dsl::InputTerminalKind::Coalesced),
798 None,
799 Some(aggregate_id.to_string()),
800 None,
801 0,
802 )),
803 Some(InputTerminalOutcome::Abandoned { reason }) => {
804 let abandon_attempt_count = match reason {
805 InputAbandonReason::MaxAttemptsExhausted { attempts } => u64::from(*attempts),
806 _ => u64::from(seed.attempt_count),
807 };
808 Ok((
809 Some(dsl::InputTerminalKind::Abandoned),
810 None,
811 None,
812 input_terminality_parts(seed.terminal_outcome.as_ref()).1,
813 abandon_attempt_count,
814 ))
815 }
816 }
817}
818
819fn input_terminality_parts(
820 outcome: Option<&InputTerminalOutcome>,
821) -> (
822 Option<dsl::InputTerminalKind>,
823 Option<dsl::InputAbandonReason>,
824) {
825 match outcome {
826 None => (None, None),
827 Some(InputTerminalOutcome::Consumed) => (Some(dsl::InputTerminalKind::Consumed), None),
828 Some(InputTerminalOutcome::Superseded { .. }) => {
829 (Some(dsl::InputTerminalKind::Superseded), None)
830 }
831 Some(InputTerminalOutcome::Coalesced { .. }) => {
832 (Some(dsl::InputTerminalKind::Coalesced), None)
833 }
834 Some(InputTerminalOutcome::Abandoned { reason }) => (
835 Some(dsl::InputTerminalKind::Abandoned),
836 Some(match reason {
837 InputAbandonReason::Retired => dsl::InputAbandonReason::Retired,
838 InputAbandonReason::Reset => dsl::InputAbandonReason::Reset,
839 InputAbandonReason::Stopped => dsl::InputAbandonReason::Stopped,
840 InputAbandonReason::Destroyed => dsl::InputAbandonReason::Destroyed,
841 InputAbandonReason::Cancelled => dsl::InputAbandonReason::Cancelled,
842 InputAbandonReason::MaxAttemptsExhausted { .. } => {
843 dsl::InputAbandonReason::MaxAttemptsExhausted
844 }
845 }),
846 ),
847 }
848}
849
850#[derive(Debug, Default)]
851struct UnavailableBlobStore;
852
853impl UnavailableBlobStore {
854 fn error() -> BlobStoreError {
855 BlobStoreError::Unsupported(
856 "persistent runtime constructed without blob store; blob-backed inputs require a BlobStore"
857 .to_string(),
858 )
859 }
860}
861
862#[cfg(not(target_arch = "wasm32"))]
863struct PersistentAuthAuthorityBundle {
864 store: StdMutex<Weak<dyn RuntimeStore>>,
865 auth_lease: Arc<crate::handles::RuntimeAuthLeaseHandle>,
866 oauth_flows: Arc<crate::handles::RuntimeOAuthFlowHandle>,
867}
868
869#[cfg(not(target_arch = "wasm32"))]
870#[derive(Debug, Clone, PartialEq, Eq, Hash)]
871enum PersistentAuthAuthorityKey {
872 Durable(String),
873 Process(usize),
874}
875
876#[cfg(not(target_arch = "wasm32"))]
877static PERSISTENT_AUTH_AUTHORITIES: OnceLock<
878 StdMutex<HashMap<PersistentAuthAuthorityKey, Arc<PersistentAuthAuthorityBundle>>>,
879> = OnceLock::new();
880
881#[cfg(not(target_arch = "wasm32"))]
882fn runtime_store_identity(store: &Arc<dyn RuntimeStore>) -> PersistentAuthAuthorityKey {
883 store
884 .auth_authority_key()
885 .map(PersistentAuthAuthorityKey::Durable)
886 .unwrap_or_else(|| {
887 PersistentAuthAuthorityKey::Process(Arc::as_ptr(store).cast::<()>() as usize)
888 })
889}
890
891fn runtime_stores_share_authority(a: &Arc<dyn RuntimeStore>, b: &Arc<dyn RuntimeStore>) -> bool {
892 match (a.auth_authority_key(), b.auth_authority_key()) {
893 (Some(a), Some(b)) => a == b,
894 _ => Arc::ptr_eq(a, b),
895 }
896}
897
898fn generated_runtime_auth_lease_handle(
899 handle: Arc<crate::handles::RuntimeAuthLeaseHandle>,
900) -> meerkat_core::handles::GeneratedAuthLeaseHandle {
901 #[allow(clippy::expect_used)]
902 crate::protocol_auth_lease_lifecycle_publication::generated_auth_lease_handle(handle)
903 .expect("runtime AuthLeaseHandle must be certified by generated AuthMachine authority")
904}
905
906#[cfg(not(target_arch = "wasm32"))]
912#[derive(Clone)]
913pub struct ProviderAuthRuntimeAuthority {
914 auth_lease: meerkat_core::handles::GeneratedAuthLeaseHandle,
915 oauth_flows: Arc<dyn meerkat_auth_core::oauth_flow::OAuthFlowAuthority>,
916}
917
918#[cfg(not(target_arch = "wasm32"))]
919impl ProviderAuthRuntimeAuthority {
920 pub fn generated_auth_lease_handle(&self) -> meerkat_core::handles::GeneratedAuthLeaseHandle {
921 self.auth_lease.clone()
922 }
923
924 pub fn oauth_flow_authority(
925 &self,
926 ) -> Arc<dyn meerkat_auth_core::oauth_flow::OAuthFlowAuthority> {
927 Arc::clone(&self.oauth_flows)
928 }
929}
930
931#[cfg(not(target_arch = "wasm32"))]
932fn persistent_auth_authorities(
933 store: &Arc<dyn RuntimeStore>,
934) -> Arc<PersistentAuthAuthorityBundle> {
935 let key = runtime_store_identity(store);
936 let authorities = PERSISTENT_AUTH_AUTHORITIES.get_or_init(|| StdMutex::new(HashMap::new()));
937 let mut authorities = authorities
938 .lock()
939 .unwrap_or_else(std::sync::PoisonError::into_inner);
940 if let Some(existing) = authorities.get(&key) {
941 let stored_store_alive = existing
942 .store
943 .lock()
944 .unwrap_or_else(std::sync::PoisonError::into_inner)
945 .upgrade()
946 .is_some();
947 if matches!(key, PersistentAuthAuthorityKey::Durable(_)) || stored_store_alive {
948 existing.oauth_flows.bind_persistent_store(store);
949 *existing
950 .store
951 .lock()
952 .unwrap_or_else(std::sync::PoisonError::into_inner) = Arc::downgrade(store);
953 return Arc::clone(existing);
954 }
955 }
956 let auth_lease = Arc::new(crate::handles::RuntimeAuthLeaseHandle::new());
957 let oauth_flows = Arc::new(
958 crate::handles::RuntimeOAuthFlowHandle::new_with_persistent_store_and_auth_lease(
959 std::time::Duration::from_secs(10 * 60),
960 Arc::clone(&auth_lease),
961 store,
962 ),
963 );
964 let bundle = Arc::new(PersistentAuthAuthorityBundle {
965 store: StdMutex::new(Arc::downgrade(store)),
966 auth_lease,
967 oauth_flows,
968 });
969 authorities.insert(key, Arc::clone(&bundle));
970 bundle
971}
972
973#[cfg(all(test, not(target_arch = "wasm32")))]
974pub(crate) fn clear_persistent_auth_authorities_for_test() {
975 if let Some(authorities) = PERSISTENT_AUTH_AUTHORITIES.get() {
976 authorities
977 .lock()
978 .unwrap_or_else(std::sync::PoisonError::into_inner)
979 .clear();
980 }
981}
982
983#[cfg_attr(target_arch = "wasm32", async_trait::async_trait(?Send))]
984#[cfg_attr(not(target_arch = "wasm32"), async_trait::async_trait)]
985impl BlobStore for UnavailableBlobStore {
986 async fn put_image(&self, _media_type: &str, _data: &str) -> Result<BlobRef, BlobStoreError> {
987 Err(Self::error())
988 }
989
990 async fn get(&self, _blob_id: &BlobId) -> Result<BlobPayload, BlobStoreError> {
991 Err(Self::error())
992 }
993
994 async fn delete(&self, _blob_id: &BlobId) -> Result<(), BlobStoreError> {
995 Err(Self::error())
996 }
997
998 async fn exists(&self, _blob_id: &BlobId) -> Result<bool, BlobStoreError> {
999 Err(Self::error())
1000 }
1001
1002 fn is_persistent(&self) -> bool {
1003 false
1004 }
1005}
1006
1007#[cfg(not(target_arch = "wasm32"))]
1008type MeerkatMachineCommandFuture<'a> = Pin<
1009 Box<
1010 dyn Future<Output = Result<MeerkatMachineCommandResult, MeerkatMachineCommandError>>
1011 + Send
1012 + 'a,
1013 >,
1014>;
1015
1016#[cfg(target_arch = "wasm32")]
1017type MeerkatMachineCommandFuture<'a> = Pin<
1018 Box<dyn Future<Output = Result<MeerkatMachineCommandResult, MeerkatMachineCommandError>> + 'a>,
1019>;
1020
1021#[cfg(test)]
1022pub(crate) use driver::fail_machine_run;
1023pub(crate) use driver::{
1024 DriverEntry, SharedCompletionRegistry, SharedDriver, cancel_runtime_loop_run,
1025 commit_machine_terminal_run, commit_runtime_loop_run, fail_runtime_loop_run,
1026 machine_authorize_runtime_loop_batch, machine_batch_primitive_projections,
1027 machine_batch_runtime_semantics, machine_commit_prepared_destroy,
1028 machine_commit_service_turn_terminal_receipt, machine_prepare_bindings_projection,
1029 machine_prepare_destroy, machine_recover_ephemeral_driver,
1030 machine_recover_persistent_inputs_from_observed, machine_recycle_preserving_work,
1031 machine_reset, machine_retire, machine_stop_runtime, prepare_runtime_loop_batch_start,
1032};
1033pub(crate) mod driver;
1034
1035mod comms_drain;
1036pub mod composition;
1037mod dispatch_control;
1038mod dispatch_drain;
1039mod dispatch_ingress;
1040mod dispatch_session;
1041#[allow(unused_variables, dead_code, clippy::cmp_owned)]
1042#[allow(clippy::assign_op_pattern)]
1043pub mod dsl;
1044pub(crate) mod dsl_authority;
1045mod dsl_effects;
1046mod durability_health;
1047mod llm_reconfigure;
1048mod runtime_control;
1049mod session_management;
1050mod traits;
1051mod visibility;
1052
1053pub(crate) use durability_health::{DurabilityHealthHandle, DurabilityReloadRequired};
1054
1055pub(crate) use session_management::{
1056 DeleteOpsFinalizationAuthority, RetainOpsFinalizationAuthority,
1057};
1058
1059pub use composition::{MeerkatCompositionSignalDispatcher, MeerkatConsumerSurface};
1060
1061pub use comms_drain::{
1062 CommsDrainMode, CommsDrainPhase, DrainExitReason, PeerEndpointStageError, PeerIngressOwner,
1063 SupervisorBinding, SupervisorBindingStageError,
1064};
1065pub(crate) use comms_drain::{
1066 CommsDrainSlot, GeneratedSupervisorBinding, GeneratedSupervisorRotationReceipt,
1067 GeneratedSupervisorRotationSubmit, SupervisorAuthorizeAdmission, SupervisorBindAdmission,
1068 SupervisorBridgeCommandAdmission, SupervisorRotationObservation, SupervisorRotationSubmission,
1069 SupervisorRotationTaskSlot,
1070};
1071pub(crate) use dsl_effects::{DslTransitionEffects, apply_dsl_transition_on_authority};
1072pub(crate) use visibility::MachineToolVisibilityOwner;
1073
1074struct StagedSessionDslInput {
1075 previous_snapshot: dsl::MeerkatMachineAuthoritySnapshot,
1076 committed_snapshot: dsl::MeerkatMachineAuthoritySnapshot,
1077 effects: DslTransitionEffects,
1078}
1079
1080impl StagedSessionDslInput {
1081 fn revived_stopped_session(&self) -> bool {
1088 self.effects.as_slice().iter().any(|effect| {
1089 matches!(
1090 effect,
1091 dsl::MeerkatMachineEffect::RuntimeNotice {
1092 kind: dsl::RuntimeNoticeKind::Recover,
1093 ..
1094 }
1095 )
1096 })
1097 }
1098
1099 fn has_routed_signal_effect(&self) -> bool {
1104 self.effects
1105 .as_slice()
1106 .iter()
1107 .any(|effect| composition::lift_routed_signal(effect).is_some())
1108 }
1109}
1110
1111#[derive(Clone, Copy)]
1112enum CommittedEffectDispatchFailure {
1113 PreserveCommittedDslState,
1114}
1115
1116type UnregisterTeardownResult = Result<(), RuntimeDriverError>;
1117type RuntimeStopCleanupResult = Result<(), RuntimeDriverError>;
1118
1119#[derive(Clone)]
1121struct UnregisterTeardownCoordinator {
1122 epoch_id: meerkat_core::RuntimeEpochId,
1123 coordinator_id: uuid::Uuid,
1124 result_rx: crate::tokio::sync::watch::Receiver<Option<UnregisterTeardownResult>>,
1125}
1126
1127#[derive(Clone)]
1131struct RuntimeStopCleanupCoordinator {
1132 epoch_id: meerkat_core::RuntimeEpochId,
1133 coordinator_id: uuid::Uuid,
1134 teardown_slot: Option<Arc<crate::runtime_loop::RuntimeLoopTeardownSlot>>,
1135 result_rx: crate::tokio::sync::watch::Receiver<Option<RuntimeStopCleanupResult>>,
1136}
1137
1138#[derive(Clone)]
1139struct PendingUnregisterFinalization {
1140 finalization_id: uuid::Uuid,
1141 durability_authority: session_management::RuntimeOpsLifecycleDurabilityAuthority,
1142 committed_snapshot: dsl::MeerkatMachineAuthoritySnapshot,
1143}
1144
1145struct UnregisterTeardownMechanicalObservations {
1146 runtime_loop_forced_abort: std::sync::atomic::AtomicBool,
1147 comms_drain_forced_abort: std::sync::atomic::AtomicBool,
1148}
1149
1150#[cfg(not(target_arch = "wasm32"))]
1154struct OpsLifecyclePersistenceWorker {
1155 handle: std::thread::JoinHandle<()>,
1156}
1157
1158#[cfg(target_arch = "wasm32")]
1159struct OpsLifecyclePersistenceWorker {
1160 handle: crate::tokio::task::JoinHandle<()>,
1161}
1162
1163impl UnregisterTeardownMechanicalObservations {
1164 fn new() -> Self {
1165 Self {
1166 runtime_loop_forced_abort: std::sync::atomic::AtomicBool::new(false),
1167 comms_drain_forced_abort: std::sync::atomic::AtomicBool::new(false),
1168 }
1169 }
1170
1171 fn from_durable_process_recovery(
1172 progress: Option<&crate::store::MachineUnregisterProgressSnapshot>,
1173 ) -> Self {
1174 let observations = Self::new();
1175 if let Some(progress) = progress {
1176 observations.runtime_loop_forced_abort.store(
1180 progress.runtime_loop_drain_pending(),
1181 std::sync::atomic::Ordering::Release,
1182 );
1183 observations.comms_drain_forced_abort.store(
1184 progress.comms_drain_exit_pending(),
1185 std::sync::atomic::Ordering::Release,
1186 );
1187 }
1188 observations
1189 }
1190}
1191
1192struct RuntimeSessionEntry {
1194 runtime_id: LogicalRuntimeId,
1196 mutation_gate: Arc<Mutex<()>>,
1208 durability_health: Option<DurabilityHealthHandle>,
1216 #[cfg(feature = "live")]
1221 live_lifecycle_gate: Arc<Mutex<()>>,
1222 supervisor_rotation_task: Arc<SupervisorRotationTaskSlot>,
1225 driver: SharedDriver,
1227 control_projection: Arc<StdRwLock<crate::driver::ephemeral::RuntimeControlProjection>>,
1233 ops_lifecycle: Arc<crate::ops_lifecycle::RuntimeOpsLifecycleRegistry>,
1235 ops_lifecycle_persistence_worker: Option<OpsLifecyclePersistenceWorker>,
1239 epoch_id: meerkat_core::RuntimeEpochId,
1241 handle_teardown_gate: Arc<crate::handles::HandleTeardownGate>,
1246 materialization_claim_state:
1251 Arc<std::sync::Mutex<crate::RuntimeActorMaterializationClaimState>>,
1252 cursor_state: Arc<meerkat_core::EpochCursorState>,
1254 completions: SharedCompletionRegistry,
1256 tool_visibility_owner: Arc<MachineToolVisibilityOwner>,
1258 canonical_runtime_bindings: Option<meerkat_core::SessionRuntimeBindings>,
1265 attachment_slot: RuntimeLoopAttachmentSlot,
1270 runtime_loop_teardown: Option<Arc<crate::runtime_loop::RuntimeLoopTeardownSlot>>,
1277 unregister_coordinator: Option<UnregisterTeardownCoordinator>,
1279 runtime_stop_cleanup_coordinator: Option<RuntimeStopCleanupCoordinator>,
1282 pending_revival_lifecycle_persist: Arc<std::sync::atomic::AtomicBool>,
1286 pending_unregister_finalization: Option<PendingUnregisterFinalization>,
1291 unregister_teardown_observations: Arc<UnregisterTeardownMechanicalObservations>,
1294 publication_handle: Option<Arc<dyn meerkat_core::lifecycle::CoreExecutorPublicationHandle>>,
1300 post_stop_cleanup_handle:
1304 Option<Arc<dyn meerkat_core::lifecycle::CoreExecutorPostStopCleanupHandle>>,
1305 post_stop_cleanup_attachment_id: Option<RuntimeLoopAttachmentId>,
1307 post_stop_cleanup_complete: bool,
1309 post_stop_cleanup_gate: Arc<Mutex<()>>,
1315 provisional_interrupt_handle:
1318 Option<Arc<dyn meerkat_core::lifecycle::CoreExecutorInterruptHandle>>,
1319 provisional_materialization_claim_id: Option<uuid::Uuid>,
1323 dsl_authority: Arc<std::sync::Mutex<dsl::MeerkatMachineAuthority>>,
1333 drain_slot: CommsDrainSlot,
1343}
1344
1345#[derive(Clone)]
1346struct MemberIncarnationRegistration {
1347 incarnation: meerkat_contracts::wire::supervisor_bridge::BridgeMemberIncarnation,
1348 session_mutation_gate: Arc<Mutex<()>>,
1349 tracked_turn_journal: Option<Arc<dyn crate::member_observation::TrackedTurnJournal>>,
1350}
1351
1352enum MemberResidencyState {
1353 PeerOnly,
1355 VacantPlaced,
1358 Placed(MemberIncarnationRegistration),
1360}
1361
1362struct MemberResidencySlot {
1363 gate: Arc<Mutex<()>>,
1364 state: StdRwLock<MemberResidencyState>,
1365}
1366
1367struct MemberEffectAuthorityLease {
1368 slot: Arc<MemberResidencySlot>,
1369 slot_guard: crate::tokio::sync::OwnedMutexGuard<()>,
1370 session_mutation_gate: Arc<Mutex<()>>,
1371}
1372
1373pub(crate) struct MemberEffectAuthorityGuard {
1378 _slot_guard: crate::tokio::sync::OwnedMutexGuard<()>,
1379 _session_guard: crate::tokio::sync::OwnedMutexGuard<()>,
1380}
1381
1382pub(crate) struct MemberLiveAuthorityGuard {
1391 _slot_guard: crate::tokio::sync::OwnedMutexGuard<()>,
1392 _registration_guard: crate::tokio::sync::OwnedMutexGuard<()>,
1393}
1394
1395impl MemberResidencySlot {
1396 fn peer_only() -> Self {
1397 Self {
1398 gate: Arc::new(Mutex::new(())),
1399 state: StdRwLock::new(MemberResidencyState::PeerOnly),
1400 }
1401 }
1402}
1403
1404pub struct MachineSessionArchiveLease {
1409 session_id: SessionId,
1410 runtime_id: LogicalRuntimeId,
1411 driver: SharedDriver,
1412 completions: SharedCompletionRegistry,
1413 wake_tx: Option<mpsc::Sender<()>>,
1414 publication_handle: Option<Arc<dyn meerkat_core::lifecycle::CoreExecutorPublicationHandle>>,
1415 recovered_registration_for_archive: bool,
1421 _registration_transaction_guard: crate::tokio::sync::OwnedMutexGuard<()>,
1425 _live_lifecycle_lease: Option<crate::member_live::MemberLiveLifecycleLease>,
1428 _mutation_guard: crate::tokio::sync::OwnedMutexGuard<()>,
1429}
1430
1431#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
1434pub enum LocalSessionMaterializationMode {
1435 #[default]
1438 Ordinary,
1439 MissingLiveRevival,
1442}
1443
1444pub struct PreparedSessionMaterialization {
1451 machine: Arc<MeerkatMachine>,
1452 bindings: meerkat_core::SessionRuntimeBindings,
1453 claim_id: uuid::Uuid,
1454 claim_state: Arc<std::sync::Mutex<crate::RuntimeActorMaterializationClaimState>>,
1455 cleanup_spawner: MachineCleanupTaskSpawner,
1456 archived_resume_authorization_issued: bool,
1457 armed: bool,
1458}
1459
1460pub struct PreparedArchivedResumeCommitLease {
1468 machine: Arc<MeerkatMachine>,
1469 session_id: SessionId,
1470 epoch_id: meerkat_core::RuntimeEpochId,
1471 claim_id: uuid::Uuid,
1472 claim_state: Arc<std::sync::Mutex<crate::RuntimeActorMaterializationClaimState>>,
1473 _mutation_guard: crate::tokio::sync::OwnedMutexGuard<()>,
1474}
1475
1476pub struct AuthorizedArchivedResumeCommitLease {
1481 prepared: PreparedArchivedResumeCommitLease,
1482}
1483
1484impl PreparedArchivedResumeCommitLease {
1485 pub fn session_id(&self) -> &SessionId {
1486 &self.session_id
1487 }
1488
1489 pub fn confirm_runtime_store_authority(
1493 self,
1494 runtime_store: &Arc<dyn crate::RuntimeStore>,
1495 ) -> Result<AuthorizedArchivedResumeCommitLease, RuntimeDriverError> {
1496 if !self.machine.shares_runtime_store_authority(runtime_store) {
1497 return Err(RuntimeDriverError::StaleAuthority {
1498 reason: format!(
1499 "archived-resume authorization for session {} used a runtime store not owned by the prepared machine",
1500 self.session_id
1501 ),
1502 });
1503 }
1504 Ok(AuthorizedArchivedResumeCommitLease { prepared: self })
1505 }
1506}
1507
1508impl AuthorizedArchivedResumeCommitLease {
1509 pub fn session_id(&self) -> &SessionId {
1510 self.prepared.session_id()
1511 }
1512
1513 pub async fn reset_retired_runtime(&mut self) -> Result<ResetReport, RuntimeDriverError> {
1515 self.prepared
1516 .machine
1517 .reset_runtime_for_authorized_archived_resume(self)
1518 .await
1519 }
1520}
1521
1522#[must_use = "archived-resume authorization must be consumed by actor materialization"]
1531pub struct ArchivedSessionActorMaterializationAuthorization {
1532 machine: Arc<MeerkatMachine>,
1533 session_id: SessionId,
1534 epoch_id: meerkat_core::RuntimeEpochId,
1535 claim_id: uuid::Uuid,
1536 claim_state: Arc<std::sync::Mutex<crate::RuntimeActorMaterializationClaimState>>,
1537 dsl_authority: Arc<std::sync::Mutex<dsl::MeerkatMachineAuthority>>,
1538 teardown_gate: Arc<crate::handles::HandleTeardownGate>,
1539}
1540
1541impl ArchivedSessionActorMaterializationAuthorization {
1542 pub(crate) async fn begin(
1543 self,
1544 bindings: &meerkat_core::SessionRuntimeBindings,
1545 ) -> Result<crate::RuntimeActorMaterializationPermit, crate::RuntimeActorMaterializationError>
1546 {
1547 let binding_authority = crate::validated_session_runtime_bindings_authority(bindings)?;
1548 if bindings.session_id() != &self.session_id
1549 || bindings.epoch_id() != &self.epoch_id
1550 || binding_authority.materialization_claim_id != Some(self.claim_id)
1551 || !Arc::ptr_eq(
1552 &binding_authority.materialization_claim_state,
1553 &self.claim_state,
1554 )
1555 || !Arc::ptr_eq(&binding_authority.dsl_authority, &self.dsl_authority)
1556 || !Arc::ptr_eq(&binding_authority.teardown_gate, &self.teardown_gate)
1557 {
1558 return Err(crate::RuntimeActorMaterializationError::InvalidAuthority(
1559 "archived-resume authorization does not match the exact prepared binding"
1560 .to_string(),
1561 ));
1562 }
1563
1564 let mutation_guard = self
1565 .machine
1566 .lock_current_durability_ready_session_mutation_gate(&self.session_id)
1567 .await
1568 .map_err(|_| crate::RuntimeActorMaterializationError::RegistrationClosed)?;
1569 {
1570 let sessions = self.machine.sessions.read().await;
1571 let entry = sessions
1572 .get(&self.session_id)
1573 .ok_or(crate::RuntimeActorMaterializationError::RegistrationClosed)?;
1574 let exact_registration = entry.epoch_id == self.epoch_id
1575 && Arc::ptr_eq(&entry.materialization_claim_state, &self.claim_state)
1576 && Arc::ptr_eq(&entry.dsl_authority, &self.dsl_authority)
1577 && Arc::ptr_eq(&entry.handle_teardown_gate, &self.teardown_gate)
1578 && entry.provisional_materialization_claim_id == Some(self.claim_id)
1579 && !entry.physical_attachment_is_live();
1580 let exact_claim = self
1581 .claim_state
1582 .lock()
1583 .unwrap_or_else(std::sync::PoisonError::into_inner)
1584 .exact_claim_is(
1585 self.claim_id,
1586 &[
1587 crate::RuntimeActorMaterializationClaimPhase::Prepared,
1588 crate::RuntimeActorMaterializationClaimPhase::Staged,
1589 ],
1590 );
1591 if !exact_registration || !exact_claim {
1592 return Err(crate::RuntimeActorMaterializationError::RegistrationClosed);
1593 }
1594 }
1595
1596 crate::begin_session_runtime_actor_materialization_with_phase_policy(
1597 bindings,
1598 crate::RuntimeActorMaterializationPhasePolicy::RequireArchivedRevivalBoundary,
1599 Some(mutation_guard),
1600 )
1601 }
1602}
1603
1604pub struct PreparedAttachedSessionActorRecovery {
1612 bindings: meerkat_core::SessionRuntimeBindings,
1613 witness: RuntimeExecutorAttachmentWitness,
1614 claim_id: uuid::Uuid,
1615 claim_state: Arc<std::sync::Mutex<crate::RuntimeActorMaterializationClaimState>>,
1616 previous_phase: crate::RuntimeActorMaterializationClaimPhase,
1617 mutation_guard: Option<crate::tokio::sync::OwnedMutexGuard<()>>,
1618 armed: bool,
1619}
1620
1621#[derive(Clone)]
1626struct RuntimeExecutorAttachmentMaterializationClaim {
1627 claim_id: uuid::Uuid,
1628 claim_state: Arc<std::sync::Mutex<crate::RuntimeActorMaterializationClaimState>>,
1629 epoch_id: meerkat_core::RuntimeEpochId,
1630}
1631
1632#[derive(Clone)]
1643pub struct RuntimeSessionRegistrationWitness {
1644 machine: std::sync::Weak<MeerkatMachineShared>,
1645 session_id: SessionId,
1646 epoch_id: meerkat_core::RuntimeEpochId,
1647 registration_gate: std::sync::Weak<Mutex<()>>,
1648}
1649
1650impl RuntimeSessionRegistrationWitness {
1651 fn new(
1652 machine: std::sync::Weak<MeerkatMachineShared>,
1653 session_id: SessionId,
1654 epoch_id: meerkat_core::RuntimeEpochId,
1655 registration_gate: std::sync::Weak<Mutex<()>>,
1656 ) -> Self {
1657 Self {
1658 machine,
1659 session_id,
1660 epoch_id,
1661 registration_gate,
1662 }
1663 }
1664
1665 pub fn session_id(&self) -> &SessionId {
1666 &self.session_id
1667 }
1668
1669 pub fn epoch_id(&self) -> &meerkat_core::RuntimeEpochId {
1670 &self.epoch_id
1671 }
1672
1673 fn belongs_to(&self, machine: &MeerkatMachine) -> bool {
1674 std::ptr::eq(self.machine.as_ptr(), Arc::as_ptr(&machine.shared))
1675 }
1676
1677 fn matches_entry(&self, entry: &RuntimeSessionEntry) -> bool {
1678 self.registration_gate
1679 .upgrade()
1680 .is_some_and(|gate| Arc::ptr_eq(&gate, &entry.mutation_gate))
1681 }
1682
1683 fn registration_gate(&self) -> Option<Arc<Mutex<()>>> {
1684 self.registration_gate.upgrade()
1685 }
1686}
1687
1688impl std::fmt::Debug for RuntimeSessionRegistrationWitness {
1689 fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
1690 formatter
1691 .debug_struct("RuntimeSessionRegistrationWitness")
1692 .field("session_id", &self.session_id)
1693 .field("epoch_id", &self.epoch_id)
1694 .finish_non_exhaustive()
1695 }
1696}
1697
1698impl PartialEq for RuntimeSessionRegistrationWitness {
1699 fn eq(&self, other: &Self) -> bool {
1700 self.session_id == other.session_id
1701 && self.epoch_id == other.epoch_id
1702 && std::sync::Weak::ptr_eq(&self.machine, &other.machine)
1703 && std::sync::Weak::ptr_eq(&self.registration_gate, &other.registration_gate)
1704 }
1705}
1706
1707impl Eq for RuntimeSessionRegistrationWitness {}
1708
1709#[derive(Debug, Clone, PartialEq, Eq)]
1714pub struct RuntimeSessionLifecycleObservation {
1715 session_id: SessionId,
1716 runtime_id: LogicalRuntimeId,
1717 lifecycle: crate::store::MachineLifecycleObservation,
1718}
1719
1720impl RuntimeSessionLifecycleObservation {
1721 fn new(
1722 session_id: SessionId,
1723 runtime_id: LogicalRuntimeId,
1724 lifecycle: crate::store::MachineLifecycleObservation,
1725 ) -> Self {
1726 Self {
1727 session_id,
1728 runtime_id,
1729 lifecycle,
1730 }
1731 }
1732
1733 #[must_use]
1734 pub fn session_id(&self) -> &SessionId {
1735 &self.session_id
1736 }
1737
1738 #[must_use]
1739 pub fn runtime_id(&self) -> &LogicalRuntimeId {
1740 &self.runtime_id
1741 }
1742
1743 #[must_use]
1744 pub fn lifecycle(&self) -> &crate::store::MachineLifecycleObservation {
1745 &self.lifecycle
1746 }
1747}
1748
1749#[derive(Debug, Clone, PartialEq, Eq)]
1755pub enum RuntimeSessionRegistrationOutcome {
1756 Applied {
1759 registration: RuntimeSessionRegistrationWitness,
1760 lifecycle: RuntimeSessionLifecycleObservation,
1761 },
1762 AlreadyExact {
1766 registration: RuntimeSessionRegistrationWitness,
1767 lifecycle: RuntimeSessionLifecycleObservation,
1768 },
1769 Conflict {
1772 current: RuntimeSessionLifecycleObservation,
1773 reason: String,
1774 },
1775 RepairBlocked {
1777 evidence_digest: Option<String>,
1778 reason: String,
1779 },
1780 Backoff { reason: String },
1783}
1784
1785#[derive(Clone)]
1793pub struct RuntimeExecutorAttachmentWitness {
1794 machine: std::sync::Weak<MeerkatMachineShared>,
1795 session_id: SessionId,
1796 epoch_id: meerkat_core::RuntimeEpochId,
1797 attachment_id: RuntimeLoopAttachmentId,
1798}
1799
1800impl RuntimeExecutorAttachmentWitness {
1801 fn new(
1802 machine: std::sync::Weak<MeerkatMachineShared>,
1803 session_id: SessionId,
1804 epoch_id: meerkat_core::RuntimeEpochId,
1805 attachment_id: RuntimeLoopAttachmentId,
1806 ) -> Self {
1807 Self {
1808 machine,
1809 session_id,
1810 epoch_id,
1811 attachment_id,
1812 }
1813 }
1814
1815 pub fn session_id(&self) -> &SessionId {
1816 &self.session_id
1817 }
1818
1819 pub fn epoch_id(&self) -> &meerkat_core::RuntimeEpochId {
1820 &self.epoch_id
1821 }
1822
1823 fn belongs_to(&self, machine: &MeerkatMachine) -> bool {
1824 std::ptr::eq(self.machine.as_ptr(), Arc::as_ptr(&machine.shared))
1825 }
1826}
1827
1828impl std::fmt::Debug for RuntimeExecutorAttachmentWitness {
1829 fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
1830 formatter
1831 .debug_struct("RuntimeExecutorAttachmentWitness")
1832 .field("session_id", &self.session_id)
1833 .field("epoch_id", &self.epoch_id)
1834 .finish_non_exhaustive()
1835 }
1836}
1837
1838impl PartialEq for RuntimeExecutorAttachmentWitness {
1839 fn eq(&self, other: &Self) -> bool {
1840 self.session_id == other.session_id
1841 && self.epoch_id == other.epoch_id
1842 && self.attachment_id == other.attachment_id
1843 && std::sync::Weak::ptr_eq(&self.machine, &other.machine)
1844 }
1845}
1846
1847impl Eq for RuntimeExecutorAttachmentWitness {}
1848
1849pub enum EnsureRuntimeExecutorAttachment {
1856 Existing(RuntimeExecutorAttachmentWitness),
1857 Pending(PendingRuntimeExecutorAttachment),
1858}
1859
1860#[cfg(not(target_arch = "wasm32"))]
1872static MACHINE_CLEANUP_RUNTIME: OnceLock<StdMutex<Option<crate::tokio::runtime::Runtime>>> =
1873 OnceLock::new();
1874
1875#[cfg(not(target_arch = "wasm32"))]
1880const MACHINE_CLEANUP_THREAD_STACK_SIZE: usize = 16 * 1024 * 1024;
1881
1882#[derive(Clone)]
1889struct MachineCleanupTaskSpawner {
1890 #[cfg(not(target_arch = "wasm32"))]
1891 handle: crate::tokio::runtime::Handle,
1892}
1893
1894#[doc(hidden)]
1897#[derive(Clone)]
1898pub struct RuntimeCleanupTaskSpawner {
1899 inner: MachineCleanupTaskSpawner,
1900}
1901
1902impl RuntimeCleanupTaskSpawner {
1903 pub fn acquire() -> Result<Self, RuntimeDriverError> {
1904 Ok(Self {
1905 inner: MachineCleanupTaskSpawner::acquire()?,
1906 })
1907 }
1908
1909 #[cfg(not(target_arch = "wasm32"))]
1910 pub fn spawn_detached<F>(&self, future: F)
1911 where
1912 F: Future<Output = ()> + Send + 'static,
1913 {
1914 self.inner.spawn(future);
1915 }
1916
1917 #[cfg(target_arch = "wasm32")]
1918 pub fn spawn_detached<F>(&self, future: F)
1919 where
1920 F: Future<Output = ()> + 'static,
1921 {
1922 self.inner.spawn(future);
1923 }
1924}
1925
1926impl MachineCleanupTaskSpawner {
1927 #[cfg(not(target_arch = "wasm32"))]
1928 fn acquire() -> Result<Self, RuntimeDriverError> {
1929 let slot = MACHINE_CLEANUP_RUNTIME.get_or_init(|| StdMutex::new(None));
1930 let mut runtime = slot
1931 .lock()
1932 .unwrap_or_else(std::sync::PoisonError::into_inner);
1933 if let Some(runtime) = runtime.as_ref() {
1934 return Ok(Self {
1935 handle: runtime.handle().clone(),
1936 });
1937 }
1938
1939 let candidate = crate::tokio::runtime::Builder::new_multi_thread()
1940 .worker_threads(1)
1941 .thread_name("meerkat-machine-cleanup")
1942 .thread_stack_size(MACHINE_CLEANUP_THREAD_STACK_SIZE)
1943 .enable_all()
1944 .build()
1945 .map_err(|error| {
1946 RuntimeDriverError::Internal(format!(
1947 "failed to start machine cleanup runtime: {error}"
1948 ))
1949 })?;
1950 let handle = candidate.handle().clone();
1951 *runtime = Some(candidate);
1952 Ok(Self { handle })
1953 }
1954
1955 #[cfg(target_arch = "wasm32")]
1956 fn acquire() -> Result<Self, RuntimeDriverError> {
1957 Ok(Self {})
1958 }
1959
1960 #[cfg(not(target_arch = "wasm32"))]
1961 fn spawn<F, T>(&self, future: F) -> crate::tokio::task::JoinHandle<T>
1962 where
1963 F: Future<Output = T> + Send + 'static,
1964 T: Send + 'static,
1965 {
1966 self.handle.spawn(future)
1967 }
1968
1969 #[cfg(target_arch = "wasm32")]
1970 fn spawn<F, T>(&self, future: F) -> crate::tokio::task::JoinHandle<T>
1971 where
1972 F: Future<Output = T> + 'static,
1973 T: 'static,
1974 {
1975 crate::tokio::spawn(future)
1976 }
1977}
1978
1979impl std::fmt::Debug for EnsureRuntimeExecutorAttachment {
1980 fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
1981 match self {
1982 Self::Existing(witness) => formatter.debug_tuple("Existing").field(witness).finish(),
1983 Self::Pending(pending) => formatter.debug_tuple("Pending").field(pending).finish(),
1984 }
1985 }
1986}
1987
1988#[must_use = "dropping a pending runtime attachment aborts that exact attachment"]
1997pub struct PendingRuntimeExecutorAttachment {
1998 machine: Arc<MeerkatMachine>,
1999 witness: RuntimeExecutorAttachmentWitness,
2000 mutation_guard: Option<crate::tokio::sync::OwnedMutexGuard<()>>,
2001 cleanup_spawner: MachineCleanupTaskSpawner,
2002 should_wake: bool,
2003 persist_lifecycle_on_commit: bool,
2004 armed: bool,
2005}
2006
2007#[must_use = "dropping a retained runtime attachment aborts that exact attachment"]
2016pub struct CommittedRuntimeExecutorAttachmentPublicationLease {
2017 machine: Arc<MeerkatMachine>,
2018 witness: RuntimeExecutorAttachmentWitness,
2019 mutation_guard: Option<crate::tokio::sync::OwnedMutexGuard<()>>,
2020 cleanup_spawner: MachineCleanupTaskSpawner,
2021 should_wake: bool,
2022 persist_lifecycle_on_commit: bool,
2023 armed: bool,
2024}
2025
2026#[must_use = "dropping a prepared runtime attachment retirement starts exact retirement"]
2036pub struct PreparedRuntimeExecutorAttachmentRetirement {
2037 machine: Arc<MeerkatMachine>,
2038 witness: RuntimeExecutorAttachmentWitness,
2039 mutation_guard: Option<crate::tokio::sync::OwnedMutexGuard<()>>,
2040 cleanup_spawner: MachineCleanupTaskSpawner,
2041 armed: bool,
2042}
2043
2044#[must_use = "drop is cancellation-safe, but await completion to observe retirement errors"]
2050pub struct RuntimeExecutorAttachmentRetirementCompletion {
2051 result_rx: crate::tokio::sync::oneshot::Receiver<Result<bool, RuntimeDriverError>>,
2052}
2053
2054impl RuntimeExecutorAttachmentRetirementCompletion {
2055 fn new(
2056 result_rx: crate::tokio::sync::oneshot::Receiver<Result<bool, RuntimeDriverError>>,
2057 ) -> Self {
2058 Self { result_rx }
2059 }
2060
2061 pub async fn wait(self) -> Result<bool, RuntimeDriverError> {
2062 self.result_rx.await.map_err(|error| {
2063 RuntimeDriverError::Internal(format!(
2064 "owned exact runtime attachment retirement ended without a result: {error}"
2065 ))
2066 })?
2067 }
2068}
2069
2070impl PreparedRuntimeExecutorAttachmentRetirement {
2071 fn new(
2072 machine: Arc<MeerkatMachine>,
2073 witness: RuntimeExecutorAttachmentWitness,
2074 mutation_guard: crate::tokio::sync::OwnedMutexGuard<()>,
2075 cleanup_spawner: MachineCleanupTaskSpawner,
2076 ) -> Self {
2077 Self {
2078 machine,
2079 witness,
2080 mutation_guard: Some(mutation_guard),
2081 cleanup_spawner,
2082 armed: true,
2083 }
2084 }
2085
2086 pub fn witness(&self) -> &RuntimeExecutorAttachmentWitness {
2087 &self.witness
2088 }
2089
2090 pub fn commit(
2094 mut self,
2095 ) -> Result<RuntimeExecutorAttachmentRetirementCompletion, RuntimeDriverError> {
2096 let guard = self.mutation_guard.take().ok_or_else(|| {
2097 RuntimeDriverError::Internal(
2098 "prepared exact attachment retirement lost its mutation fence".to_string(),
2099 )
2100 })?;
2101 let completion = self
2102 .machine
2103 .spawn_executor_attachment_retirement_with_guard(
2104 self.witness.clone(),
2105 guard,
2106 self.cleanup_spawner.clone(),
2107 );
2108 self.armed = false;
2109 Ok(completion)
2110 }
2111
2112 pub async fn commit_under_runtime_turn_finalization_boundary(
2117 mut self,
2118 ) -> Result<bool, RuntimeDriverError> {
2119 self.machine
2120 .complete_executor_attachment_cleanup_under_runtime_turn_boundary(&self.witness)
2121 .await?;
2122 let guard = self.mutation_guard.take().ok_or_else(|| {
2123 RuntimeDriverError::Internal(
2124 "prepared exact attachment retirement lost its mutation fence".to_string(),
2125 )
2126 })?;
2127 self.armed = false;
2128 self.machine
2129 .unregister_executor_attachment_if_current_with_guard(self.witness.clone(), guard)
2130 .await
2131 }
2132}
2133
2134impl Drop for PreparedRuntimeExecutorAttachmentRetirement {
2135 fn drop(&mut self) {
2136 if !self.armed {
2137 return;
2138 }
2139 let Some(guard) = self.mutation_guard.take() else {
2140 return;
2141 };
2142 let _completion = self
2143 .machine
2144 .spawn_executor_attachment_retirement_with_guard(
2145 self.witness.clone(),
2146 guard,
2147 self.cleanup_spawner.clone(),
2148 );
2149 self.armed = false;
2150 }
2151}
2152
2153impl std::fmt::Debug for PendingRuntimeExecutorAttachment {
2154 fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
2155 formatter
2156 .debug_struct("PendingRuntimeExecutorAttachment")
2157 .field("witness", &self.witness)
2158 .field("should_wake", &self.should_wake)
2159 .field("armed", &self.armed)
2160 .finish()
2161 }
2162}
2163
2164impl PendingRuntimeExecutorAttachment {
2165 fn new(
2166 machine: Arc<MeerkatMachine>,
2167 witness: RuntimeExecutorAttachmentWitness,
2168 mutation_guard: crate::tokio::sync::OwnedMutexGuard<()>,
2169 cleanup_spawner: MachineCleanupTaskSpawner,
2170 should_wake: bool,
2171 persist_lifecycle_on_commit: bool,
2172 ) -> Self {
2173 Self {
2174 machine,
2175 witness,
2176 mutation_guard: Some(mutation_guard),
2177 cleanup_spawner,
2178 should_wake,
2179 persist_lifecycle_on_commit,
2180 armed: true,
2181 }
2182 }
2183
2184 pub fn witness(&self) -> &RuntimeExecutorAttachmentWitness {
2185 &self.witness
2186 }
2187
2188 #[doc(hidden)]
2189 pub fn cleanup_task_spawner(&self) -> RuntimeCleanupTaskSpawner {
2190 RuntimeCleanupTaskSpawner {
2191 inner: self.cleanup_spawner.clone(),
2192 }
2193 }
2194
2195 pub async fn commit(self) -> Result<RuntimeExecutorAttachmentWitness, RuntimeDriverError> {
2196 self.commit_with_predecessor_mode(|_| Ok(()), false).await
2197 }
2198
2199 pub async fn commit_replacing_predecessor(
2210 self,
2211 ) -> Result<RuntimeExecutorAttachmentWitness, RuntimeDriverError> {
2212 self.commit_with_predecessor_mode(|_| Ok(()), true).await
2213 }
2214
2215 pub async fn commit_with<F>(
2223 self,
2224 on_committed: F,
2225 ) -> Result<RuntimeExecutorAttachmentWitness, RuntimeDriverError>
2226 where
2227 F: FnOnce(&RuntimeExecutorAttachmentWitness) -> Result<(), RuntimeDriverError>
2228 + Send
2229 + 'static,
2230 {
2231 self.commit_with_predecessor_mode(on_committed, false).await
2232 }
2233
2234 pub async fn commit_with_replacing_predecessor<F>(
2243 self,
2244 on_committed: F,
2245 ) -> Result<RuntimeExecutorAttachmentWitness, RuntimeDriverError>
2246 where
2247 F: FnOnce(&RuntimeExecutorAttachmentWitness) -> Result<(), RuntimeDriverError>
2248 + Send
2249 + 'static,
2250 {
2251 self.commit_with_predecessor_mode(on_committed, true).await
2252 }
2253
2254 async fn commit_with_predecessor_mode<F>(
2255 self,
2256 on_committed: F,
2257 replaces_predecessor: bool,
2258 ) -> Result<RuntimeExecutorAttachmentWitness, RuntimeDriverError>
2259 where
2260 F: FnOnce(&RuntimeExecutorAttachmentWitness) -> Result<(), RuntimeDriverError>
2261 + Send
2262 + 'static,
2263 {
2264 let cleanup_spawner = self.cleanup_spawner.clone();
2265 let completion = cleanup_spawner.spawn(async move {
2266 let mut pending = self;
2267 let result = pending
2268 .try_commit_with(on_committed, false, replaces_predecessor)
2269 .await;
2270 match result {
2271 Ok(witness) => Ok(witness),
2272 Err(commit_error) => match pending.abort().await {
2273 Ok(()) => Err(commit_error),
2274 Err(cleanup_error) => Err(RuntimeDriverError::Internal(format!(
2275 "{commit_error}; exact attachment cleanup also failed: {cleanup_error}"
2276 ))),
2277 },
2278 }
2279 });
2280 completion.await.map_err(|error| {
2281 RuntimeDriverError::Internal(format!(
2282 "owned runtime attachment commit ended without a result: {error}"
2283 ))
2284 })?
2285 }
2286
2287 pub async fn try_commit_with_under_runtime_turn_finalization_boundary<F>(
2300 &mut self,
2301 replaces_predecessor: bool,
2302 on_committed: F,
2303 ) -> Result<RuntimeExecutorAttachmentWitness, RuntimeDriverError>
2304 where
2305 F: FnOnce(&RuntimeExecutorAttachmentWitness) -> Result<(), RuntimeDriverError>,
2306 {
2307 self.try_commit_with(on_committed, false, replaces_predecessor)
2308 .await
2309 }
2310
2311 pub async fn try_commit_with_retained_publication_lease_under_runtime_turn_finalization_boundary(
2316 &mut self,
2317 ) -> Result<CommittedRuntimeExecutorAttachmentPublicationLease, RuntimeDriverError> {
2318 let witness = self.try_commit_with(|_| Ok(()), true, false).await?;
2319 let mutation_guard = self.mutation_guard.take().ok_or_else(|| {
2320 RuntimeDriverError::Internal(
2321 "committed attachment lost its retained publication mutation fence".to_string(),
2322 )
2323 })?;
2324 Ok(CommittedRuntimeExecutorAttachmentPublicationLease {
2327 machine: Arc::clone(&self.machine),
2328 witness,
2329 mutation_guard: Some(mutation_guard),
2330 cleanup_spawner: self.cleanup_spawner.clone(),
2331 should_wake: self.should_wake,
2332 persist_lifecycle_on_commit: self.persist_lifecycle_on_commit,
2333 armed: true,
2334 })
2335 }
2336
2337 async fn try_commit_with<F>(
2338 &mut self,
2339 on_committed: F,
2340 retain_mutation_guard: bool,
2341 replaces_predecessor: bool,
2342 ) -> Result<RuntimeExecutorAttachmentWitness, RuntimeDriverError>
2343 where
2344 F: FnOnce(&RuntimeExecutorAttachmentWitness) -> Result<(), RuntimeDriverError>,
2345 {
2346 let guard = self.mutation_guard.as_ref().ok_or_else(|| {
2347 RuntimeDriverError::Internal(
2348 "pending runtime attachment lost its mutation fence before commit".to_string(),
2349 )
2350 })?;
2351 if retain_mutation_guard && replaces_predecessor {
2352 return Err(RuntimeDriverError::Internal(
2353 "predecessor request terminalization belongs to final serving publication"
2354 .to_string(),
2355 ));
2356 }
2357 let predecessor_request_state = if replaces_predecessor {
2358 let sessions = self.machine.sessions.read().await;
2359 let entry =
2360 sessions
2361 .get(self.witness.session_id())
2362 .ok_or(RuntimeDriverError::NotReady {
2363 state: RuntimeState::Destroyed,
2364 })?;
2365 let exact_pending = entry.epoch_id == self.witness.epoch_id
2366 && matches!(
2367 &entry.attachment_slot,
2368 RuntimeLoopAttachmentSlot::Pending(attachment)
2369 if attachment.id == self.witness.attachment_id
2370 && !attachment.wake_tx.is_closed()
2371 && !attachment.effect_tx.is_closed()
2372 );
2373 if !exact_pending {
2374 return Err(RuntimeDriverError::StaleAuthority {
2375 reason: format!(
2376 "replacement attachment for session {} changed before waiter fencing",
2377 self.witness.session_id()
2378 ),
2379 });
2380 }
2381 Some((
2382 entry.driver.clone(),
2383 entry.completions.clone(),
2384 entry.publication_handle(),
2385 ))
2386 } else {
2387 None
2388 };
2389 let mut predecessor_terminal_publication = None;
2390 if let Some((driver, completions, publication_handle)) = predecessor_request_state {
2391 const REPLACED_REASON: &str =
2392 "executor attachment was replaced before request completion";
2393 let (predecessor_input_ids, candidate_owner_input_id) = {
2394 let mut driver = driver.lock().await;
2395 let predecessor_input_ids = driver.as_driver().active_input_ids();
2396 if predecessor_input_ids.is_empty() {
2397 (predecessor_input_ids, None)
2398 } else {
2399 let prepared = driver.prepare_runless_runtime_terminated_interaction_outboxes(
2404 &predecessor_input_ids,
2405 REPLACED_REASON.to_string(),
2406 )?;
2407 if let Err(error) = driver
2408 .abandon_pending_inputs(crate::input_state::InputAbandonReason::Cancelled)
2409 .await
2410 {
2411 driver.rollback_prepared_runless_interaction_terminal_outboxes(prepared);
2412 return Err(error);
2413 }
2414 let candidate_owner_input_id = crate::meerkat_machine::driver::DriverEntry::commit_prepared_runless_interaction_terminal_outboxes(prepared);
2415 (predecessor_input_ids, candidate_owner_input_id)
2416 }
2417 };
2418 if !predecessor_input_ids.is_empty() {
2419 completions.lock().await.fail_inputs(
2426 predecessor_input_ids.iter().cloned(),
2427 crate::completion::CompletionWaitError::AttachmentReplaced,
2428 );
2429 self.should_wake = false;
2430 if let Some(candidate_owner_input_id) = candidate_owner_input_id {
2431 predecessor_terminal_publication = Some((
2432 driver,
2433 publication_handle,
2434 predecessor_input_ids,
2435 candidate_owner_input_id,
2436 ));
2437 }
2438 }
2439 }
2440 let result = self
2441 .machine
2442 .commit_pending_executor_attachment(
2443 &self.witness,
2444 guard,
2445 self.should_wake,
2446 self.persist_lifecycle_on_commit,
2447 retain_mutation_guard,
2448 on_committed,
2449 )
2450 .await;
2451 if result.is_ok() {
2452 if !retain_mutation_guard {
2453 self.mutation_guard.take();
2454 }
2455 self.armed = false;
2456 if let Some((driver, publication_handle, input_ids, candidate_owner_input_id)) =
2457 predecessor_terminal_publication
2458 {
2459 let _terminal_publication = self.cleanup_spawner.spawn(async move {
2465 let deadline = Instant::now() + std::time::Duration::from_secs(5);
2466 if let Err(error) = crate::control_plane::publish_and_resolve_runless_runtime_termination_before(
2467 &driver,
2468 None,
2469 publication_handle.as_deref(),
2470 &input_ids,
2471 Some(&candidate_owner_input_id),
2472 "executor attachment was replaced before request completion",
2473 Some(deadline),
2474 )
2475 .await
2476 {
2477 tracing::warn!(
2478 %error,
2479 "replacement terminal publication remains durably pending"
2480 );
2481 }
2482 });
2483 }
2484 }
2485 result
2486 }
2487
2488 pub async fn abort(mut self) -> Result<(), RuntimeDriverError> {
2489 let guard = self.mutation_guard.take().ok_or_else(|| {
2490 RuntimeDriverError::Internal(
2491 "pending runtime attachment lost its mutation fence before abort".to_string(),
2492 )
2493 })?;
2494 let machine = Arc::clone(&self.machine);
2495 let witness = self.witness.clone();
2496 let completion = self.cleanup_spawner.spawn(async move {
2497 machine
2498 .abort_pending_executor_attachment(witness, guard)
2499 .await
2500 });
2501 self.armed = false;
2502 completion.await.map_err(|error| {
2503 RuntimeDriverError::Internal(format!(
2504 "owned runtime attachment abort ended without a result: {error}"
2505 ))
2506 })?
2507 }
2508
2509 pub async fn abort_under_runtime_turn_finalization_boundary(
2513 mut self,
2514 ) -> Result<(), RuntimeDriverError> {
2515 self.machine
2516 .complete_executor_attachment_cleanup_under_runtime_turn_boundary(&self.witness)
2517 .await?;
2518 let guard = self.mutation_guard.take().ok_or_else(|| {
2519 RuntimeDriverError::Internal(
2520 "pending runtime attachment lost its mutation fence before boundary-owned abort"
2521 .to_string(),
2522 )
2523 })?;
2524 let machine = Arc::clone(&self.machine);
2525 let witness = self.witness.clone();
2526 let completion = self.cleanup_spawner.spawn(async move {
2527 machine
2528 .abort_pending_executor_attachment(witness, guard)
2529 .await
2530 });
2531 self.armed = false;
2532 completion.await.map_err(|error| {
2533 RuntimeDriverError::Internal(format!(
2534 "owned boundary-aware runtime attachment abort ended without a result: {error}"
2535 ))
2536 })?
2537 }
2538}
2539
2540impl CommittedRuntimeExecutorAttachmentPublicationLease {
2541 pub fn witness(&self) -> &RuntimeExecutorAttachmentWitness {
2542 &self.witness
2543 }
2544
2545 #[doc(hidden)]
2546 pub fn cleanup_task_spawner(&self) -> RuntimeCleanupTaskSpawner {
2547 RuntimeCleanupTaskSpawner {
2548 inner: self.cleanup_spawner.clone(),
2549 }
2550 }
2551
2552 pub async fn abandon_recovered_predecessor_inputs(
2564 &mut self,
2565 ) -> Result<usize, RuntimeDriverError> {
2566 let _guard = self.mutation_guard.as_ref().ok_or_else(|| {
2567 RuntimeDriverError::Internal(
2568 "retained attachment lost its mutation fence before predecessor abandonment"
2569 .to_string(),
2570 )
2571 })?;
2572 let (driver, completions) = {
2573 let sessions = self.machine.sessions.read().await;
2574 let entry = sessions.get(self.witness.session_id()).ok_or_else(|| {
2575 RuntimeDriverError::StaleAuthority {
2576 reason: format!(
2577 "recovered predecessor session {} disappeared before abandonment",
2578 self.witness.session_id()
2579 ),
2580 }
2581 })?;
2582 let exact_pending = entry.epoch_id == self.witness.epoch_id
2583 && matches!(
2584 &entry.attachment_slot,
2585 RuntimeLoopAttachmentSlot::Pending(attachment)
2586 if attachment.id == self.witness.attachment_id
2587 && !attachment.wake_tx.is_closed()
2588 && !attachment.effect_tx.is_closed()
2589 );
2590 if !exact_pending {
2591 return Err(RuntimeDriverError::StaleAuthority {
2592 reason: format!(
2593 "recovered predecessor attachment for session {} changed before abandonment",
2594 self.witness.session_id()
2595 ),
2596 });
2597 }
2598 (entry.driver.clone(), entry.completions.clone())
2599 };
2600
2601 let predecessor_input_ids = {
2602 let mut driver = driver.lock().await;
2603 let predecessor_input_ids = driver.as_driver().active_input_ids();
2604 if predecessor_input_ids.is_empty() {
2605 return Ok(0);
2606 }
2607 let abandoned = driver
2608 .abandon_pending_inputs(crate::input_state::InputAbandonReason::Cancelled)
2609 .await?;
2610 if abandoned != predecessor_input_ids.len() {
2611 return Err(RuntimeDriverError::Internal(format!(
2612 "recovered predecessor abandonment for session {} terminalized {abandoned} of {} active inputs",
2613 self.witness.session_id(),
2614 predecessor_input_ids.len()
2615 )));
2616 }
2617 for input_id in &predecessor_input_ids {
2618 let stored = driver
2619 .as_driver()
2620 .stored_input_state(input_id)
2621 .ok_or_else(|| {
2622 RuntimeDriverError::Internal(format!(
2623 "recovered predecessor input {input_id} disappeared after abandonment"
2624 ))
2625 })?;
2626 if !matches!(
2627 stored.seed.terminal_outcome,
2628 Some(crate::input_state::InputTerminalOutcome::Abandoned {
2629 reason: crate::input_state::InputAbandonReason::Cancelled,
2630 })
2631 ) || stored.state.interaction_terminal_outbox.is_some()
2632 {
2633 return Err(RuntimeDriverError::Internal(format!(
2634 "recovered predecessor input {input_id} did not reach silent Cancelled abandonment"
2635 )));
2636 }
2637 }
2638 predecessor_input_ids
2639 };
2640 let abandoned = predecessor_input_ids.len();
2641 completions.lock().await.fail_inputs(
2642 predecessor_input_ids,
2643 crate::completion::CompletionWaitError::AttachmentReplaced,
2644 );
2645 Ok(abandoned)
2646 }
2647
2648 pub async fn commit_with<F>(
2651 mut self,
2652 on_committed: F,
2653 ) -> Result<RuntimeExecutorAttachmentWitness, RuntimeDriverError>
2654 where
2655 F: FnOnce(&RuntimeExecutorAttachmentWitness) -> Result<(), RuntimeDriverError>,
2656 {
2657 let guard = self.mutation_guard.as_ref().ok_or_else(|| {
2658 RuntimeDriverError::Internal(
2659 "retained attachment publication lost its mutation fence".to_string(),
2660 )
2661 })?;
2662 let witness = self
2663 .machine
2664 .commit_retained_executor_attachment_publication(
2665 &self.witness,
2666 guard,
2667 self.should_wake,
2668 self.persist_lifecycle_on_commit,
2669 move |witness, _| on_committed(witness),
2670 )
2671 .await?;
2672 self.mutation_guard.take();
2673 self.armed = false;
2674 Ok(witness)
2675 }
2676
2677 pub async fn commit_with_member_residency<F>(
2680 &mut self,
2681 residency_update: MemberResidencyUpdate,
2682 incarnation: meerkat_contracts::wire::supervisor_bridge::BridgeMemberIncarnation,
2683 tracked_turn_journal: Option<Arc<dyn crate::member_observation::TrackedTurnJournal>>,
2684 on_committed: F,
2685 ) -> Result<(RuntimeExecutorAttachmentWitness, MemberResidencyPublication), RuntimeDriverError>
2686 where
2687 F: FnOnce(&RuntimeExecutorAttachmentWitness) -> Result<(), RuntimeDriverError>,
2688 {
2689 residency_update.validate_for_retained_attachment(
2690 &self.machine,
2691 &self.witness,
2692 &incarnation,
2693 )?;
2694 let guard = self.mutation_guard.as_ref().ok_or_else(|| {
2695 RuntimeDriverError::Internal(
2696 "retained host publication lost its mutation fence".to_string(),
2697 )
2698 })?;
2699 let mut staged_residency_publication = None;
2700 let commit = self
2701 .machine
2702 .commit_retained_executor_attachment_publication(
2703 &self.witness,
2704 guard,
2705 self.should_wake,
2706 self.persist_lifecycle_on_commit,
2707 |witness, session_mutation_gate| {
2708 on_committed(witness)?;
2709 staged_residency_publication =
2710 Some(residency_update.stage_under_retained_mutation_gate(
2711 incarnation,
2712 tracked_turn_journal,
2713 session_mutation_gate,
2714 )?);
2715 Ok(())
2716 },
2717 )
2718 .await;
2719 let witness = match commit {
2720 Ok(witness) => witness,
2721 Err(error) => {
2722 drop(staged_residency_publication);
2726 return Err(error);
2727 }
2728 };
2729 let residency_publication = staged_residency_publication
2730 .take()
2731 .ok_or_else(|| {
2732 RuntimeDriverError::Internal(format!(
2733 "retained host publication for session {} committed without residency",
2734 self.witness.session_id()
2735 ))
2736 })?
2737 .finalize()?;
2738 self.mutation_guard.take();
2739 self.armed = false;
2740 Ok((witness, residency_publication))
2741 }
2742
2743 pub fn begin_abort(
2752 mut self,
2753 ) -> Result<RuntimeExecutorAttachmentRetirementCompletion, RuntimeDriverError> {
2754 let guard = self.mutation_guard.take().ok_or_else(|| {
2755 RuntimeDriverError::Internal(
2756 "committed publication lease lost its mutation fence before abort".to_string(),
2757 )
2758 })?;
2759 let completion = self
2760 .machine
2761 .spawn_executor_attachment_retirement_with_guard(
2762 self.witness.clone(),
2763 guard,
2764 self.cleanup_spawner.clone(),
2765 );
2766 self.armed = false;
2767 Ok(completion)
2768 }
2769
2770 pub async fn abort_under_runtime_turn_finalization_boundary(
2772 mut self,
2773 ) -> Result<bool, RuntimeDriverError> {
2774 self.machine
2775 .complete_executor_attachment_cleanup_under_runtime_turn_boundary(&self.witness)
2776 .await?;
2777 let guard = self.mutation_guard.take().ok_or_else(|| {
2778 RuntimeDriverError::Internal(
2779 "committed publication lease lost its mutation fence before abort".to_string(),
2780 )
2781 })?;
2782 let machine = Arc::clone(&self.machine);
2783 let witness = self.witness.clone();
2784 let completion = self.cleanup_spawner.spawn(async move {
2785 machine
2786 .unregister_executor_attachment_if_current_with_guard(witness, guard)
2787 .await
2788 });
2789 self.armed = false;
2790 completion.await.map_err(|error| {
2791 RuntimeDriverError::Internal(format!(
2792 "owned committed attachment abort ended without a result: {error}"
2793 ))
2794 })?
2795 }
2796}
2797
2798impl Drop for CommittedRuntimeExecutorAttachmentPublicationLease {
2799 fn drop(&mut self) {
2800 if !self.armed {
2801 return;
2802 }
2803 let Some(guard) = self.mutation_guard.take() else {
2804 return;
2805 };
2806 self.armed = false;
2807 let machine = Arc::clone(&self.machine);
2808 let witness = self.witness.clone();
2809 self.cleanup_spawner.spawn(async move {
2810 if let Err(error) = machine
2811 .unregister_executor_attachment_if_current_with_guard(witness.clone(), guard)
2812 .await
2813 {
2814 tracing::warn!(
2815 session_id = %witness.session_id(),
2816 %error,
2817 "drop-time committed attachment publication abort failed"
2818 );
2819 }
2820 });
2821 }
2822}
2823
2824impl Drop for PendingRuntimeExecutorAttachment {
2825 fn drop(&mut self) {
2826 if !self.armed {
2827 return;
2828 }
2829 let Some(guard) = self.mutation_guard.take() else {
2830 return;
2831 };
2832 self.armed = false;
2833 let machine = Arc::clone(&self.machine);
2834 let witness = self.witness.clone();
2835 self.cleanup_spawner.spawn(async move {
2836 if let Err(error) = machine
2837 .abort_pending_executor_attachment(witness.clone(), guard)
2838 .await
2839 {
2840 tracing::warn!(
2841 session_id = %witness.session_id(),
2842 %error,
2843 "drop-time exact runtime attachment abort failed"
2844 );
2845 }
2846 });
2847 }
2848}
2849
2850impl std::fmt::Debug for PreparedSessionMaterialization {
2851 fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
2852 formatter
2853 .debug_struct("PreparedSessionMaterialization")
2854 .field("session_id", self.bindings.session_id())
2855 .field("epoch_id", self.bindings.epoch_id())
2856 .field("claim_id", &self.claim_id)
2857 .field("armed", &self.armed)
2858 .finish()
2859 }
2860}
2861
2862impl std::fmt::Debug for PreparedAttachedSessionActorRecovery {
2863 fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
2864 formatter
2865 .debug_struct("PreparedAttachedSessionActorRecovery")
2866 .field("session_id", self.bindings.session_id())
2867 .field("epoch_id", self.bindings.epoch_id())
2868 .field("witness", &self.witness)
2869 .field("armed", &self.armed)
2870 .finish()
2871 }
2872}
2873
2874impl PreparedAttachedSessionActorRecovery {
2875 fn new(
2876 bindings: meerkat_core::SessionRuntimeBindings,
2877 witness: RuntimeExecutorAttachmentWitness,
2878 claim_id: uuid::Uuid,
2879 claim_state: Arc<std::sync::Mutex<crate::RuntimeActorMaterializationClaimState>>,
2880 previous_phase: crate::RuntimeActorMaterializationClaimPhase,
2881 mutation_guard: crate::tokio::sync::OwnedMutexGuard<()>,
2882 ) -> Self {
2883 Self {
2884 bindings,
2885 witness,
2886 claim_id,
2887 claim_state,
2888 previous_phase,
2889 mutation_guard: Some(mutation_guard),
2890 armed: true,
2891 }
2892 }
2893
2894 pub fn bindings(&self) -> &meerkat_core::SessionRuntimeBindings {
2895 &self.bindings
2896 }
2897
2898 pub fn bindings_clone(&self) -> meerkat_core::SessionRuntimeBindings {
2899 self.bindings.clone()
2900 }
2901
2902 pub fn witness(&self) -> &RuntimeExecutorAttachmentWitness {
2903 &self.witness
2904 }
2905
2906 pub fn commit_actor(&mut self) -> Result<(), RuntimeDriverError> {
2909 if !self.armed {
2910 return Err(RuntimeDriverError::StaleAuthority {
2911 reason: "attached actor-recovery lease is no longer active".to_string(),
2912 });
2913 }
2914 let changed = {
2915 let mut state = self
2916 .claim_state
2917 .lock()
2918 .unwrap_or_else(std::sync::PoisonError::into_inner);
2919 if !state.exact_claim_is(
2920 self.claim_id,
2921 &[crate::RuntimeActorMaterializationClaimPhase::ActorMaterializedPendingCommit],
2922 ) {
2923 return Err(RuntimeDriverError::StaleAuthority {
2924 reason: format!(
2925 "actor recovery for session {} did not retain its exact materialization claim",
2926 self.bindings.session_id()
2927 ),
2928 });
2929 }
2930 state.current = None;
2931 state.phase = crate::RuntimeActorMaterializationClaimPhase::RetainedActor;
2932 Arc::clone(&state.changed)
2933 };
2934 changed.notify_waiters();
2935 self.armed = false;
2936 self.mutation_guard.take();
2937 Ok(())
2938 }
2939}
2940
2941impl Drop for PreparedAttachedSessionActorRecovery {
2942 fn drop(&mut self) {
2943 if !self.armed {
2944 return;
2945 }
2946 let changed = {
2947 let mut state = self
2948 .claim_state
2949 .lock()
2950 .unwrap_or_else(std::sync::PoisonError::into_inner);
2951 if state.current == Some(self.claim_id) {
2952 state.current = None;
2953 state.phase = if state.phase
2954 == crate::RuntimeActorMaterializationClaimPhase::ActorMaterializedPendingCommit
2955 {
2956 crate::RuntimeActorMaterializationClaimPhase::RetainedActor
2957 } else {
2958 self.previous_phase
2959 };
2960 Some(Arc::clone(&state.changed))
2961 } else {
2962 None
2963 }
2964 };
2965 if let Some(changed) = changed {
2966 changed.notify_waiters();
2967 }
2968 self.armed = false;
2969 self.mutation_guard.take();
2970 }
2971}
2972
2973impl PreparedSessionMaterialization {
2974 fn new(
2975 machine: Arc<MeerkatMachine>,
2976 bindings: meerkat_core::SessionRuntimeBindings,
2977 claim_id: uuid::Uuid,
2978 cleanup_spawner: MachineCleanupTaskSpawner,
2979 ) -> Result<Self, RuntimeDriverError> {
2980 let authority = bindings
2981 .__runtime_authority()
2982 .downcast_ref::<crate::SessionRuntimeBindingsAuthority>()
2983 .ok_or_else(|| RuntimeDriverError::ValidationFailed {
2984 reason: "prepared materialization lacks MeerkatMachine authority".to_string(),
2985 })?;
2986 if authority.materialization_claim_id != Some(claim_id) {
2987 return Err(RuntimeDriverError::StaleAuthority {
2988 reason: "prepared materialization binding lost its exact claim".to_string(),
2989 });
2990 }
2991 let claim_state = Arc::clone(&authority.materialization_claim_state);
2992 Ok(Self {
2993 machine,
2994 bindings,
2995 claim_id,
2996 claim_state,
2997 cleanup_spawner,
2998 archived_resume_authorization_issued: false,
2999 armed: true,
3000 })
3001 }
3002
3003 pub fn bindings(&self) -> &meerkat_core::SessionRuntimeBindings {
3004 &self.bindings
3005 }
3006
3007 pub fn bindings_clone(&self) -> meerkat_core::SessionRuntimeBindings {
3008 self.bindings.clone()
3009 }
3010
3011 pub fn session_id(&self) -> &SessionId {
3012 self.bindings.session_id()
3013 }
3014
3015 #[doc(hidden)]
3016 pub fn cleanup_task_spawner(&self) -> RuntimeCleanupTaskSpawner {
3017 RuntimeCleanupTaskSpawner {
3018 inner: self.cleanup_spawner.clone(),
3019 }
3020 }
3021
3022 #[must_use]
3025 pub async fn owns_current_materialization_claim(&self) -> bool {
3026 if !self.armed {
3027 return false;
3028 }
3029 let Some(_mutation_guard) = self
3030 .machine
3031 .lock_current_durability_ready_session_mutation_gate(self.session_id())
3032 .await
3033 .ok()
3034 else {
3035 return false;
3036 };
3037 let Ok(authority) = crate::validated_session_runtime_bindings_authority(&self.bindings)
3038 else {
3039 return false;
3040 };
3041 let sessions = self.machine.sessions.read().await;
3042 sessions.get(self.session_id()).is_some_and(|entry| {
3043 entry.epoch_id == *self.bindings.epoch_id()
3044 && Arc::ptr_eq(&entry.materialization_claim_state, &self.claim_state)
3045 && Arc::ptr_eq(&entry.dsl_authority, &authority.dsl_authority)
3046 && Arc::ptr_eq(&entry.handle_teardown_gate, &authority.teardown_gate)
3047 && !entry.physical_attachment_is_live()
3048 && self
3049 .claim_state
3050 .lock()
3051 .unwrap_or_else(std::sync::PoisonError::into_inner)
3052 .exact_claim_is(
3053 self.claim_id,
3054 &[
3055 crate::RuntimeActorMaterializationClaimPhase::Prepared,
3056 crate::RuntimeActorMaterializationClaimPhase::Staged,
3057 crate::RuntimeActorMaterializationClaimPhase::ActorCreating,
3058 crate::RuntimeActorMaterializationClaimPhase::ActorMaterializedPendingCommit,
3059 crate::RuntimeActorMaterializationClaimPhase::Aborting,
3060 ],
3061 )
3062 })
3063 }
3064
3065 pub fn archived_resume_authorization(
3073 &mut self,
3074 ) -> Result<ArchivedSessionActorMaterializationAuthorization, RuntimeDriverError> {
3075 if !self.armed {
3076 return Err(RuntimeDriverError::StaleAuthority {
3077 reason: format!(
3078 "prepared materialization for session {} no longer owns archived-resume authority",
3079 self.session_id()
3080 ),
3081 });
3082 }
3083 if self.archived_resume_authorization_issued {
3084 return Err(RuntimeDriverError::StaleAuthority {
3085 reason: format!(
3086 "prepared materialization for session {} already issued its one-shot archived-resume authorization",
3087 self.session_id()
3088 ),
3089 });
3090 }
3091 let authority = self
3092 .bindings
3093 .__runtime_authority()
3094 .downcast_ref::<crate::SessionRuntimeBindingsAuthority>()
3095 .ok_or_else(|| RuntimeDriverError::ValidationFailed {
3096 reason: "prepared archived-resume materialization lacks machine authority"
3097 .to_string(),
3098 })?;
3099 if authority.materialization_claim_id != Some(self.claim_id)
3100 || !Arc::ptr_eq(&authority.materialization_claim_state, &self.claim_state)
3101 {
3102 return Err(RuntimeDriverError::StaleAuthority {
3103 reason: format!(
3104 "prepared archived-resume claim for session {} is no longer exact",
3105 self.session_id()
3106 ),
3107 });
3108 }
3109 let authorization = ArchivedSessionActorMaterializationAuthorization {
3110 machine: Arc::clone(&self.machine),
3111 session_id: self.session_id().clone(),
3112 epoch_id: self.bindings.epoch_id().clone(),
3113 claim_id: self.claim_id,
3114 claim_state: Arc::clone(&self.claim_state),
3115 dsl_authority: Arc::clone(&authority.dsl_authority),
3116 teardown_gate: Arc::clone(&authority.teardown_gate),
3117 };
3118 self.archived_resume_authorization_issued = true;
3119 Ok(authorization)
3120 }
3121
3122 pub async fn acquire_archived_resume_commit_lease(
3127 &self,
3128 ) -> Result<PreparedArchivedResumeCommitLease, RuntimeDriverError> {
3129 if !self.armed || !self.archived_resume_authorization_issued {
3130 return Err(RuntimeDriverError::StaleAuthority {
3131 reason: format!(
3132 "prepared materialization for session {} does not own post-actor archived-resume authority",
3133 self.session_id()
3134 ),
3135 });
3136 }
3137 let mutation_guard = self
3138 .machine
3139 .lock_current_durability_ready_session_mutation_gate(self.session_id())
3140 .await
3141 .map_err(|error| match error {
3142 RuntimeDriverError::NotReady { .. } => RuntimeDriverError::StaleAuthority {
3143 reason: format!(
3144 "archived-resume session {} disappeared before commit lease acquisition",
3145 self.session_id()
3146 ),
3147 },
3148 error => error,
3149 })?;
3150 let exact = {
3151 let sessions = self.machine.sessions.read().await;
3152 sessions.get(self.session_id()).is_some_and(|entry| {
3153 entry.epoch_id == *self.bindings.epoch_id()
3154 && Arc::ptr_eq(&entry.materialization_claim_state, &self.claim_state)
3155 && !entry.physical_attachment_is_live()
3156 && matches!(
3157 entry.control_snapshot().phase,
3158 RuntimeState::Retired | RuntimeState::Idle
3159 )
3160 && self
3161 .claim_state
3162 .lock()
3163 .unwrap_or_else(std::sync::PoisonError::into_inner)
3164 .exact_claim_is(
3165 self.claim_id,
3166 &[crate::RuntimeActorMaterializationClaimPhase::ActorMaterializedPendingCommit],
3167 )
3168 })
3169 };
3170 if !exact {
3171 return Err(RuntimeDriverError::StaleAuthority {
3172 reason: format!(
3173 "archived-resume session {} lost its exact quiescent post-actor claim",
3174 self.session_id()
3175 ),
3176 });
3177 }
3178 Ok(PreparedArchivedResumeCommitLease {
3179 machine: Arc::clone(&self.machine),
3180 session_id: self.session_id().clone(),
3181 epoch_id: self.bindings.epoch_id().clone(),
3182 claim_id: self.claim_id,
3183 claim_state: Arc::clone(&self.claim_state),
3184 _mutation_guard: mutation_guard,
3185 })
3186 }
3187
3188 pub async fn ensure_executor_attachment<F>(
3193 &mut self,
3194 executor_factory: F,
3195 ) -> Result<EnsureRuntimeExecutorAttachment, RuntimeDriverError>
3196 where
3197 F: FnOnce(
3198 RuntimeExecutorAttachmentWitness,
3199 ) -> Box<dyn meerkat_core::lifecycle::CoreExecutor>
3200 + Send
3201 + 'static,
3202 {
3203 self.ensure_executor_attachment_with_boundary_mode(executor_factory, false)
3204 .await
3205 }
3206
3207 #[doc(hidden)]
3214 pub async fn ensure_executor_attachment_under_runtime_turn_finalization_boundary<F>(
3215 &mut self,
3216 executor_factory: F,
3217 ) -> Result<EnsureRuntimeExecutorAttachment, RuntimeDriverError>
3218 where
3219 F: FnOnce(
3220 RuntimeExecutorAttachmentWitness,
3221 ) -> Box<dyn meerkat_core::lifecycle::CoreExecutor>
3222 + Send
3223 + 'static,
3224 {
3225 self.ensure_executor_attachment_with_boundary_mode(executor_factory, true)
3226 .await
3227 }
3228
3229 async fn ensure_executor_attachment_with_boundary_mode<F>(
3230 &mut self,
3231 executor_factory: F,
3232 turn_finalization_boundary_already_held: bool,
3233 ) -> Result<EnsureRuntimeExecutorAttachment, RuntimeDriverError>
3234 where
3235 F: FnOnce(
3236 RuntimeExecutorAttachmentWitness,
3237 ) -> Box<dyn meerkat_core::lifecycle::CoreExecutor>
3238 + Send
3239 + 'static,
3240 {
3241 if !self.armed {
3242 return Err(RuntimeDriverError::StaleAuthority {
3243 reason: format!(
3244 "prepared materialization for session {} no longer owns attachment authority",
3245 self.session_id()
3246 ),
3247 });
3248 }
3249 let expected_claim = RuntimeExecutorAttachmentMaterializationClaim {
3250 claim_id: self.claim_id,
3251 claim_state: Arc::clone(&self.claim_state),
3252 epoch_id: self.bindings.epoch_id().clone(),
3253 };
3254 let outcome = self
3255 .machine
3256 .ensure_session_with_executor_factory_for_materialization(
3257 self.session_id().clone(),
3258 expected_claim,
3259 turn_finalization_boundary_already_held,
3260 executor_factory,
3261 )
3262 .await?;
3263 if matches!(&outcome, EnsureRuntimeExecutorAttachment::Pending(_)) {
3264 self.armed = false;
3268 }
3269 Ok(outcome)
3270 }
3271
3272 fn request_abort(&self) -> bool {
3273 let mut state = self
3274 .claim_state
3275 .lock()
3276 .unwrap_or_else(std::sync::PoisonError::into_inner);
3277 if !state.exact_claim_is(
3278 self.claim_id,
3279 &[
3280 crate::RuntimeActorMaterializationClaimPhase::Prepared,
3281 crate::RuntimeActorMaterializationClaimPhase::Staged,
3282 crate::RuntimeActorMaterializationClaimPhase::ActorCreating,
3283 crate::RuntimeActorMaterializationClaimPhase::ActorMaterializedPendingCommit,
3284 crate::RuntimeActorMaterializationClaimPhase::Aborting,
3285 ],
3286 ) {
3287 return false;
3288 }
3289 state.phase = crate::RuntimeActorMaterializationClaimPhase::Aborting;
3290 true
3291 }
3292
3293 pub async fn rollback_now(&mut self) -> Result<bool, RuntimeDriverError> {
3297 self.rollback_now_with_turn_finalization_boundary(false)
3298 .await
3299 }
3300
3301 pub async fn rollback_now_under_turn_finalization_boundary(
3304 &mut self,
3305 ) -> Result<bool, RuntimeDriverError> {
3306 self.rollback_now_with_turn_finalization_boundary(true)
3307 .await
3308 }
3309
3310 async fn rollback_now_with_turn_finalization_boundary(
3311 &mut self,
3312 turn_finalization_boundary_already_held: bool,
3313 ) -> Result<bool, RuntimeDriverError> {
3314 if !self.armed {
3315 return Ok(false);
3316 }
3317 if !self.request_abort() {
3318 self.armed = false;
3319 return Ok(false);
3320 }
3321 let rolled_back = self
3322 .machine
3323 .abort_prepared_session_materialization_claim(
3324 self.bindings.session_id(),
3325 self.claim_id,
3326 Some(self.bindings.epoch_id()),
3327 Some(&self.claim_state),
3328 turn_finalization_boundary_already_held,
3329 )
3330 .await?;
3331 self.armed = false;
3332 Ok(rolled_back)
3333 }
3334
3335 pub async fn commit_staged(&mut self) -> Result<(), RuntimeDriverError> {
3339 self.machine
3340 .commit_prepared_session_materialization_staged(&self.bindings)
3341 .await?;
3342 self.armed = false;
3343 Ok(())
3344 }
3345
3346 pub async fn commit_actor_unattached(&mut self) -> Result<(), RuntimeDriverError> {
3350 self.machine
3351 .commit_prepared_session_actor_unattached(&self.bindings)
3352 .await?;
3353 self.armed = false;
3354 Ok(())
3355 }
3356}
3357
3358impl Drop for PreparedSessionMaterialization {
3359 fn drop(&mut self) {
3360 if !self.armed || !self.request_abort() {
3361 return;
3362 }
3363 let machine = Arc::clone(&self.machine);
3364 let session_id = self.bindings.session_id().clone();
3365 let epoch_id = self.bindings.epoch_id().clone();
3366 let claim_id = self.claim_id;
3367 let claim_state = Arc::clone(&self.claim_state);
3368 self.cleanup_spawner.spawn(async move {
3369 if let Err(error) = machine
3370 .abort_prepared_session_materialization_claim(
3371 &session_id,
3372 claim_id,
3373 Some(&epoch_id),
3374 Some(&claim_state),
3375 false,
3376 )
3377 .await
3378 {
3379 tracing::warn!(
3380 %session_id,
3381 %error,
3382 "drop-time exact prepared materialization rollback failed"
3383 );
3384 }
3385 });
3386 }
3387}
3388
3389pub(super) struct PendingPreparedMaterialization {
3390 machine: Arc<MeerkatMachine>,
3391 session_id: SessionId,
3392 claim_id: uuid::Uuid,
3393 cleanup_spawner: MachineCleanupTaskSpawner,
3394 claim_state_slot: Arc<
3395 std::sync::Mutex<
3396 Option<Arc<std::sync::Mutex<crate::RuntimeActorMaterializationClaimState>>>,
3397 >,
3398 >,
3399 armed: bool,
3400}
3401
3402impl PendingPreparedMaterialization {
3403 pub(super) fn new(
3404 machine: Arc<MeerkatMachine>,
3405 session_id: SessionId,
3406 claim_id: uuid::Uuid,
3407 ) -> Result<Self, RuntimeDriverError> {
3408 Ok(Self {
3409 machine,
3410 session_id,
3411 claim_id,
3412 cleanup_spawner: MachineCleanupTaskSpawner::acquire()?,
3413 claim_state_slot: Arc::new(std::sync::Mutex::new(None)),
3414 armed: true,
3415 })
3416 }
3417
3418 fn cleanup_spawner(&self) -> MachineCleanupTaskSpawner {
3419 self.cleanup_spawner.clone()
3420 }
3421
3422 pub(super) fn claim_state_slot(
3423 &self,
3424 ) -> Arc<
3425 std::sync::Mutex<
3426 Option<Arc<std::sync::Mutex<crate::RuntimeActorMaterializationClaimState>>>,
3427 >,
3428 > {
3429 Arc::clone(&self.claim_state_slot)
3430 }
3431
3432 pub(super) fn disarm(&mut self) {
3433 self.armed = false;
3434 }
3435}
3436
3437impl Drop for PendingPreparedMaterialization {
3438 fn drop(&mut self) {
3439 if !self.armed {
3440 return;
3441 }
3442 let claim_state = self
3443 .claim_state_slot
3444 .lock()
3445 .unwrap_or_else(std::sync::PoisonError::into_inner)
3446 .clone();
3447 if let Some(claim_state) = claim_state {
3448 let mut state = claim_state
3449 .lock()
3450 .unwrap_or_else(std::sync::PoisonError::into_inner);
3451 if state.current == Some(self.claim_id)
3452 && state.phase != crate::RuntimeActorMaterializationClaimPhase::RetainedActor
3453 {
3454 state.phase = crate::RuntimeActorMaterializationClaimPhase::Aborting;
3455 }
3456 }
3457 let machine = Arc::clone(&self.machine);
3458 let session_id = self.session_id.clone();
3459 let claim_id = self.claim_id;
3460 self.cleanup_spawner.spawn(async move {
3461 if let Err(error) = machine
3462 .abort_prepared_session_materialization_claim(
3463 &session_id,
3464 claim_id,
3465 None,
3466 None,
3467 false,
3468 )
3469 .await
3470 {
3471 tracing::warn!(
3472 %session_id,
3473 %error,
3474 "cancelled binding preparation exact rollback failed"
3475 );
3476 }
3477 });
3478 }
3479}
3480
3481pub struct MachineServiceTurnCommitLease {
3489 session_id: SessionId,
3490 run_id: RunId,
3491 driver: SharedDriver,
3492 _mutation_guard: crate::tokio::sync::OwnedMutexGuard<()>,
3493}
3494
3495pub struct MachineServiceTurnIdentity {
3499 session_id: SessionId,
3500 driver: SharedDriver,
3501}
3502
3503impl MachineServiceTurnCommitLease {
3504 #[must_use]
3506 pub fn run_id(&self) -> &RunId {
3507 &self.run_id
3508 }
3509}
3510
3511impl MachineSessionArchiveLease {
3512 pub fn owns_recovered_registration(&self) -> bool {
3515 self.recovered_registration_for_archive
3516 }
3517
3518 pub async fn runtime_state(&self) -> RuntimeState {
3521 self.driver.lock().await.runtime_state()
3522 }
3523
3524 pub async fn runtime_is_retired(&self) -> bool {
3528 self.runtime_state().await == RuntimeState::Retired
3529 }
3530}
3531
3532#[derive(Debug, Clone, Copy, PartialEq, Eq)]
3537struct RuntimeLoopAttachmentId(uuid::Uuid);
3538
3539impl RuntimeLoopAttachmentId {
3540 fn new() -> Self {
3541 Self(uuid::Uuid::new_v4())
3542 }
3543}
3544
3545struct RuntimeEffectDispatchAttachmentWitness {
3549 mutation_gate: Arc<Mutex<()>>,
3550 driver: SharedDriver,
3551 dsl_authority: Arc<std::sync::Mutex<dsl::MeerkatMachineAuthority>>,
3552 attachment_id: RuntimeLoopAttachmentId,
3553 effect_tx: mpsc::Sender<crate::effect::RuntimeEffect>,
3554}
3555
3556struct RuntimeLiveBoundaryAttachmentWitness {
3560 mutation_gate: Arc<Mutex<()>>,
3561 driver: SharedDriver,
3562 dsl_authority: Arc<std::sync::Mutex<dsl::MeerkatMachineAuthority>>,
3563 attachment_id: RuntimeLoopAttachmentId,
3564 boundary_handle: Arc<dyn meerkat_core::lifecycle::CoreExecutorBoundaryHandle>,
3565}
3566
3567struct RuntimeAcceptedBoundaryCancelPlan {
3568 witness: RuntimeEffectDispatchAttachmentWitness,
3569 boundary_handle: Option<Arc<dyn meerkat_core::lifecycle::CoreExecutorBoundaryHandle>>,
3570 pending_dispatch: PendingBoundaryCancelDispatchGuard,
3571 expected_run_id: RunId,
3572 projected_effect: crate::effect::ProjectedRuntimeEffect,
3573 dispatch_generation: u64,
3574 dispatch_lifecycle_phase: dsl::MeerkatPhase,
3575}
3576
3577#[derive(Debug, Clone, Copy, PartialEq, Eq)]
3578enum BoundaryCancelDispatchState {
3579 Current,
3580 ConsumedConverged,
3581 SameGenerationOrphan,
3582 Superseded,
3583}
3584
3585struct RuntimeEffectDispatchMemberAuthority {
3586 lease: MemberEffectAuthorityLease,
3587 expected_member: Option<meerkat_contracts::wire::supervisor_bridge::BridgeMemberIncarnation>,
3588}
3589
3590struct PendingBoundaryCancelDispatchGuard {
3593 dsl_authority: Arc<std::sync::Mutex<dsl::MeerkatMachineAuthority>>,
3594 dispatch_generation: u64,
3595 armed: bool,
3596}
3597
3598impl PendingBoundaryCancelDispatchGuard {
3599 fn new(
3600 dsl_authority: Arc<std::sync::Mutex<dsl::MeerkatMachineAuthority>>,
3601 dispatch_generation: u64,
3602 ) -> Self {
3603 Self {
3604 dsl_authority,
3605 dispatch_generation,
3606 armed: true,
3607 }
3608 }
3609
3610 fn disarm(&mut self) {
3611 self.armed = false;
3612 }
3613}
3614
3615impl Drop for PendingBoundaryCancelDispatchGuard {
3616 fn drop(&mut self) {
3617 if self.armed
3618 && let Err(error) = MeerkatMachine::abort_dsl_boundary_cancel_dispatch_if_current(
3619 &self.dsl_authority,
3620 self.dispatch_generation,
3621 )
3622 {
3623 tracing::error!(
3624 dispatch_generation = self.dispatch_generation,
3625 error = %error,
3626 "failed to compensate an abandoned boundary-cancel dispatch"
3627 );
3628 }
3629 }
3630}
3631
3632struct AcceptedIngressFallbackWakeGuard {
3635 wake_tx: Option<mpsc::Sender<()>>,
3636 armed: bool,
3637}
3638
3639impl AcceptedIngressFallbackWakeGuard {
3640 fn new(wake_tx: Option<mpsc::Sender<()>>, armed: bool) -> Self {
3641 Self { wake_tx, armed }
3642 }
3643
3644 fn disarm(&mut self) {
3645 self.armed = false;
3646 }
3647}
3648
3649impl Drop for AcceptedIngressFallbackWakeGuard {
3650 fn drop(&mut self) {
3651 if self.armed
3652 && let Some(wake_tx) = self.wake_tx.as_ref()
3653 {
3654 let _ = wake_tx.try_send(());
3655 }
3656 }
3657}
3658
3659struct RuntimeLoopAttachment {
3660 id: RuntimeLoopAttachmentId,
3661 wake_tx: mpsc::Sender<()>,
3662 effect_tx: mpsc::Sender<crate::effect::RuntimeEffect>,
3663 serving_release: Option<crate::runtime_loop::RuntimeLoopServingRelease>,
3664 boundary_handle: Option<Arc<dyn meerkat_core::lifecycle::CoreExecutorBoundaryHandle>>,
3665 interrupt_handle: Option<Arc<dyn meerkat_core::lifecycle::CoreExecutorInterruptHandle>>,
3666 loop_handle: tokio::task::JoinHandle<()>,
3667}
3668
3669enum RuntimeLoopAttachmentSlot {
3671 Empty,
3672 Pending(RuntimeLoopAttachment),
3673 Attached(RuntimeLoopAttachment),
3674}
3675
3676impl RuntimeSessionEntry {
3677 fn require_durability_ready(&self) -> Result<(), DurabilityReloadRequired> {
3678 match self.durability_health.as_ref() {
3679 Some(health) => health.require_ready(),
3680 None => Ok(()),
3681 }
3682 }
3683
3684 fn dsl_mutation_blocked_by_unregister(
3685 &self,
3686 session_id: &SessionId,
3687 ) -> Option<RuntimeDriverError> {
3688 if self.pending_unregister_finalization.is_some() {
3689 return Some(RuntimeDriverError::UnregisterFinalizationOutcomeUnknown {
3690 reason: format!(
3691 "session {session_id} retains an ambiguous unregister finalization; retry unregister before applying any other lifecycle mutation"
3692 ),
3693 });
3694 }
3695 self.handle_teardown_gate
3696 .is_closed()
3697 .then(|| RuntimeDriverError::ValidationFailed {
3698 reason: format!("session {session_id} is a teardown-only unregister retry anchor"),
3699 })
3700 }
3701
3702 fn registration_blocked_by_unregister(
3703 &self,
3704 session_id: &SessionId,
3705 ) -> Option<RuntimeDriverError> {
3706 if let Some(error) = self.dsl_mutation_blocked_by_unregister(session_id) {
3707 return Some(error);
3708 }
3709 if self.unregister_coordinator.is_some() {
3710 return Some(RuntimeDriverError::NotReady {
3711 state: self.control_snapshot().phase,
3712 });
3713 }
3714 let Some(coordinator) = self.runtime_stop_cleanup_coordinator.as_ref() else {
3715 return None;
3716 };
3717 match coordinator.result_rx.borrow().clone() {
3718 None => Some(RuntimeDriverError::RuntimeStopInProgress {
3719 runtime_id: self.runtime_id.clone(),
3720 }),
3721 Some(Ok(())) => None,
3722 Some(Err(error)) => Some(error),
3723 }
3724 }
3725
3726 fn control_snapshot(&self) -> crate::driver::ephemeral::RuntimeControlProjection {
3727 self.control_projection
3728 .read()
3729 .map(|guard| guard.clone())
3730 .unwrap_or_else(|poisoned| {
3731 tracing::error!("runtime control projection lock poisoned");
3732 poisoned.into_inner().clone()
3733 })
3734 }
3735
3736 fn sync_control_projection_from_dsl_authority(&self) {
3737 let next = {
3738 let authority = self
3739 .dsl_authority
3740 .lock()
3741 .unwrap_or_else(std::sync::PoisonError::into_inner);
3742 crate::driver::ephemeral::RuntimeControlProjection {
3743 phase: crate::meerkat_machine::dsl_authority::runtime_phase_from_authority(
3744 &authority,
3745 ),
3746 current_run_id:
3747 crate::meerkat_machine::dsl_authority::current_run_id_from_authority(&authority),
3748 pre_run_phase: crate::meerkat_machine::dsl_authority::pre_run_phase_from_authority(
3749 &authority,
3750 ),
3751 }
3752 };
3753 *self
3754 .control_projection
3755 .write()
3756 .unwrap_or_else(std::sync::PoisonError::into_inner) = next;
3757 }
3758
3759 fn physical_attachment_is_live(&self) -> bool {
3760 match &self.attachment_slot {
3761 RuntimeLoopAttachmentSlot::Pending(attachment)
3762 | RuntimeLoopAttachmentSlot::Attached(attachment) => {
3763 !attachment.wake_tx.is_closed() && !attachment.effect_tx.is_closed()
3764 }
3765 RuntimeLoopAttachmentSlot::Empty => false,
3766 }
3767 }
3768
3769 fn attachment_is_live(&self) -> bool {
3770 matches!(
3771 &self.attachment_slot,
3772 RuntimeLoopAttachmentSlot::Attached(attachment)
3773 if !attachment.wake_tx.is_closed() && !attachment.effect_tx.is_closed()
3774 )
3775 }
3776
3777 fn generated_executor_registration_active(&self) -> bool {
3778 let authority = self
3779 .dsl_authority
3780 .lock()
3781 .unwrap_or_else(std::sync::PoisonError::into_inner);
3782 matches!(
3783 authority.state().registration_phase,
3784 dsl::RegistrationPhase::Active
3785 )
3786 }
3787
3788 fn generated_executor_registration_has_viable_attachment(&self) -> bool {
3789 self.generated_executor_registration_active()
3790 && match &self.attachment_slot {
3791 RuntimeLoopAttachmentSlot::Empty => true,
3792 RuntimeLoopAttachmentSlot::Pending(_) | RuntimeLoopAttachmentSlot::Attached(_) => {
3793 self.physical_attachment_is_live()
3794 }
3795 }
3796 }
3797
3798 fn close_handle_teardown_gate(&self) {
3799 let _guard = self
3800 .dsl_authority
3801 .lock()
3802 .unwrap_or_else(std::sync::PoisonError::into_inner);
3803 self.handle_teardown_gate.close();
3804 }
3805
3806 fn generated_executor_registration_active_or_draining(&self) -> bool {
3814 let authority = self
3815 .dsl_authority
3816 .lock()
3817 .unwrap_or_else(std::sync::PoisonError::into_inner);
3818 matches!(
3819 authority.state().registration_phase,
3820 dsl::RegistrationPhase::Active | dsl::RegistrationPhase::Draining
3821 )
3822 }
3823
3824 fn generated_service_turn_binding_open(&self, session_id: &SessionId) -> bool {
3825 let authority = self
3826 .dsl_authority
3827 .lock()
3828 .unwrap_or_else(std::sync::PoisonError::into_inner);
3829 let state = authority.state();
3830 self.handle_teardown_gate.is_open()
3831 && state.session_id.as_ref() == Some(&dsl::SessionId::from_domain(session_id))
3832 && state.registration_phase != dsl::RegistrationPhase::Draining
3833 }
3834
3835 fn generated_stop_deferred(&self) -> bool {
3836 self.dsl_authority
3837 .lock()
3838 .unwrap_or_else(std::sync::PoisonError::into_inner)
3839 .state()
3840 .runtime_stop_deferred
3841 }
3842
3843 fn stage_generated_executor_registration_claim(
3844 &self,
3845 session_id: &SessionId,
3846 ) -> Result<StagedSessionDslInput, String> {
3847 let staged = MeerkatMachine::stage_dsl_transition_on_authority(
3848 &self.dsl_authority,
3849 dsl::MeerkatMachineInput::EnsureSessionWithExecutor {
3850 session_id: dsl::SessionId::from_domain(session_id),
3851 },
3852 "EnsureSessionWithExecutor",
3853 )?;
3854 if self.generated_executor_registration_active() {
3855 Ok(staged)
3856 } else {
3857 let mut authority = self
3858 .dsl_authority
3859 .lock()
3860 .unwrap_or_else(std::sync::PoisonError::into_inner);
3861 authority.restore_snapshot(staged.previous_snapshot);
3862 Err("generated MeerkatMachine did not grant active executor registration".into())
3863 }
3864 }
3865
3866 fn stage_generated_executor_exit_observation(&self) -> Result<StagedSessionDslInput, String> {
3867 MeerkatMachine::stage_runtime_owner_dsl_transition_on_authority(
3868 &self.dsl_authority,
3869 crate::meerkat_machine_types::MeerkatMachineFieldlessRuntimeInternalInput::RuntimeExecutorExited,
3870 )
3871 }
3872
3873 fn has_live_attachment(&self) -> bool {
3876 self.attachment_is_live()
3877 }
3878
3879 #[allow(clippy::too_many_arguments)]
3882 fn attach_runtime_loop(
3883 &mut self,
3884 id: RuntimeLoopAttachmentId,
3885 wake_tx: mpsc::Sender<()>,
3886 effect_tx: mpsc::Sender<crate::effect::RuntimeEffect>,
3887 serving_release: crate::runtime_loop::RuntimeLoopServingRelease,
3888 boundary_handle: Option<Arc<dyn meerkat_core::lifecycle::CoreExecutorBoundaryHandle>>,
3889 interrupt_handle: Option<Arc<dyn meerkat_core::lifecycle::CoreExecutorInterruptHandle>>,
3890 publication_handle: Option<Arc<dyn meerkat_core::lifecycle::CoreExecutorPublicationHandle>>,
3891 post_stop_cleanup_handle: Option<
3892 Arc<dyn meerkat_core::lifecycle::CoreExecutorPostStopCleanupHandle>,
3893 >,
3894 machine_managed_post_stop_unregister: bool,
3895 expected_materialization_claim: Option<&RuntimeExecutorAttachmentMaterializationClaim>,
3896 spawned_loop: crate::runtime_loop::SpawnedRuntimeLoop,
3897 ) -> Result<(), crate::runtime_loop::SpawnedRuntimeLoop> {
3898 if spawned_loop.startup_authority_transfer.is_some()
3899 || spawned_loop.serving_release.is_some()
3900 {
3901 return Err(spawned_loop);
3902 }
3903 let claim_changed = {
3904 let mut state = self
3905 .materialization_claim_state
3906 .lock()
3907 .unwrap_or_else(std::sync::PoisonError::into_inner);
3908 match expected_materialization_claim {
3909 Some(expected)
3910 if Arc::ptr_eq(
3911 &self.materialization_claim_state,
3912 &expected.claim_state,
3913 ) && state.exact_claim_is(
3914 expected.claim_id,
3915 &[crate::RuntimeActorMaterializationClaimPhase::ActorMaterializedPendingCommit],
3916 ) => {}
3917 Some(_) => return Err(spawned_loop),
3918 None
3919 if state.current.is_none()
3920 && matches!(
3921 state.phase,
3922 crate::RuntimeActorMaterializationClaimPhase::Vacant
3923 | crate::RuntimeActorMaterializationClaimPhase::RetainedActor
3924 ) => {}
3925 None => return Err(spawned_loop),
3926 }
3927 let Some(next_legacy_generation) = state.legacy_capability_generation.checked_add(1)
3928 else {
3929 return Err(spawned_loop);
3930 };
3931 state.legacy_capability_generation = next_legacy_generation;
3932 let claim_changed = state.current.take().is_some();
3933 state.phase = crate::RuntimeActorMaterializationClaimPhase::Vacant;
3934 state.rollback_registration_available = false;
3935 claim_changed.then(|| Arc::clone(&state.changed))
3936 };
3937 if let Some(changed) = claim_changed {
3938 changed.notify_waiters();
3939 }
3940 let crate::runtime_loop::SpawnedRuntimeLoop {
3941 loop_handle,
3942 teardown_slot,
3943 startup: _,
3944 startup_authority_transfer: _,
3945 serving_release: _,
3946 } = spawned_loop;
3947 self.provisional_interrupt_handle = None;
3948 self.provisional_materialization_claim_id = None;
3949 self.runtime_stop_cleanup_coordinator = None;
3950 self.runtime_loop_teardown = Some(Arc::clone(&teardown_slot));
3951 self.publication_handle = publication_handle;
3952 self.post_stop_cleanup_handle = post_stop_cleanup_handle;
3953 self.post_stop_cleanup_attachment_id = machine_managed_post_stop_unregister.then_some(id);
3954 self.post_stop_cleanup_complete = !machine_managed_post_stop_unregister;
3955 self.post_stop_cleanup_gate = Arc::new(Mutex::new(()));
3956 self.attachment_slot = RuntimeLoopAttachmentSlot::Pending(RuntimeLoopAttachment {
3957 id,
3958 wake_tx,
3959 effect_tx,
3960 serving_release: Some(serving_release),
3961 boundary_handle,
3962 interrupt_handle,
3963 loop_handle,
3964 });
3965 Ok(())
3966 }
3967
3968 fn owns_runtime_loop_attachment(&self, expected: RuntimeLoopAttachmentId) -> bool {
3969 matches!(
3970 &self.attachment_slot,
3971 RuntimeLoopAttachmentSlot::Pending(attachment)
3972 | RuntimeLoopAttachmentSlot::Attached(attachment)
3973 if attachment.id == expected
3974 )
3975 }
3976
3977 fn live_attachment_id(&self) -> Option<RuntimeLoopAttachmentId> {
3978 match &self.attachment_slot {
3979 RuntimeLoopAttachmentSlot::Pending(attachment)
3980 | RuntimeLoopAttachmentSlot::Attached(attachment)
3981 if !attachment.wake_tx.is_closed() && !attachment.effect_tx.is_closed() =>
3982 {
3983 Some(attachment.id)
3984 }
3985 _ => None,
3986 }
3987 }
3988
3989 fn take_runtime_loop_attachment(&mut self) -> Option<RuntimeLoopAttachment> {
3997 match std::mem::replace(&mut self.attachment_slot, RuntimeLoopAttachmentSlot::Empty) {
3998 RuntimeLoopAttachmentSlot::Pending(attachment) => Some(attachment),
3999 RuntimeLoopAttachmentSlot::Attached(attachment) => Some(attachment),
4000 RuntimeLoopAttachmentSlot::Empty => None,
4001 }
4002 }
4003
4004 fn clear_dead_attachment(&mut self) -> bool {
4005 if matches!(
4006 self.attachment_slot,
4007 RuntimeLoopAttachmentSlot::Pending(_) | RuntimeLoopAttachmentSlot::Attached(_)
4008 ) && !self.physical_attachment_is_live()
4009 {
4010 self.attachment_slot = RuntimeLoopAttachmentSlot::Empty;
4011 return true;
4012 }
4013 false
4014 }
4015
4016 fn retire_completed_runtime_stop_after_revival(
4017 &mut self,
4018 session_id: &SessionId,
4019 ) -> Result<(), RuntimeDriverError> {
4020 if !matches!(self.attachment_slot, RuntimeLoopAttachmentSlot::Empty) {
4021 return Err(RuntimeDriverError::Internal(format!(
4022 "revived session {session_id} still carries a runtime-loop attachment"
4023 )));
4024 }
4025 match self.runtime_stop_cleanup_coordinator.as_ref() {
4026 None if self.runtime_loop_teardown.is_none() => return Ok(()),
4027 None => {
4028 return Err(RuntimeDriverError::Internal(format!(
4029 "revived session {session_id} carries a teardown slot without its stop coordinator"
4030 )));
4031 }
4032 Some(coordinator) => {
4033 let coordinator_slot_is_current = match (
4034 coordinator.teardown_slot.as_ref(),
4035 self.runtime_loop_teardown.as_ref(),
4036 ) {
4037 (Some(coordinator_slot), Some(current_slot)) => {
4038 Arc::ptr_eq(coordinator_slot, current_slot)
4039 }
4040 (None, None) => true,
4041 _ => false,
4042 };
4043 if !coordinator_slot_is_current {
4044 return Err(RuntimeDriverError::Internal(format!(
4045 "revived session {session_id} carries a stale runtime-stop teardown owner"
4046 )));
4047 }
4048 match coordinator.result_rx.borrow().clone() {
4049 Some(Ok(())) => {}
4050 None => {
4051 return Err(RuntimeDriverError::RuntimeStopInProgress {
4052 runtime_id: self.runtime_id.clone(),
4053 });
4054 }
4055 Some(Err(error)) => return Err(error),
4056 }
4057 }
4058 }
4059 self.runtime_stop_cleanup_coordinator = None;
4060 self.runtime_loop_teardown = None;
4061 Ok(())
4062 }
4063
4064 fn wake_sender(&self) -> Option<mpsc::Sender<()>> {
4065 match &self.attachment_slot {
4066 RuntimeLoopAttachmentSlot::Attached(attachment)
4067 if !attachment.wake_tx.is_closed() && !attachment.effect_tx.is_closed() =>
4068 {
4069 Some(attachment.wake_tx.clone())
4070 }
4071 _ => None,
4072 }
4073 }
4074
4075 fn effect_sender(&self) -> Option<mpsc::Sender<crate::effect::RuntimeEffect>> {
4076 match &self.attachment_slot {
4077 RuntimeLoopAttachmentSlot::Attached(attachment)
4078 if !attachment.wake_tx.is_closed() && !attachment.effect_tx.is_closed() =>
4079 {
4080 Some(attachment.effect_tx.clone())
4081 }
4082 _ => None,
4083 }
4084 }
4085
4086 fn boundary_handle(
4087 &self,
4088 ) -> Option<Arc<dyn meerkat_core::lifecycle::CoreExecutorBoundaryHandle>> {
4089 match &self.attachment_slot {
4090 RuntimeLoopAttachmentSlot::Attached(attachment)
4091 if !attachment.wake_tx.is_closed() && !attachment.effect_tx.is_closed() =>
4092 {
4093 attachment.boundary_handle.clone()
4094 }
4095 _ => None,
4096 }
4097 }
4098
4099 fn interrupt_handle(
4100 &self,
4101 ) -> Option<Arc<dyn meerkat_core::lifecycle::CoreExecutorInterruptHandle>> {
4102 match &self.attachment_slot {
4103 RuntimeLoopAttachmentSlot::Attached(attachment)
4104 if !attachment.wake_tx.is_closed() && !attachment.effect_tx.is_closed() =>
4105 {
4106 attachment.interrupt_handle.clone()
4107 }
4108 _ => self.provisional_interrupt_handle.clone(),
4109 }
4110 }
4111
4112 fn publication_handle(
4113 &self,
4114 ) -> Option<Arc<dyn meerkat_core::lifecycle::CoreExecutorPublicationHandle>> {
4115 self.publication_handle.clone()
4116 }
4117
4118 fn install_provisional_interrupt_handle(
4119 &mut self,
4120 claim_id: uuid::Uuid,
4121 handle: Arc<dyn meerkat_core::lifecycle::CoreExecutorInterruptHandle>,
4122 ) {
4123 if !self.physical_attachment_is_live() {
4124 self.provisional_materialization_claim_id = Some(claim_id);
4125 self.provisional_interrupt_handle = Some(handle);
4126 }
4127 }
4128
4129 fn install_provisional_post_stop_cleanup_handle(
4130 &mut self,
4131 claim_id: uuid::Uuid,
4132 handle: Arc<dyn meerkat_core::lifecycle::CoreExecutorPostStopCleanupHandle>,
4133 ) {
4134 if !self.physical_attachment_is_live() {
4135 self.provisional_materialization_claim_id = Some(claim_id);
4136 self.post_stop_cleanup_handle = Some(handle);
4137 self.post_stop_cleanup_attachment_id = Some(RuntimeLoopAttachmentId::new());
4138 self.post_stop_cleanup_complete = false;
4139 self.post_stop_cleanup_gate = Arc::new(Mutex::new(()));
4140 }
4141 }
4142}
4143
4144impl MeerkatMachine {
4145 #[cfg(test)]
4146 pub(crate) async fn model_routing_handle_for_test(
4147 &self,
4148 session_id: &SessionId,
4149 ) -> Option<Arc<crate::handles::RuntimeModelRoutingHandle>> {
4150 let (dsl_authority, handle_teardown_gate, durability_health, visibility_owner) = {
4151 let sessions = self.sessions.read().await;
4152 let entry = sessions.get(session_id)?;
4153 (
4154 Arc::clone(&entry.dsl_authority),
4155 Arc::clone(&entry.handle_teardown_gate),
4156 entry.durability_health.clone(),
4157 Arc::clone(&entry.tool_visibility_owner),
4158 )
4159 };
4160 Some(Arc::new(
4161 crate::handles::RuntimeModelRoutingHandle::new_with_visibility_owner(
4162 Arc::new(
4163 crate::handles::HandleDslAuthority::from_shared_with_runtime_gates(
4164 dsl_authority,
4165 handle_teardown_gate,
4166 durability_health,
4167 ),
4168 ),
4169 visibility_owner,
4170 ),
4171 ))
4172 }
4173
4174 async fn session_mutation_gate(&self, session_id: &SessionId) -> Option<Arc<Mutex<()>>> {
4180 let sessions = self.sessions.read().await;
4181 sessions
4182 .get(session_id)
4183 .map(|entry| Arc::clone(&entry.mutation_gate))
4184 }
4185
4186 async fn lock_current_session_mutation_gate(
4187 &self,
4188 session_id: &SessionId,
4189 ) -> Option<crate::tokio::sync::OwnedMutexGuard<()>> {
4190 loop {
4191 let gate = self.session_mutation_gate(session_id).await?;
4192 let gate_guard = Arc::clone(&gate).lock_owned().await;
4193 let sessions = self.sessions.read().await;
4194 let entry = sessions.get(session_id)?;
4195 if Arc::ptr_eq(&entry.mutation_gate, &gate) {
4196 return Some(gate_guard);
4197 }
4198 }
4199 }
4200
4201 async fn lock_current_durability_ready_session_mutation_gate(
4210 &self,
4211 session_id: &SessionId,
4212 ) -> Result<crate::tokio::sync::OwnedMutexGuard<()>, RuntimeDriverError> {
4213 loop {
4214 let gate = self.session_mutation_gate(session_id).await.ok_or(
4215 RuntimeDriverError::NotReady {
4216 state: RuntimeState::Destroyed,
4217 },
4218 )?;
4219 let gate_guard = Arc::clone(&gate).lock_owned().await;
4220 let (reload_required, interrupt_handle, detached_attachment) = {
4221 let mut sessions = self.sessions.write().await;
4222 let entry = sessions
4223 .get_mut(session_id)
4224 .ok_or(RuntimeDriverError::NotReady {
4225 state: RuntimeState::Destroyed,
4226 })?;
4227 if !Arc::ptr_eq(&entry.mutation_gate, &gate) {
4228 continue;
4229 }
4230 match entry.require_durability_ready() {
4231 Ok(()) => return Ok(gate_guard),
4232 Err(reload_required) => {
4233 let interrupt_handle = entry.interrupt_handle();
4234 let detached_attachment = entry.take_runtime_loop_attachment();
4235 entry.provisional_interrupt_handle = None;
4236 (reload_required, interrupt_handle, detached_attachment)
4237 }
4238 }
4239 };
4240
4241 drop(detached_attachment);
4244 drop(gate_guard);
4249 if let Some(interrupt_handle) = interrupt_handle
4250 && let Err(error) = interrupt_handle
4251 .hard_cancel_current_run(format!(
4252 "durability reload required after `{}`: {}",
4253 reload_required.operation(),
4254 reload_required.reason()
4255 ))
4256 .await
4257 {
4258 tracing::warn!(
4259 %session_id,
4260 operation = reload_required.operation(),
4261 reason = reload_required.reason(),
4262 %error,
4263 "failed to hard-cancel executor after durability degradation"
4264 );
4265 }
4266 return Err(RuntimeDriverError::RecoveryRepairBlocked {
4267 evidence_digest: None,
4268 reason: reload_required.to_string(),
4269 });
4270 }
4271 }
4272
4273 #[cfg(any(test, feature = "test-support"))]
4277 #[doc(hidden)]
4278 pub fn arm_runtime_loop_before_queue_authority_test_hook(
4279 &self,
4280 session_id: SessionId,
4281 ) -> (
4282 crate::tokio::sync::oneshot::Receiver<()>,
4283 crate::tokio::sync::oneshot::Sender<()>,
4284 ) {
4285 let (entered_tx, entered_rx) = crate::tokio::sync::oneshot::channel();
4286 let (release_tx, release_rx) = crate::tokio::sync::oneshot::channel();
4287 let mut hook = self
4288 .test_runtime_loop_before_queue_authority
4289 .lock()
4290 .unwrap_or_else(std::sync::PoisonError::into_inner);
4291 assert!(
4292 hook.is_none(),
4293 "runtime-loop queue-authority test hook already armed"
4294 );
4295 *hook = Some((session_id, entered_tx, release_rx));
4296 (entered_rx, release_tx)
4297 }
4298
4299 #[cfg(any(test, feature = "test-support"))]
4300 pub(crate) async fn run_runtime_loop_before_queue_authority_test_hook(
4301 &self,
4302 session_id: &SessionId,
4303 ) {
4304 let armed = {
4305 let mut hook = self
4306 .test_runtime_loop_before_queue_authority
4307 .lock()
4308 .unwrap_or_else(std::sync::PoisonError::into_inner);
4309 if hook
4310 .as_ref()
4311 .is_some_and(|(armed_session_id, _, _)| armed_session_id == session_id)
4312 {
4313 hook.take()
4314 } else {
4315 None
4316 }
4317 };
4318 if let Some((_, entered_tx, release_rx)) = armed {
4319 let _ = entered_tx.send(());
4320 let _ = release_rx.await;
4321 }
4322 }
4323
4324 #[cfg(test)]
4325 fn arm_control_command_after_logical_lookup_test_hook(
4326 &self,
4327 kind: ControlCommandLookupTestKind,
4328 session_id: SessionId,
4329 ) -> (
4330 crate::tokio::sync::oneshot::Receiver<()>,
4331 crate::tokio::sync::oneshot::Sender<()>,
4332 ) {
4333 let (entered_tx, entered_rx) = crate::tokio::sync::oneshot::channel();
4334 let (release_tx, release_rx) = crate::tokio::sync::oneshot::channel();
4335 let mut hook = self
4336 .test_control_command_after_logical_lookup
4337 .lock()
4338 .unwrap_or_else(std::sync::PoisonError::into_inner);
4339 assert!(
4340 hook.is_none(),
4341 "control-command lookup test hook already armed"
4342 );
4343 *hook = Some((kind, session_id, entered_tx, release_rx));
4344 (entered_rx, release_tx)
4345 }
4346
4347 #[cfg(test)]
4348 async fn run_control_command_after_logical_lookup_test_hook(
4349 &self,
4350 kind: ControlCommandLookupTestKind,
4351 session_id: &SessionId,
4352 ) {
4353 let armed = {
4354 let mut hook = self
4355 .test_control_command_after_logical_lookup
4356 .lock()
4357 .unwrap_or_else(std::sync::PoisonError::into_inner);
4358 if hook
4359 .as_ref()
4360 .is_some_and(|(armed_kind, armed_session_id, _, _)| {
4361 *armed_kind == kind && armed_session_id == session_id
4362 })
4363 {
4364 hook.take()
4365 } else {
4366 None
4367 }
4368 };
4369 if let Some((_, _, entered_tx, release_rx)) = armed {
4370 let _ = entered_tx.send(());
4371 let _ = release_rx.await;
4372 }
4373 }
4374
4375 async fn lock_registration_gate(
4376 &self,
4377 session_id: &SessionId,
4378 ) -> Result<crate::tokio::sync::OwnedMutexGuard<()>, RuntimeDriverError> {
4379 let gate_guard = self
4380 .lock_current_durability_ready_session_mutation_gate(session_id)
4381 .await?;
4382 let blocked = {
4383 let sessions = self.sessions.read().await;
4384 let entry = sessions
4385 .get(session_id)
4386 .ok_or(RuntimeDriverError::NotReady {
4387 state: RuntimeState::Destroyed,
4388 })?;
4389 entry.registration_blocked_by_unregister(session_id)
4390 };
4391 if let Some(error) = blocked {
4392 return Err(error);
4393 }
4394 Ok(gate_guard)
4395 }
4396
4397 #[cfg(feature = "live")]
4398 async fn session_live_lifecycle_gate(&self, session_id: &SessionId) -> Option<Arc<Mutex<()>>> {
4399 let sessions = self.sessions.read().await;
4400 sessions
4401 .get(session_id)
4402 .map(|entry| Arc::clone(&entry.live_lifecycle_gate))
4403 }
4404
4405 #[cfg(feature = "live")]
4406 async fn lock_current_session_live_lifecycle_gate(
4407 &self,
4408 session_id: &SessionId,
4409 ) -> Option<(Arc<Mutex<()>>, crate::tokio::sync::OwnedMutexGuard<()>)> {
4410 loop {
4411 let gate = self.session_live_lifecycle_gate(session_id).await?;
4412 let guard = Arc::clone(&gate).lock_owned().await;
4413 let sessions = self.sessions.read().await;
4414 let entry = sessions.get(session_id)?;
4415 if Arc::ptr_eq(&entry.live_lifecycle_gate, &gate) {
4416 return Some((gate, guard));
4417 }
4418 }
4419 }
4420
4421 #[cfg(feature = "live")]
4422 async fn lock_session_mutation_gate_for_live_lifecycle_lease(
4423 &self,
4424 session_id: &SessionId,
4425 lease: &crate::member_live::MemberLiveLifecycleLease,
4426 ) -> Option<crate::tokio::sync::OwnedMutexGuard<()>> {
4427 let mutation_gate = {
4428 let sessions = self.sessions.read().await;
4429 let entry = sessions.get(session_id)?;
4430 if !lease.matches_gate(&entry.live_lifecycle_gate) {
4431 return None;
4432 }
4433 Arc::clone(&entry.mutation_gate)
4434 };
4435 let guard = Arc::clone(&mutation_gate).lock_owned().await;
4436 let sessions = self.sessions.read().await;
4437 let entry = sessions.get(session_id)?;
4438 if lease.matches_gate(&entry.live_lifecycle_gate)
4439 && Arc::ptr_eq(&entry.mutation_gate, &mutation_gate)
4440 {
4441 Some(guard)
4442 } else {
4443 None
4444 }
4445 }
4446
4447 pub(crate) async fn lock_current_session_driver_gate(
4448 &self,
4449 session_id: &SessionId,
4450 driver: &SharedDriver,
4451 ) -> Result<crate::tokio::sync::OwnedMutexGuard<()>, RuntimeDriverError> {
4452 let gate_guard = self
4453 .lock_current_durability_ready_session_mutation_gate(session_id)
4454 .await?;
4455 {
4456 let sessions = self.sessions.read().await;
4457 let entry = sessions
4458 .get(session_id)
4459 .ok_or(RuntimeDriverError::NotReady {
4460 state: RuntimeState::Destroyed,
4461 })?;
4462 if !Arc::ptr_eq(&entry.driver, driver) {
4463 return Err(RuntimeDriverError::NotReady {
4464 state: RuntimeState::Destroyed,
4465 });
4466 }
4467 }
4468 Ok(gate_guard)
4469 }
4470
4471 pub(crate) async fn lock_current_runtime_loop_driver_authority(
4472 &self,
4473 session_id: &SessionId,
4474 driver: &SharedDriver,
4475 ) -> Result<crate::tokio::sync::OwnedMutexGuard<()>, RuntimeDriverError> {
4476 let gate_guard = self
4477 .lock_current_session_driver_gate(session_id, driver)
4478 .await?;
4479 {
4480 let sessions = self.sessions.read().await;
4481 let entry = sessions
4482 .get(session_id)
4483 .ok_or(RuntimeDriverError::NotReady {
4484 state: RuntimeState::Destroyed,
4485 })?;
4486 if !entry.generated_executor_registration_active_or_draining() {
4487 return Err(RuntimeDriverError::ValidationFailed {
4488 reason:
4489 "generated MeerkatMachine has no active runtime-loop executor registration"
4490 .to_string(),
4491 });
4492 }
4493 }
4494 Ok(gate_guard)
4495 }
4496
4497 pub(crate) async fn current_runtime_stop_cleanup_in_progress(
4498 &self,
4499 session_id: &SessionId,
4500 driver: &SharedDriver,
4501 ) -> Result<bool, RuntimeDriverError> {
4502 let sessions = self.sessions.read().await;
4503 let entry = sessions
4504 .get(session_id)
4505 .ok_or(RuntimeDriverError::NotReady {
4506 state: RuntimeState::Destroyed,
4507 })?;
4508 if !Arc::ptr_eq(&entry.driver, driver) {
4509 return Err(RuntimeDriverError::NotReady {
4510 state: RuntimeState::Destroyed,
4511 });
4512 }
4513 let Some(coordinator) = entry.runtime_stop_cleanup_coordinator.as_ref() else {
4514 return Ok(false);
4515 };
4516 if coordinator.epoch_id != entry.epoch_id {
4517 return Err(RuntimeDriverError::Internal(format!(
4518 "stale runtime-stop cleanup coordinator epoch for session {session_id}"
4519 )));
4520 }
4521 Ok(coordinator.result_rx.borrow().is_none())
4522 }
4523
4524 async fn session_dsl_authority(
4525 &self,
4526 session_id: &SessionId,
4527 ) -> Result<Arc<std::sync::Mutex<dsl::MeerkatMachineAuthority>>, String> {
4528 let sessions = self.sessions.read().await;
4529 sessions
4530 .get(session_id)
4531 .map(|entry| Arc::clone(&entry.dsl_authority))
4532 .ok_or_else(|| {
4533 RuntimeDriverError::NotReady {
4534 state: RuntimeState::Destroyed,
4535 }
4536 .to_string()
4537 })
4538 }
4539
4540 #[cfg(any(test, feature = "test-support"))]
4541 async fn session_handle_teardown_gate(
4542 &self,
4543 session_id: &SessionId,
4544 ) -> Result<Arc<crate::handles::HandleTeardownGate>, String> {
4545 let sessions = self.sessions.read().await;
4546 sessions
4547 .get(session_id)
4548 .map(|entry| Arc::clone(&entry.handle_teardown_gate))
4549 .ok_or_else(|| {
4550 RuntimeDriverError::NotReady {
4551 state: RuntimeState::Destroyed,
4552 }
4553 .to_string()
4554 })
4555 }
4556
4557 #[cfg(any(test, feature = "test-support"))]
4558 async fn session_durability_health(
4559 &self,
4560 session_id: &SessionId,
4561 ) -> Result<Option<DurabilityHealthHandle>, String> {
4562 let sessions = self.sessions.read().await;
4563 sessions
4564 .get(session_id)
4565 .map(|entry| entry.durability_health.clone())
4566 .ok_or_else(|| {
4567 RuntimeDriverError::NotReady {
4568 state: RuntimeState::Destroyed,
4569 }
4570 .to_string()
4571 })
4572 }
4573
4574 #[cfg(any(test, feature = "test-support"))]
4582 pub async fn test_install_session_peer_comms_handle_on_runtime(
4583 &self,
4584 session_id: &SessionId,
4585 runtime: &(dyn meerkat_core::handles::PeerCommsInstallTarget + '_),
4586 ) -> Result<(), String> {
4587 let dsl = self
4588 .session_dsl_authority(session_id)
4589 .await
4590 .map_err(|error| format!("session dsl authority unavailable: {error}"))?;
4591 let teardown_gate = self
4592 .session_handle_teardown_gate(session_id)
4593 .await
4594 .map_err(|error| format!("session handle teardown gate unavailable: {error}"))?;
4595 let durability_health = self
4596 .session_durability_health(session_id)
4597 .await
4598 .map_err(|error| format!("session durability health unavailable: {error}"))?;
4599 let handle = std::sync::Arc::new(
4600 crate::handles::HandleDslAuthority::from_shared_with_runtime_gates(
4601 dsl,
4602 teardown_gate,
4603 durability_health,
4604 ),
4605 );
4606 crate::handles::RuntimePeerCommsHandle::install_generated_on(handle, runtime)
4607 }
4608
4609 fn preview_dsl_input_on_state(
4610 state: &dsl::MeerkatMachineState,
4611 input: dsl::MeerkatMachineInput,
4612 context: &str,
4613 ) -> Result<Vec<dsl::MeerkatMachineEffect>, String> {
4614 let mut preview = dsl::MeerkatMachineAuthority::recover_from_state(state.clone())
4615 .map_err(|err| dsl_authority::map_error(err, context))?;
4616 dsl::MeerkatMachineMutator::apply(&mut preview, input)
4617 .map(|transition| transition.into_effects())
4618 .map_err(|err| dsl_authority::map_error(err, context))
4619 }
4620
4621 async fn preview_session_dsl_input(
4622 &self,
4623 session_id: &SessionId,
4624 input: dsl::MeerkatMachineInput,
4625 context: &str,
4626 ) -> Result<Vec<dsl::MeerkatMachineEffect>, String> {
4627 let authority = self.session_dsl_authority(session_id).await?;
4628 let state = {
4629 let authority = authority
4630 .lock()
4631 .unwrap_or_else(std::sync::PoisonError::into_inner);
4632 authority.state().clone()
4633 };
4634 Self::preview_dsl_input_on_state(&state, input, context)
4635 }
4636
4637 async fn session_dsl_state(
4638 &self,
4639 session_id: &SessionId,
4640 ) -> Result<dsl::MeerkatMachineState, RuntimeControlPlaneError> {
4641 let authority = self
4642 .session_dsl_authority(session_id)
4643 .await
4644 .map_err(RuntimeControlPlaneError::Internal)?;
4645 let authority = authority
4646 .lock()
4647 .unwrap_or_else(std::sync::PoisonError::into_inner);
4648 Ok(authority.state().clone())
4649 }
4650
4651 async fn commit_session_dsl_transition(
4652 &self,
4653 session_id: &SessionId,
4654 staged: StagedSessionDslInput,
4655 context: &str,
4656 ) -> Result<(), String> {
4657 self.commit_session_dsl_transition_with_dispatch_failure(
4658 session_id,
4659 staged,
4660 context,
4661 CommittedEffectDispatchFailure::PreserveCommittedDslState,
4662 )
4663 .await
4664 }
4665
4666 async fn commit_session_dsl_transition_preserving_committed_state(
4667 &self,
4668 session_id: &SessionId,
4669 staged: StagedSessionDslInput,
4670 context: &str,
4671 ) -> Result<(), String> {
4672 self.commit_session_dsl_transition_with_dispatch_failure(
4673 session_id,
4674 staged,
4675 context,
4676 CommittedEffectDispatchFailure::PreserveCommittedDslState,
4677 )
4678 .await
4679 }
4680
4681 async fn commit_session_dsl_transition_with_dispatch_failure(
4682 &self,
4683 _session_id: &SessionId,
4684 staged: StagedSessionDslInput,
4685 context: &str,
4686 dispatch_failure: CommittedEffectDispatchFailure,
4687 ) -> Result<(), String> {
4688 if let Err(error) = self
4689 .dispatch_routed_signals_from_effects(&staged.effects)
4690 .await
4691 {
4692 let CommittedEffectDispatchFailure::PreserveCommittedDslState = dispatch_failure;
4693 return Err(format!(
4694 "DSL authority ({context}): committed effect dispatch failed: {error}"
4695 ));
4696 }
4697 Ok(())
4698 }
4699
4700 async fn dispatch_routed_signals_from_effects(
4701 &self,
4702 effects: &[dsl::MeerkatMachineEffect],
4703 ) -> Result<(), String> {
4704 let dispatcher = {
4705 self.composition_signal_dispatcher
4706 .read()
4707 .unwrap_or_else(std::sync::PoisonError::into_inner)
4708 .clone()
4709 };
4710 let Some(dispatcher) = dispatcher else {
4711 return Ok(());
4712 };
4713
4714 for effect in effects {
4715 if let Some(signal) = composition::lift_routed_signal(effect) {
4716 composition::dispatch_routed_signal(&dispatcher, signal).await?;
4717 }
4718 }
4719 Ok(())
4720 }
4721
4722 #[allow(clippy::too_many_arguments)]
4725 async fn dispatch_cancel_after_boundary_runtime_effect(
4726 &self,
4727 session_id: &SessionId,
4728 witness: RuntimeEffectDispatchAttachmentWitness,
4729 held_mutation_gate: crate::tokio::sync::OwnedMutexGuard<()>,
4730 boundary_handle: Option<Arc<dyn meerkat_core::lifecycle::CoreExecutorBoundaryHandle>>,
4731 member_authority: Option<RuntimeEffectDispatchMemberAuthority>,
4732 pending_dispatch: PendingBoundaryCancelDispatchGuard,
4733 expected_run_id: Option<&RunId>,
4734 projected_effect: crate::effect::ProjectedRuntimeEffect,
4735 dispatch_generation: u64,
4736 dispatch_lifecycle_phase: dsl::MeerkatPhase,
4737 context: &str,
4738 ) -> Result<crate::tokio::sync::OwnedMutexGuard<()>, RuntimeDriverError> {
4739 let cleanup_spawner = MachineCleanupTaskSpawner::acquire()?;
4740 let machine = self.clone();
4741 let session_id = session_id.clone();
4742 let expected_run_id = expected_run_id.cloned();
4743 let context = context.to_string();
4744 let completion = cleanup_spawner.spawn(async move {
4745 machine
4746 .dispatch_cancel_after_boundary_runtime_effect_owned(
4747 &session_id,
4748 witness,
4749 held_mutation_gate,
4750 boundary_handle,
4751 member_authority,
4752 pending_dispatch,
4753 expected_run_id.as_ref(),
4754 projected_effect,
4755 dispatch_generation,
4756 dispatch_lifecycle_phase,
4757 &context,
4758 )
4759 .await
4760 });
4761 completion.await.map_err(|error| {
4762 RuntimeDriverError::Internal(format!(
4763 "process-owned boundary-effect dispatch ended without a result: {error}"
4764 ))
4765 })?
4766 }
4767
4768 #[allow(clippy::too_many_arguments)]
4771 async fn dispatch_cancel_after_boundary_runtime_effect_owned(
4772 &self,
4773 session_id: &SessionId,
4774 witness: RuntimeEffectDispatchAttachmentWitness,
4775 held_mutation_gate: crate::tokio::sync::OwnedMutexGuard<()>,
4776 boundary_handle: Option<Arc<dyn meerkat_core::lifecycle::CoreExecutorBoundaryHandle>>,
4777 member_authority: Option<RuntimeEffectDispatchMemberAuthority>,
4778 mut pending_dispatch: PendingBoundaryCancelDispatchGuard,
4779 expected_run_id: Option<&RunId>,
4780 projected_effect: crate::effect::ProjectedRuntimeEffect,
4781 dispatch_generation: u64,
4782 dispatch_lifecycle_phase: dsl::MeerkatPhase,
4783 context: &str,
4784 ) -> Result<crate::tokio::sync::OwnedMutexGuard<()>, RuntimeDriverError> {
4785 drop(held_mutation_gate);
4790 let live_dispatch_result = self
4791 .dispatch_cancel_after_boundary_live_handle(
4792 boundary_handle,
4793 expected_run_id,
4794 &projected_effect,
4795 context,
4796 )
4797 .await;
4798
4799 let pre_reserve_state = Self::classify_dsl_boundary_cancel_dispatch(
4803 &witness.dsl_authority,
4804 dispatch_generation,
4805 &dispatch_lifecycle_phase,
4806 expected_run_id,
4807 );
4808 let effect_permit = if live_dispatch_result.is_ok()
4809 && pre_reserve_state == BoundaryCancelDispatchState::Current
4810 {
4811 Some(witness.effect_tx.clone().reserve_owned().await)
4812 } else {
4813 None
4814 };
4815
4816 let gate_guard = Arc::clone(&witness.mutation_gate).lock_owned().await;
4817 let current_effect_tx = {
4818 let sessions = self.sessions.read().await;
4819 match sessions.get(session_id) {
4820 None => Err(RuntimeDriverError::StaleAuthority {
4821 reason: format!(
4822 "{context}: runtime session disappeared during boundary callback"
4823 ),
4824 }),
4825 Some(entry)
4826 if !Arc::ptr_eq(&entry.mutation_gate, &witness.mutation_gate)
4827 || !Arc::ptr_eq(&entry.driver, &witness.driver)
4828 || !Arc::ptr_eq(&entry.dsl_authority, &witness.dsl_authority)
4829 || !entry.owns_runtime_loop_attachment(witness.attachment_id) =>
4830 {
4831 Err(RuntimeDriverError::StaleAuthority {
4832 reason: format!(
4833 "{context}: runtime attachment changed during boundary callback"
4834 ),
4835 })
4836 }
4837 Some(entry) => {
4838 entry.require_durability_ready().map_err(|required| {
4839 RuntimeDriverError::RecoveryRepairBlocked {
4840 evidence_digest: None,
4841 reason: required.to_string(),
4842 }
4843 })?;
4844 match entry.effect_sender() {
4845 None => Err(RuntimeDriverError::StaleAuthority {
4846 reason: format!(
4847 "{context}: runtime effect channel disappeared during boundary callback"
4848 ),
4849 }),
4850 Some(current_effect_tx)
4851 if !witness.effect_tx.same_channel(¤t_effect_tx) =>
4852 {
4853 Err(RuntimeDriverError::StaleAuthority {
4854 reason: format!(
4855 "{context}: runtime effect channel changed during boundary callback"
4856 ),
4857 })
4858 }
4859 Some(current_effect_tx) => Ok(current_effect_tx),
4860 }
4861 }
4862 }
4863 };
4864 let current_effect_tx = current_effect_tx?;
4865
4866 if let Some(member_authority) = &member_authority {
4867 self.validate_member_effect_authority_lease_current(
4868 session_id,
4869 &member_authority.lease,
4870 member_authority.expected_member.as_ref(),
4871 )?;
4872 }
4873
4874 if let Err(error) = live_dispatch_result {
4880 return Err(error);
4881 }
4882
4883 match Self::classify_dsl_boundary_cancel_dispatch(
4888 &witness.dsl_authority,
4889 dispatch_generation,
4890 &dispatch_lifecycle_phase,
4891 expected_run_id,
4892 ) {
4893 BoundaryCancelDispatchState::ConsumedConverged => return Ok(gate_guard),
4894 BoundaryCancelDispatchState::SameGenerationOrphan
4895 | BoundaryCancelDispatchState::Superseded => {
4896 return Err(RuntimeDriverError::StaleAuthority {
4897 reason: format!(
4898 "{context}: boundary-cancel dispatch authority changed during callback"
4899 ),
4900 });
4901 }
4902 BoundaryCancelDispatchState::Current => {}
4903 }
4904
4905 let state = self
4906 .existing_session_runtime_state(session_id)
4907 .await
4908 .unwrap_or(RuntimeState::Destroyed);
4909 self.reject_unregistration_drain_ingress(session_id, state)
4910 .await?;
4911
4912 debug_assert!(witness.effect_tx.same_channel(¤t_effect_tx));
4913 let effect_permit = effect_permit
4914 .ok_or_else(|| RuntimeDriverError::StaleAuthority {
4915 reason: format!(
4916 "{context}: boundary-cancel dispatch became current after capacity reservation was skipped"
4917 ),
4918 })?
4919 .map_err(|_| RuntimeDriverError::NotReady {
4920 state: RuntimeState::Idle,
4921 })?;
4922 let _effect_tx = effect_permit.send(projected_effect.into_effect());
4923 pending_dispatch.disarm();
4924 Ok(gate_guard)
4925 }
4926
4927 async fn dispatch_accepted_ingress_boundary_work(
4932 &self,
4933 session_id: &SessionId,
4934 held_mutation_gate: crate::tokio::sync::OwnedMutexGuard<()>,
4935 cancel_plan: Option<RuntimeAcceptedBoundaryCancelPlan>,
4936 _completions: crate::meerkat_machine::driver::SharedCompletionRegistry,
4937 wake_tx: Option<mpsc::Sender<()>>,
4938 should_wake: bool,
4939 ) -> Result<crate::tokio::sync::OwnedMutexGuard<()>, RuntimeDriverError> {
4940 let cleanup_spawner = MachineCleanupTaskSpawner::acquire()?;
4941 let machine = self.clone();
4942 let session_id = session_id.clone();
4943 let completion = cleanup_spawner.spawn(async move {
4944 let mut gate_guard = held_mutation_gate;
4945 if let Some(cancel_plan) = cancel_plan {
4946 gate_guard = machine
4947 .dispatch_cancel_after_boundary_runtime_effect_owned(
4948 &session_id,
4949 cancel_plan.witness,
4950 gate_guard,
4951 cancel_plan.boundary_handle,
4952 None,
4953 cancel_plan.pending_dispatch,
4954 Some(&cancel_plan.expected_run_id),
4955 cancel_plan.projected_effect,
4956 cancel_plan.dispatch_generation,
4957 cancel_plan.dispatch_lifecycle_phase,
4958 "AcceptWithCompletion",
4959 )
4960 .await?;
4961 }
4962 if should_wake && let Some(wake_tx) = wake_tx {
4963 let _ = wake_tx.try_send(());
4964 }
4965 Ok(gate_guard)
4966 });
4967 completion.await.map_err(|error| {
4968 RuntimeDriverError::Internal(format!(
4969 "process-owned accepted-ingress boundary transaction ended without a result: {error}"
4970 ))
4971 })?
4972 }
4973
4974 async fn dispatch_cancel_after_boundary_live_handle(
4980 &self,
4981 boundary_handle: Option<Arc<dyn meerkat_core::lifecycle::CoreExecutorBoundaryHandle>>,
4982 expected_run_id: Option<&RunId>,
4983 projected_effect: &crate::effect::ProjectedRuntimeEffect,
4984 context: &str,
4985 ) -> Result<(), RuntimeDriverError> {
4986 let reason = projected_effect.reason().to_string();
4987 if let (Some(boundary_handle), Some(expected_run_id)) = (boundary_handle, expected_run_id) {
4992 boundary_handle
4993 .cancel_after_boundary(expected_run_id, reason)
4994 .await
4995 .map_err(|err| {
4996 RuntimeDriverError::Internal(format!(
4997 "{context}: failed to apply live boundary cancel: {err}"
4998 ))
4999 })?;
5000 }
5001 Ok(())
5002 }
5003
5004 async fn restore_session_dsl_state(
5005 &self,
5006 session_id: &SessionId,
5007 snapshot: dsl::MeerkatMachineAuthoritySnapshot,
5008 ) {
5009 if let Ok(authority) = self.session_dsl_authority(session_id).await {
5010 Self::restore_dsl_authority_snapshot(&authority, snapshot);
5011 }
5012 }
5013
5014 async fn restore_session_dsl_state_if_current(
5015 &self,
5016 session_id: &SessionId,
5017 expected_current: dsl::MeerkatMachineAuthoritySnapshot,
5018 restore: dsl::MeerkatMachineAuthoritySnapshot,
5019 ) -> bool {
5020 let Ok(authority) = self.session_dsl_authority(session_id).await else {
5021 return false;
5022 };
5023 Self::restore_dsl_authority_snapshot_if_current(&authority, expected_current, restore)
5024 }
5025
5026 fn classify_dsl_boundary_cancel_dispatch(
5027 authority: &Arc<std::sync::Mutex<dsl::MeerkatMachineAuthority>>,
5028 dispatch_generation: u64,
5029 dispatch_lifecycle_phase: &dsl::MeerkatPhase,
5030 expected_run_id: Option<&RunId>,
5031 ) -> BoundaryCancelDispatchState {
5032 let authority = authority
5033 .lock()
5034 .unwrap_or_else(std::sync::PoisonError::into_inner);
5035 let state = authority.state();
5036 if state.boundary_cancel_dispatch_generation != dispatch_generation {
5037 return BoundaryCancelDispatchState::Superseded;
5038 }
5039 if !state.boundary_cancel_dispatch_pending {
5040 return BoundaryCancelDispatchState::ConsumedConverged;
5041 }
5042 let expected_dsl_run_id = expected_run_id.map(dsl::RunId::from_domain);
5043 if state.lifecycle_phase == *dispatch_lifecycle_phase
5044 && state.current_run_id.as_ref() == expected_dsl_run_id.as_ref()
5045 {
5046 BoundaryCancelDispatchState::Current
5047 } else {
5048 BoundaryCancelDispatchState::SameGenerationOrphan
5049 }
5050 }
5051
5052 fn abort_dsl_boundary_cancel_dispatch_if_current(
5059 authority: &Arc<std::sync::Mutex<dsl::MeerkatMachineAuthority>>,
5060 dispatch_generation: u64,
5061 ) -> Result<bool, RuntimeDriverError> {
5062 let mut authority = authority
5063 .lock()
5064 .unwrap_or_else(std::sync::PoisonError::into_inner);
5065 if !authority.state().boundary_cancel_dispatch_pending
5066 || authority.state().boundary_cancel_dispatch_generation != dispatch_generation
5067 {
5068 return Ok(false);
5069 }
5070 dsl::MeerkatMachineMutator::apply(
5071 &mut *authority,
5072 dsl::MeerkatMachineInput::AbortCancelAfterBoundaryDispatch {
5073 dispatch_generation,
5074 },
5075 )
5076 .map(|_| true)
5077 .map_err(|err| {
5078 RuntimeDriverError::Internal(dsl_authority::map_error(
5079 err,
5080 "AbortCancelAfterBoundaryDispatch",
5081 ))
5082 })
5083 }
5084
5085 fn restore_dsl_authority_snapshot(
5086 authority: &Arc<std::sync::Mutex<dsl::MeerkatMachineAuthority>>,
5087 snapshot: dsl::MeerkatMachineAuthoritySnapshot,
5088 ) {
5089 let mut authority = authority
5090 .lock()
5091 .unwrap_or_else(std::sync::PoisonError::into_inner);
5092 authority.restore_snapshot(snapshot);
5093 }
5094
5095 fn restore_dsl_authority_snapshot_if_current(
5096 authority: &Arc<std::sync::Mutex<dsl::MeerkatMachineAuthority>>,
5097 expected_current: dsl::MeerkatMachineAuthoritySnapshot,
5098 restore: dsl::MeerkatMachineAuthoritySnapshot,
5099 ) -> bool {
5100 let mut authority = authority
5101 .lock()
5102 .unwrap_or_else(std::sync::PoisonError::into_inner);
5103 let current = authority.snapshot();
5104 if current.state() == expected_current.state() {
5105 authority.restore_snapshot(restore);
5106 true
5107 } else {
5108 false
5109 }
5110 }
5111}
5112
5113#[derive(Debug, Clone, Copy)]
5116pub struct MachineSessionControlAuthority {
5117 _private: (),
5118}
5119
5120#[cfg(feature = "live")]
5121struct LiveOpenAdmissionGeneratedAuthorityBridgeToken;
5122
5123#[cfg(feature = "live")]
5124struct LiveCloseResultGeneratedAuthorityBridgeToken;
5125
5126#[cfg(feature = "live")]
5127struct LiveChannelStatusResultGeneratedAuthorityBridgeToken;
5128
5129#[cfg(feature = "live")]
5130static LIVE_OPEN_ADMISSION_GENERATED_AUTHORITY_BRIDGE_TOKEN:
5131 LiveOpenAdmissionGeneratedAuthorityBridgeToken = LiveOpenAdmissionGeneratedAuthorityBridgeToken;
5132
5133#[cfg(feature = "live")]
5134static LIVE_CLOSE_RESULT_GENERATED_AUTHORITY_BRIDGE_TOKEN:
5135 LiveCloseResultGeneratedAuthorityBridgeToken = LiveCloseResultGeneratedAuthorityBridgeToken;
5136
5137#[cfg(feature = "live")]
5138static LIVE_CHANNEL_STATUS_RESULT_GENERATED_AUTHORITY_BRIDGE_TOKEN:
5139 LiveChannelStatusResultGeneratedAuthorityBridgeToken =
5140 LiveChannelStatusResultGeneratedAuthorityBridgeToken;
5141
5142#[cfg(feature = "live")]
5143fn live_open_admission_generated_authority_bridge_token()
5144-> &'static (dyn std::any::Any + Send + Sync) {
5145 &LIVE_OPEN_ADMISSION_GENERATED_AUTHORITY_BRIDGE_TOKEN
5146}
5147
5148#[cfg(feature = "live")]
5149fn live_close_result_generated_authority_bridge_token() -> &'static (dyn std::any::Any + Send + Sync)
5150{
5151 &LIVE_CLOSE_RESULT_GENERATED_AUTHORITY_BRIDGE_TOKEN
5152}
5153
5154#[cfg(feature = "live")]
5155fn live_channel_status_result_generated_authority_bridge_token()
5156-> &'static (dyn std::any::Any + Send + Sync) {
5157 &LIVE_CHANNEL_STATUS_RESULT_GENERATED_AUTHORITY_BRIDGE_TOKEN
5158}
5159
5160#[cfg(feature = "live")]
5161#[doc(hidden)]
5162#[allow(improper_ctypes_definitions, unsafe_code)]
5163#[unsafe(export_name = concat!(
5164 "__meerkat_runtime_generated_authority_bridge_token_is_valid_v1_live_open_admission_",
5165 env!("MEERKAT_GENERATED_AUTHORITY_BRIDGE_SYMBOL_SUFFIX")
5166))]
5167pub extern "Rust" fn live_open_admission_generated_authority_bridge_token_is_valid(
5168 token: &(dyn std::any::Any + Send + Sync),
5169) -> bool {
5170 token.is::<LiveOpenAdmissionGeneratedAuthorityBridgeToken>()
5171}
5172
5173#[cfg(feature = "live")]
5174#[doc(hidden)]
5175#[allow(improper_ctypes_definitions, unsafe_code)]
5176#[unsafe(export_name = concat!(
5177 "__meerkat_runtime_generated_authority_bridge_token_is_valid_v1_live_close_result_",
5178 env!("MEERKAT_GENERATED_AUTHORITY_BRIDGE_SYMBOL_SUFFIX")
5179))]
5180pub extern "Rust" fn live_close_result_generated_authority_bridge_token_is_valid(
5181 token: &(dyn std::any::Any + Send + Sync),
5182) -> bool {
5183 token.is::<LiveCloseResultGeneratedAuthorityBridgeToken>()
5184}
5185
5186#[cfg(feature = "live")]
5187#[doc(hidden)]
5188#[allow(improper_ctypes_definitions, unsafe_code)]
5189#[unsafe(export_name = concat!(
5190 "__meerkat_runtime_generated_authority_bridge_token_is_valid_v1_live_channel_status_result_",
5191 env!("MEERKAT_GENERATED_AUTHORITY_BRIDGE_SYMBOL_SUFFIX")
5192))]
5193pub extern "Rust" fn live_channel_status_result_generated_authority_bridge_token_is_valid(
5194 token: &(dyn std::any::Any + Send + Sync),
5195) -> bool {
5196 token.is::<LiveChannelStatusResultGeneratedAuthorityBridgeToken>()
5197}
5198
5199#[cfg(feature = "live")]
5200fn build_live_channel_open_authority(
5201 session_id: SessionId,
5202 channel_id: meerkat_live::LiveChannelId,
5203 sequence: u64,
5204) -> Result<meerkat_live::LiveChannelOpenAuthority, String> {
5205 #[allow(improper_ctypes_definitions, unsafe_code)]
5206 unsafe extern "Rust" {
5207 #[link_name = concat!(
5208 "__meerkat_live_runtime_generated_live_channel_open_authority_build_v1_",
5209 env!("MEERKAT_GENERATED_AUTHORITY_BRIDGE_SYMBOL_SUFFIX")
5210 )]
5211 fn live_generated_channel_open_authority_build(
5212 token: &'static (dyn std::any::Any + Send + Sync),
5213 session_id: SessionId,
5214 channel_id: meerkat_live::LiveChannelId,
5215 sequence: u64,
5216 ) -> Result<meerkat_live::LiveChannelOpenAuthority, String>;
5217 }
5218 #[allow(unsafe_code)]
5219 unsafe {
5220 live_generated_channel_open_authority_build(
5221 live_open_admission_generated_authority_bridge_token(),
5222 session_id,
5223 channel_id,
5224 sequence,
5225 )
5226 }
5227}
5228
5229#[cfg(feature = "live")]
5230fn build_live_channel_close_commit_authority(
5231 channel_id: String,
5232 close_sequence: u64,
5233) -> Result<meerkat_live::LiveChannelCloseCommitAuthority, String> {
5234 #[allow(improper_ctypes_definitions, unsafe_code)]
5235 unsafe extern "Rust" {
5236 #[link_name = concat!(
5237 "__meerkat_live_runtime_generated_live_channel_close_commit_authority_build_v1_",
5238 env!("MEERKAT_GENERATED_AUTHORITY_BRIDGE_SYMBOL_SUFFIX")
5239 )]
5240 fn live_generated_channel_close_commit_authority_build(
5241 token: &'static (dyn std::any::Any + Send + Sync),
5242 channel_id: String,
5243 close_sequence: u64,
5244 ) -> Result<meerkat_live::LiveChannelCloseCommitAuthority, String>;
5245 }
5246 #[allow(unsafe_code)]
5247 unsafe {
5248 live_generated_channel_close_commit_authority_build(
5249 live_close_result_generated_authority_bridge_token(),
5250 channel_id,
5251 close_sequence,
5252 )
5253 }
5254}
5255
5256#[cfg(feature = "live")]
5257fn build_live_channel_status_commit_authority(
5258 channel_id: String,
5259 status_observation_sequence: u64,
5260) -> Result<meerkat_live::LiveChannelStatusCommitAuthority, String> {
5261 #[allow(improper_ctypes_definitions, unsafe_code)]
5262 unsafe extern "Rust" {
5263 #[link_name = concat!(
5264 "__meerkat_live_runtime_generated_live_channel_status_commit_authority_build_v1_",
5265 env!("MEERKAT_GENERATED_AUTHORITY_BRIDGE_SYMBOL_SUFFIX")
5266 )]
5267 fn live_generated_channel_status_commit_authority_build(
5268 token: &'static (dyn std::any::Any + Send + Sync),
5269 channel_id: String,
5270 status_observation_sequence: u64,
5271 ) -> Result<meerkat_live::LiveChannelStatusCommitAuthority, String>;
5272 }
5273 #[allow(unsafe_code)]
5274 unsafe {
5275 live_generated_channel_status_commit_authority_build(
5276 live_channel_status_result_generated_authority_bridge_token(),
5277 channel_id,
5278 status_observation_sequence,
5279 )
5280 }
5281}
5282
5283#[derive(Debug, Clone)]
5290#[cfg(feature = "live")]
5291pub struct LiveOpenAdmissionAuthority {
5292 session_id: SessionId,
5293 channel_id: meerkat_live::LiveChannelId,
5294 admitted: bool,
5295 rejection: Option<dsl::LiveOpenAdmissionRejection>,
5296 bound_llm_identity: Option<meerkat_core::SessionLlmIdentity>,
5297 sequence: u64,
5298 channel_open_authority: Option<meerkat_live::LiveChannelOpenAuthority>,
5299}
5300
5301#[cfg(feature = "live")]
5302impl LiveOpenAdmissionAuthority {
5303 pub(crate) fn from_generated_effect(
5304 session_id: SessionId,
5305 channel_id: meerkat_live::LiveChannelId,
5306 admitted: bool,
5307 rejection: Option<dsl::LiveOpenAdmissionRejection>,
5308 bound_llm_identity: Option<dsl::SessionLlmIdentity>,
5309 sequence: u64,
5310 ) -> Result<Self, String> {
5311 let bound_llm_identity = match (admitted, bound_llm_identity) {
5312 (true, Some(identity)) => Some(identity.try_into()?),
5313 (true, None) => {
5314 return Err(
5315 "generated live-open admission was admitted without bound LLM identity"
5316 .to_string(),
5317 );
5318 }
5319 (false, _) => None,
5320 };
5321 let channel_open_authority = if admitted {
5322 Some(build_live_channel_open_authority(
5323 session_id.clone(),
5324 channel_id.clone(),
5325 sequence,
5326 )?)
5327 } else {
5328 None
5329 };
5330 Ok(Self {
5331 session_id,
5332 channel_id,
5333 admitted,
5334 rejection,
5335 bound_llm_identity,
5336 sequence,
5337 channel_open_authority,
5338 })
5339 }
5340
5341 #[must_use]
5342 pub fn session_id(&self) -> &SessionId {
5343 &self.session_id
5344 }
5345
5346 #[must_use]
5347 pub fn channel_id(&self) -> &meerkat_live::LiveChannelId {
5348 &self.channel_id
5349 }
5350
5351 #[must_use]
5352 pub fn admitted(&self) -> bool {
5353 self.admitted
5354 }
5355
5356 #[must_use]
5357 pub fn rejection(&self) -> Option<dsl::LiveOpenAdmissionRejection> {
5358 self.rejection
5359 }
5360
5361 #[must_use]
5362 pub fn bound_llm_identity(&self) -> Option<&meerkat_core::SessionLlmIdentity> {
5363 self.bound_llm_identity.as_ref()
5364 }
5365
5366 #[must_use]
5367 pub fn sequence(&self) -> u64 {
5368 self.sequence
5369 }
5370
5371 #[must_use]
5372 pub fn channel_open_authority(&self) -> Option<&meerkat_live::LiveChannelOpenAuthority> {
5373 self.channel_open_authority.as_ref()
5374 }
5375}
5376
5377#[derive(Debug, Clone, PartialEq, Eq)]
5384#[cfg(feature = "live")]
5385pub struct LiveRefreshResultAuthority {
5386 pub status: dsl::LiveRefreshPublicStatus,
5387 pub sequence: u64,
5388 pub queue_acceptance_sequence: u64,
5389}
5390
5391#[derive(Debug, Clone)]
5396#[cfg(feature = "live")]
5397pub struct LiveCloseResultAuthority {
5398 pub status: dsl::LiveClosePublicStatus,
5399 pub sequence: u64,
5400 pub close_observation_sequence: u64,
5401 channel_close_commit_authority: Option<meerkat_live::LiveChannelCloseCommitAuthority>,
5402}
5403
5404#[cfg(feature = "live")]
5405impl LiveCloseResultAuthority {
5406 pub(crate) fn from_generated_effect(
5407 channel_id: String,
5408 status: dsl::LiveClosePublicStatus,
5409 sequence: u64,
5410 close_observation_sequence: u64,
5411 ) -> Result<Self, String> {
5412 let channel_close_commit_authority = match status {
5413 dsl::LiveClosePublicStatus::Closed => Some(build_live_channel_close_commit_authority(
5414 channel_id,
5415 close_observation_sequence,
5416 )?),
5417 };
5418 Ok(Self {
5419 status,
5420 sequence,
5421 close_observation_sequence,
5422 channel_close_commit_authority,
5423 })
5424 }
5425
5426 #[must_use]
5427 pub fn channel_close_commit_authority(
5428 &self,
5429 ) -> Option<&meerkat_live::LiveChannelCloseCommitAuthority> {
5430 self.channel_close_commit_authority.as_ref()
5431 }
5432
5433 #[must_use]
5434 pub fn into_channel_close_commit_authority(
5435 self,
5436 ) -> Option<meerkat_live::LiveChannelCloseCommitAuthority> {
5437 self.channel_close_commit_authority
5438 }
5439}
5440
5441#[derive(Debug, Clone, PartialEq, Eq)]
5447#[cfg(feature = "live")]
5448pub struct LiveCommandResultAuthority {
5449 pub command: dsl::LiveCommandPublicKind,
5450 pub sequence: u64,
5451 pub command_acceptance_sequence: u64,
5452}
5453
5454#[derive(Debug, Clone, PartialEq, Eq)]
5461#[cfg(feature = "live")]
5462pub struct LiveCommandRejectionAuthority {
5463 pub command: dsl::LiveCommandPublicKind,
5464 pub rejection: dsl::LiveCommandRejectionReason,
5465 pub public_error_class: dsl::LiveCommandRejectionPublicErrorClass,
5466 pub sequence: u64,
5467}
5468
5469#[derive(Debug, Clone, PartialEq, Eq)]
5476#[cfg(feature = "live")]
5477pub struct LiveChannelRequestRejectionAuthority {
5478 pub request: dsl::LiveChannelRequestPublicKind,
5479 pub rejection: dsl::LiveChannelRequestRejectionReason,
5480 pub public_error_class: dsl::LiveChannelRequestRejectionPublicErrorClass,
5481 pub sequence: u64,
5482}
5483
5484#[derive(Debug, Clone, PartialEq, Eq)]
5491#[cfg(feature = "live")]
5492pub struct LiveWebrtcTokenAuthority {
5493 pub token: String,
5494 pub expires_at_ms: u64,
5495 pub sequence: u64,
5496}
5497
5498#[derive(Debug, Clone, PartialEq, Eq)]
5504#[cfg(feature = "live")]
5505pub struct LiveWebrtcAnswerAdmissionAuthority {
5506 pub admitted: bool,
5507 pub rejection: Option<dsl::LiveWebrtcAnswerAdmissionRejection>,
5508 pub public_error_class: Option<dsl::LiveChannelRequestRejectionPublicErrorClass>,
5509 pub sequence: u64,
5510}
5511
5512#[derive(Debug, Clone, PartialEq, Eq)]
5519#[cfg(feature = "live")]
5520pub struct LiveWebrtcAnswerResultAuthority {
5521 pub status: dsl::LiveWebrtcAnswerPublicStatus,
5522 pub answered: bool,
5523 pub sequence: u64,
5524 pub answer_observation_sequence: u64,
5525}
5526
5527#[derive(Debug, Clone, PartialEq, Eq)]
5534#[cfg(feature = "live")]
5535pub struct LiveWebsocketTokenAuthority {
5536 pub token: String,
5537 pub expires_at_ms: u64,
5538 pub sequence: u64,
5539}
5540
5541#[derive(Debug, Clone, PartialEq, Eq)]
5547#[cfg(feature = "live")]
5548pub struct LiveWebsocketTokenAdmissionAuthority {
5549 pub admitted: bool,
5550 pub rejection: Option<dsl::LiveWebsocketTokenAdmissionRejection>,
5551 pub public_error_class: Option<dsl::LiveWebsocketTokenAdmissionPublicErrorClass>,
5552 pub sequence: u64,
5553}
5554
5555#[derive(Debug, Clone)]
5561#[cfg(feature = "live")]
5562pub struct LiveChannelStatusAuthority {
5563 pub status: dsl::LiveChannelPublicStatus,
5564 pub sequence: u64,
5565 pub status_observation_sequence: u64,
5566 pub degradation_reason: Option<dsl::LiveChannelDegradationReason>,
5567 pub degradation_detail: Option<String>,
5568 pub channel_status_commit_authority: Option<meerkat_live::LiveChannelStatusCommitAuthority>,
5569}
5570
5571#[cfg(feature = "live")]
5572impl LiveChannelStatusAuthority {
5573 pub(crate) fn from_generated_effect(
5574 channel_id: String,
5575 status: dsl::LiveChannelPublicStatus,
5576 sequence: u64,
5577 status_observation_sequence: u64,
5578 degradation_reason: Option<dsl::LiveChannelDegradationReason>,
5579 degradation_detail: Option<String>,
5580 ) -> Result<Self, String> {
5581 Ok(Self {
5582 status,
5583 sequence,
5584 status_observation_sequence,
5585 degradation_reason,
5586 degradation_detail,
5587 channel_status_commit_authority: Some(build_live_channel_status_commit_authority(
5588 channel_id,
5589 status_observation_sequence,
5590 )?),
5591 })
5592 }
5593
5594 #[must_use]
5595 pub fn channel_status_commit_authority(
5596 &self,
5597 ) -> Option<&meerkat_live::LiveChannelStatusCommitAuthority> {
5598 self.channel_status_commit_authority.as_ref()
5599 }
5600
5601 #[must_use]
5602 pub fn into_channel_status_commit_authority(
5603 self,
5604 ) -> Option<meerkat_live::LiveChannelStatusCommitAuthority> {
5605 self.channel_status_commit_authority
5606 }
5607}
5608
5609#[doc(hidden)]
5616pub struct MeerkatMachineShared {
5617 sessions: RwLock<HashMap<SessionId, RuntimeSessionEntry>>,
5619 boundary_panic_log_gate: meerkat_core::panic_payload::PanicPayloadLogGate,
5626 registration_transaction_slots: StdRwLock<HashMap<SessionId, std::sync::Weak<Mutex<()>>>>,
5631 store: Option<Arc<dyn RuntimeStore>>,
5633 blob_store: Option<Arc<dyn BlobStore>>,
5635 llm_reconfigure_host: StdRwLock<Option<Arc<dyn SessionLlmReconfigureHost>>>,
5637 member_observation_host:
5642 StdRwLock<Option<Arc<dyn crate::member_observation::MemberObservationHost>>>,
5643 member_live_host: StdRwLock<Option<Arc<dyn crate::member_live::MemberLiveHost>>>,
5649 live_commands_served: std::sync::atomic::AtomicU64,
5653 member_incarnation_slots: StdRwLock<HashMap<SessionId, Arc<MemberResidencySlot>>>,
5664 auth_lease: StdRwLock<meerkat_core::handles::GeneratedAuthLeaseHandle>,
5667 #[cfg(not(target_arch = "wasm32"))]
5670 oauth_flows: StdRwLock<Arc<dyn meerkat_auth_core::oauth_flow::OAuthFlowAuthority>>,
5671 #[cfg(feature = "live")]
5675 live_unbound_rejection_authority: crate::driver::ephemeral::SharedIngressDslAuthority,
5676 session_claims: Arc<crate::handles::RuntimeSessionClaimRegistry>,
5683 composition_signal_dispatcher:
5687 StdRwLock<Option<composition::MeerkatCompositionSignalDispatcher>>,
5688 #[cfg(feature = "test-support")]
5692 test_stop_executor_after_ensure: std::sync::atomic::AtomicBool,
5693 #[cfg(feature = "test-support")]
5697 test_pause_executor_after_ensure: std::sync::atomic::AtomicBool,
5698 #[cfg(feature = "test-support")]
5699 test_executor_after_ensure_pause_reached: crate::tokio::sync::Notify,
5700 #[cfg(feature = "test-support")]
5701 test_executor_after_ensure_pause_release: crate::tokio::sync::Notify,
5702 #[cfg(test)]
5705 test_fenced_accept_after_lease: StdMutex<
5706 Option<(
5707 crate::tokio::sync::oneshot::Sender<()>,
5708 crate::tokio::sync::oneshot::Receiver<()>,
5709 )>,
5710 >,
5711 #[cfg(test)]
5716 test_registration_transaction_contention_probe:
5717 StdMutex<Option<(SessionId, crate::tokio::sync::oneshot::Sender<bool>)>>,
5718 #[cfg(test)]
5722 test_fail_post_stop_unregister_after_fence: StdMutex<Option<SessionId>>,
5723 #[cfg(any(test, feature = "test-support"))]
5727 test_runtime_loop_before_queue_authority: StdMutex<
5728 Option<(
5729 SessionId,
5730 crate::tokio::sync::oneshot::Sender<()>,
5731 crate::tokio::sync::oneshot::Receiver<()>,
5732 )>,
5733 >,
5734 #[cfg(test)]
5739 test_control_command_after_logical_lookup: StdMutex<
5740 Option<(
5741 ControlCommandLookupTestKind,
5742 SessionId,
5743 crate::tokio::sync::oneshot::Sender<()>,
5744 crate::tokio::sync::oneshot::Receiver<()>,
5745 )>,
5746 >,
5747}
5748
5749#[cfg(test)]
5750#[derive(Clone, Copy, Debug, PartialEq, Eq)]
5751enum ControlCommandLookupTestKind {
5752 PublishEvent,
5753 Recycle,
5754 Recover,
5755 Destroy,
5756 Retire,
5757}
5758
5759pub struct MemberResidencyUpdate {
5766 adapter: Arc<MeerkatMachine>,
5767 session_id: SessionId,
5768 slot: Arc<MemberResidencySlot>,
5769 slot_guard: Option<crate::tokio::sync::OwnedMutexGuard<()>>,
5770 committed: bool,
5771}
5772
5773pub struct MemberResidencyPublication {
5781 _slot_guard: crate::tokio::sync::OwnedMutexGuard<()>,
5782}
5783
5784struct StagedMemberResidencyPublication {
5792 slot: Arc<MemberResidencySlot>,
5793 slot_guard: Option<crate::tokio::sync::OwnedMutexGuard<()>>,
5794}
5795
5796impl StagedMemberResidencyPublication {
5797 fn finalize(mut self) -> Result<MemberResidencyPublication, RuntimeDriverError> {
5798 let slot_guard = self.slot_guard.take().ok_or_else(|| {
5799 RuntimeDriverError::Internal(
5800 "staged member residency lost its exclusive slot guard".to_string(),
5801 )
5802 })?;
5803 Ok(MemberResidencyPublication {
5804 _slot_guard: slot_guard,
5805 })
5806 }
5807}
5808
5809impl Drop for StagedMemberResidencyPublication {
5810 fn drop(&mut self) {
5811 if self.slot_guard.is_none() {
5812 return;
5813 }
5814 *self
5815 .slot
5816 .state
5817 .write()
5818 .unwrap_or_else(std::sync::PoisonError::into_inner) =
5819 MemberResidencyState::VacantPlaced;
5820 }
5821}
5822
5823impl MemberResidencyUpdate {
5824 fn validate_for_retained_attachment(
5825 &self,
5826 adapter: &Arc<MeerkatMachine>,
5827 witness: &RuntimeExecutorAttachmentWitness,
5828 incarnation: &meerkat_contracts::wire::supervisor_bridge::BridgeMemberIncarnation,
5829 ) -> Result<(), RuntimeDriverError> {
5830 if !Arc::ptr_eq(&self.adapter, adapter) {
5831 return Err(RuntimeDriverError::StaleAuthority {
5832 reason: "member residency update belongs to another machine".to_string(),
5833 });
5834 }
5835 if &self.session_id != witness.session_id() {
5836 return Err(RuntimeDriverError::StaleAuthority {
5837 reason: format!(
5838 "member residency update for session {} cannot publish attachment for {}",
5839 self.session_id,
5840 witness.session_id()
5841 ),
5842 });
5843 }
5844 if self.slot_guard.is_none() {
5845 return Err(RuntimeDriverError::Internal(
5846 "member residency update lost its publication guard".to_string(),
5847 ));
5848 }
5849 self.adapter
5850 .validate_member_incarnation_registration(&self.session_id, incarnation)
5851 }
5852
5853 fn stage_under_retained_mutation_gate(
5854 mut self,
5855 incarnation: meerkat_contracts::wire::supervisor_bridge::BridgeMemberIncarnation,
5856 tracked_turn_journal: Option<Arc<dyn crate::member_observation::TrackedTurnJournal>>,
5857 session_mutation_gate: Arc<crate::tokio::sync::Mutex<()>>,
5858 ) -> Result<StagedMemberResidencyPublication, RuntimeDriverError> {
5859 let slot_guard = self.slot_guard.take().ok_or_else(|| {
5860 RuntimeDriverError::Internal(
5861 "member residency update lost its publication guard".to_string(),
5862 )
5863 })?;
5864 *self
5865 .slot
5866 .state
5867 .write()
5868 .unwrap_or_else(std::sync::PoisonError::into_inner) =
5869 MemberResidencyState::Placed(MemberIncarnationRegistration {
5870 incarnation,
5871 session_mutation_gate,
5872 tracked_turn_journal,
5873 });
5874 self.committed = true;
5875 Ok(StagedMemberResidencyPublication {
5876 slot: Arc::clone(&self.slot),
5877 slot_guard: Some(slot_guard),
5878 })
5879 }
5880
5881 pub async fn commit(
5884 self,
5885 incarnation: meerkat_contracts::wire::supervisor_bridge::BridgeMemberIncarnation,
5886 tracked_turn_journal: Option<Arc<dyn crate::member_observation::TrackedTurnJournal>>,
5887 ) -> Result<MemberResidencyPublication, RuntimeDriverError> {
5888 self.adapter
5889 .validate_member_incarnation_registration(&self.session_id, &incarnation)?;
5890 let session_mutation_gate = self
5891 .adapter
5892 .lock_current_durability_ready_session_mutation_gate(&self.session_id)
5893 .await?;
5894 let gate = {
5895 let sessions = self.adapter.sessions.read().await;
5896 Arc::clone(
5897 &sessions
5898 .get(&self.session_id)
5899 .ok_or(RuntimeDriverError::NotReady {
5900 state: RuntimeState::Destroyed,
5901 })?
5902 .mutation_gate,
5903 )
5904 };
5905 let publication = self
5906 .stage_under_retained_mutation_gate(incarnation, tracked_turn_journal, gate)
5907 .and_then(StagedMemberResidencyPublication::finalize);
5908 drop(session_mutation_gate);
5909 publication
5910 }
5911
5912 pub fn vacate(mut self) -> Result<MemberResidencyPublication, RuntimeDriverError> {
5915 let slot_guard = self.slot_guard.take().ok_or_else(|| {
5916 RuntimeDriverError::Internal(
5917 "member residency update lost its vacancy publication guard".to_string(),
5918 )
5919 })?;
5920 *self
5921 .slot
5922 .state
5923 .write()
5924 .unwrap_or_else(std::sync::PoisonError::into_inner) =
5925 MemberResidencyState::VacantPlaced;
5926 self.committed = true;
5927 Ok(MemberResidencyPublication {
5928 _slot_guard: slot_guard,
5929 })
5930 }
5931}
5932
5933impl Drop for MemberResidencyUpdate {
5934 fn drop(&mut self) {
5935 if self.committed {
5936 return;
5937 }
5938 *self
5939 .slot
5940 .state
5941 .write()
5942 .unwrap_or_else(std::sync::PoisonError::into_inner) =
5943 MemberResidencyState::VacantPlaced;
5944 }
5945}
5946
5947#[derive(Clone)]
5954pub struct MeerkatMachine {
5955 shared: Arc<MeerkatMachineShared>,
5956}
5957
5958impl std::ops::Deref for MeerkatMachine {
5959 type Target = MeerkatMachineShared;
5960
5961 fn deref(&self) -> &Self::Target {
5962 self.shared.as_ref()
5963 }
5964}
5965
5966impl MeerkatMachine {
5967 fn session_registration_transaction_slot(&self, session_id: &SessionId) -> Arc<Mutex<()>> {
5977 let mut slots = self
5978 .registration_transaction_slots
5979 .write()
5980 .unwrap_or_else(std::sync::PoisonError::into_inner);
5981 if let Some(slot) = slots.get(session_id).and_then(std::sync::Weak::upgrade) {
5982 return slot;
5983 }
5984 slots.retain(|_, slot| slot.upgrade().is_some());
5985 let slot = Arc::new(Mutex::new(()));
5986 slots.insert(session_id.clone(), Arc::downgrade(&slot));
5987 slot
5988 }
5989
5990 async fn lock_session_registration_transaction(
5991 &self,
5992 session_id: &SessionId,
5993 ) -> crate::tokio::sync::OwnedMutexGuard<()> {
5994 let slot = self.session_registration_transaction_slot(session_id);
5995 #[cfg(test)]
5996 let contention_probe = {
5997 let mut probe = self
5998 .test_registration_transaction_contention_probe
5999 .lock()
6000 .unwrap_or_else(std::sync::PoisonError::into_inner);
6001 if probe
6002 .as_ref()
6003 .is_some_and(|(probe_session_id, _)| probe_session_id == session_id)
6004 {
6005 probe.take().map(|(_, sender)| sender)
6006 } else {
6007 None
6008 }
6009 };
6010 #[cfg(test)]
6011 if let Some(contention_probe) = contention_probe {
6012 return match Arc::clone(&slot).try_lock_owned() {
6013 Ok(guard) => {
6014 let _ = contention_probe.send(false);
6015 guard
6016 }
6017 Err(_) => {
6018 let _ = contention_probe.send(true);
6019 slot.lock_owned().await
6020 }
6021 };
6022 }
6023 slot.lock_owned().await
6024 }
6025
6026 #[cfg(test)]
6027 pub(crate) fn probe_next_session_registration_transaction_contention_for_test(
6028 &self,
6029 session_id: &SessionId,
6030 ) -> crate::tokio::sync::oneshot::Receiver<bool> {
6031 let (sender, receiver) = crate::tokio::sync::oneshot::channel();
6032 *self
6033 .test_registration_transaction_contention_probe
6034 .lock()
6035 .unwrap_or_else(std::sync::PoisonError::into_inner) =
6036 Some((session_id.clone(), sender));
6037 receiver
6038 }
6039
6040 pub async fn begin_member_residency_update(
6043 self: &Arc<Self>,
6044 session_id: SessionId,
6045 ) -> MemberResidencyUpdate {
6046 let slot = self.member_incarnation_slot(&session_id);
6047 let slot_guard = Arc::clone(&slot.gate).lock_owned().await;
6048 *slot
6049 .state
6050 .write()
6051 .unwrap_or_else(std::sync::PoisonError::into_inner) =
6052 MemberResidencyState::VacantPlaced;
6053 MemberResidencyUpdate {
6054 adapter: Arc::clone(self),
6055 session_id,
6056 slot,
6057 slot_guard: Some(slot_guard),
6058 committed: false,
6059 }
6060 }
6061
6062 #[must_use]
6065 pub fn session_control_authority(&self) -> MachineSessionControlAuthority {
6066 MachineSessionControlAuthority { _private: () }
6067 }
6068
6069 pub fn set_member_observation_host(
6074 &self,
6075 host: Arc<dyn crate::member_observation::MemberObservationHost>,
6076 ) {
6077 *self
6078 .member_observation_host
6079 .write()
6080 .unwrap_or_else(std::sync::PoisonError::into_inner) = Some(host);
6081 }
6082
6083 pub(crate) fn member_observation_host(
6085 &self,
6086 ) -> Option<Arc<dyn crate::member_observation::MemberObservationHost>> {
6087 self.member_observation_host
6088 .read()
6089 .unwrap_or_else(std::sync::PoisonError::into_inner)
6090 .clone()
6091 }
6092
6093 pub fn set_member_live_host(&self, host: Arc<dyn crate::member_live::MemberLiveHost>) {
6098 *self
6099 .member_live_host
6100 .write()
6101 .unwrap_or_else(std::sync::PoisonError::into_inner) = Some(host);
6102 }
6103
6104 pub(crate) fn member_live_host(&self) -> Option<Arc<dyn crate::member_live::MemberLiveHost>> {
6106 self.member_live_host
6107 .read()
6108 .unwrap_or_else(std::sync::PoisonError::into_inner)
6109 .clone()
6110 }
6111
6112 #[cfg(feature = "live")]
6116 pub async fn acquire_live_open_lifecycle_lease(
6117 &self,
6118 session_id: &meerkat_core::types::SessionId,
6119 ) -> Result<crate::member_live::MemberLiveLifecycleLease, RuntimeDriverError> {
6120 let (gate, guard) = self
6121 .lock_current_session_live_lifecycle_gate(session_id)
6122 .await
6123 .ok_or(RuntimeDriverError::NotReady {
6124 state: RuntimeState::Destroyed,
6125 })?;
6126 Ok(crate::member_live::MemberLiveLifecycleLease::new(
6127 session_id.clone(),
6128 gate,
6129 guard,
6130 ))
6131 }
6132
6133 #[cfg(feature = "live")]
6138 pub async fn acquire_member_live_disposal_lease(
6139 &self,
6140 session_id: &meerkat_core::types::SessionId,
6141 ) -> Result<crate::member_live::MemberLiveLifecycleLease, crate::member_live::MemberLiveError>
6142 {
6143 let (gate, guard) = self
6144 .lock_current_session_live_lifecycle_gate(session_id)
6145 .await
6146 .ok_or_else(|| crate::member_live::MemberLiveError::Unavailable {
6147 reason: format!(
6148 "member-live lifecycle gate is unavailable for session {session_id}"
6149 ),
6150 })?;
6151 let lease =
6152 crate::member_live::MemberLiveLifecycleLease::new(session_id.clone(), gate, guard);
6153
6154 self.prove_member_live_absence_while_lease_held(session_id, &lease)
6155 .await?;
6156 Ok(lease)
6157 }
6158
6159 #[cfg(feature = "live")]
6165 async fn prove_member_live_absence_while_lease_held(
6166 &self,
6167 session_id: &meerkat_core::types::SessionId,
6168 lease: &crate::member_live::MemberLiveLifecycleLease,
6169 ) -> Result<(), crate::member_live::MemberLiveError> {
6170 if lease.session_id() != session_id {
6171 return Err(crate::member_live::MemberLiveError::Internal {
6172 reason: format!(
6173 "member-live lifecycle lease for session {} cannot dispose session {session_id}",
6174 lease.session_id()
6175 ),
6176 });
6177 }
6178 let lease_is_current = {
6179 let sessions = self.sessions.read().await;
6180 sessions
6181 .get(session_id)
6182 .is_some_and(|entry| lease.matches_gate(&entry.live_lifecycle_gate))
6183 };
6184 if !lease_is_current {
6185 return Err(crate::member_live::MemberLiveError::Unavailable {
6186 reason: format!("member-live lifecycle lease is stale for session {session_id}"),
6187 });
6188 }
6189
6190 let Some(channel_id) = self.live_active_channel_for_session(session_id).await else {
6191 return Ok(());
6192 };
6193 let Some(host) = self.member_live_host() else {
6194 return Err(crate::member_live::MemberLiveError::Unavailable {
6195 reason: format!(
6196 "session {session_id} still owns live channel '{channel_id}', but no transport-neutral live cleanup host is composed"
6197 ),
6198 });
6199 };
6200 let timeout = crate::member_live::MEMBER_LIVE_DISPOSAL_CEILING;
6201 match crate::tokio::time::timeout(timeout, host.close(session_id, channel_id.as_str()))
6202 .await
6203 {
6204 Ok(Ok(_)) => {}
6205 Ok(Err(error)) => return Err(error),
6206 Err(_) => {
6207 return Err(crate::member_live::MemberLiveError::Unavailable {
6208 reason: format!(
6209 "member-live disposal close for channel '{channel_id}' timed out after {}ms for session {session_id}",
6210 timeout.as_millis()
6211 ),
6212 });
6213 }
6214 }
6215 if let Some(still_active) = self.live_active_channel_for_session(session_id).await {
6216 return Err(crate::member_live::MemberLiveError::Internal {
6217 reason: format!(
6218 "member-live close returned success but machine channel '{still_active}' remains active for session {session_id}"
6219 ),
6220 });
6221 }
6222 Ok(())
6223 }
6224
6225 #[cfg(feature = "live")]
6238 pub async fn close_member_live_channel_for_disposal(
6239 &self,
6240 session_id: &meerkat_core::types::SessionId,
6241 ) -> Result<(), crate::member_live::MemberLiveError> {
6242 let _lease = self.acquire_member_live_disposal_lease(session_id).await?;
6243 Ok(())
6244 }
6245
6246 pub(crate) fn note_live_command_served(&self) {
6250 self.live_commands_served
6251 .fetch_add(1, std::sync::atomic::Ordering::Relaxed);
6252 }
6253
6254 pub fn live_commands_served(&self) -> u64 {
6256 self.live_commands_served
6257 .load(std::sync::atomic::Ordering::Relaxed)
6258 }
6259
6260 pub fn tracked_turn_journal(
6264 &self,
6265 session_id: &SessionId,
6266 ) -> Option<Arc<dyn crate::member_observation::TrackedTurnJournal>> {
6267 let slot = self
6268 .member_incarnation_slots
6269 .read()
6270 .unwrap_or_else(std::sync::PoisonError::into_inner)
6271 .get(session_id)
6272 .cloned()?;
6273 let state = slot
6274 .state
6275 .read()
6276 .unwrap_or_else(std::sync::PoisonError::into_inner);
6277 match &*state {
6278 MemberResidencyState::Placed(registration) => registration.tracked_turn_journal.clone(),
6279 MemberResidencyState::PeerOnly | MemberResidencyState::VacantPlaced => None,
6280 }
6281 }
6282
6283 fn validate_member_incarnation_registration(
6284 &self,
6285 session_id: &SessionId,
6286 incarnation: &meerkat_contracts::wire::supervisor_bridge::BridgeMemberIncarnation,
6287 ) -> Result<(), RuntimeDriverError> {
6288 if incarnation.mob_id.is_empty()
6289 || incarnation.agent_identity.is_empty()
6290 || incarnation.host_id.is_empty()
6291 || incarnation.member_session_id != session_id.to_string()
6292 || incarnation.binding_generation == 0
6293 || incarnation.fence_token == 0
6294 {
6295 return Err(RuntimeDriverError::ValidationFailed {
6296 reason: format!(
6297 "member incarnation registration requires complete identities plus nonzero host-binding generation and fence for session '{session_id}': {incarnation:?}"
6298 ),
6299 });
6300 }
6301 Ok(())
6302 }
6303
6304 pub fn member_incarnation(
6309 &self,
6310 session_id: &SessionId,
6311 ) -> Option<meerkat_contracts::wire::supervisor_bridge::BridgeMemberIncarnation> {
6312 let slot = self
6313 .member_incarnation_slots
6314 .read()
6315 .unwrap_or_else(std::sync::PoisonError::into_inner)
6316 .get(session_id)
6317 .cloned()?;
6318 let state = slot
6319 .state
6320 .read()
6321 .unwrap_or_else(std::sync::PoisonError::into_inner);
6322 match &*state {
6323 MemberResidencyState::Placed(registration) => Some(registration.incarnation.clone()),
6324 MemberResidencyState::PeerOnly | MemberResidencyState::VacantPlaced => None,
6325 }
6326 }
6327
6328 #[cfg(test)]
6329 pub(crate) fn test_member_residency_is_vacant(&self, session_id: &SessionId) -> bool {
6330 self.member_incarnation_slots
6331 .read()
6332 .unwrap_or_else(std::sync::PoisonError::into_inner)
6333 .get(session_id)
6334 .is_some_and(|slot| {
6335 matches!(
6336 &*slot
6337 .state
6338 .read()
6339 .unwrap_or_else(std::sync::PoisonError::into_inner),
6340 MemberResidencyState::VacantPlaced
6341 )
6342 })
6343 }
6344
6345 fn member_incarnation_slot(&self, session_id: &SessionId) -> Arc<MemberResidencySlot> {
6346 if let Some(slot) = self
6347 .member_incarnation_slots
6348 .read()
6349 .unwrap_or_else(std::sync::PoisonError::into_inner)
6350 .get(session_id)
6351 .cloned()
6352 {
6353 return slot;
6354 }
6355 Arc::clone(
6356 self.member_incarnation_slots
6357 .write()
6358 .unwrap_or_else(std::sync::PoisonError::into_inner)
6359 .entry(session_id.clone())
6360 .or_insert_with(|| Arc::new(MemberResidencySlot::peer_only())),
6361 )
6362 }
6363
6364 pub(crate) async fn lock_member_effect_authority(
6370 &self,
6371 session_id: &SessionId,
6372 expected: &meerkat_contracts::wire::supervisor_bridge::BridgeMemberIncarnation,
6373 ) -> Result<MemberEffectAuthorityGuard, RuntimeDriverError> {
6374 self.lock_optional_member_effect_authority(session_id, Some(expected))
6375 .await
6376 }
6377
6378 pub(crate) async fn lock_member_live_authority(
6382 &self,
6383 session_id: &SessionId,
6384 expected: &meerkat_contracts::wire::supervisor_bridge::BridgeMemberIncarnation,
6385 ) -> Result<MemberLiveAuthorityGuard, RuntimeDriverError> {
6386 let lease = self
6387 .acquire_member_effect_authority_lease(session_id, Some(expected))
6388 .await?;
6389 let MemberEffectAuthorityLease {
6390 slot_guard,
6391 session_mutation_gate,
6392 ..
6393 } = lease;
6394 let registration_guard = self.lock_session_registration_transaction(session_id).await;
6395 {
6396 let sessions = self.sessions.read().await;
6397 let Some(entry) = sessions.get(session_id) else {
6398 return Err(RuntimeDriverError::StaleAuthority {
6399 reason: format!(
6400 "member live/open expected incarnation {expected:?}; runtime session is absent"
6401 ),
6402 });
6403 };
6404 if !Arc::ptr_eq(&entry.mutation_gate, &session_mutation_gate) {
6405 return Err(RuntimeDriverError::StaleAuthority {
6406 reason: format!(
6407 "member live/open expected incarnation {expected:?}; runtime session was replaced"
6408 ),
6409 });
6410 }
6411 entry.require_durability_ready().map_err(|required| {
6412 RuntimeDriverError::RecoveryRepairBlocked {
6413 evidence_digest: None,
6414 reason: required.to_string(),
6415 }
6416 })?;
6417 }
6418 let state = self
6419 .existing_session_runtime_state(session_id)
6420 .await
6421 .unwrap_or(RuntimeState::Destroyed);
6422 self.reject_unregistration_drain_ingress(session_id, state)
6423 .await?;
6424 Ok(MemberLiveAuthorityGuard {
6425 _slot_guard: slot_guard,
6426 _registration_guard: registration_guard,
6427 })
6428 }
6429
6430 pub(crate) async fn lock_optional_member_effect_authority(
6435 &self,
6436 session_id: &SessionId,
6437 expected: Option<&meerkat_contracts::wire::supervisor_bridge::BridgeMemberIncarnation>,
6438 ) -> Result<MemberEffectAuthorityGuard, RuntimeDriverError> {
6439 let lease = self
6440 .acquire_member_effect_authority_lease(session_id, expected)
6441 .await?;
6442 let MemberEffectAuthorityLease {
6443 slot_guard,
6444 session_mutation_gate,
6445 ..
6446 } = lease;
6447 let session_guard = Arc::clone(&session_mutation_gate).lock_owned().await;
6448 {
6449 let sessions = self.sessions.read().await;
6450 let Some(entry) = sessions.get(session_id) else {
6451 return Err(RuntimeDriverError::StaleAuthority {
6452 reason: format!(
6453 "member effect expected incarnation {expected:?}; runtime session is absent"
6454 ),
6455 });
6456 };
6457 if !Arc::ptr_eq(&entry.mutation_gate, &session_mutation_gate) {
6458 return Err(RuntimeDriverError::StaleAuthority {
6459 reason: format!(
6460 "member effect expected incarnation {expected:?}; runtime session was replaced"
6461 ),
6462 });
6463 }
6464 entry.require_durability_ready().map_err(|required| {
6465 RuntimeDriverError::RecoveryRepairBlocked {
6466 evidence_digest: None,
6467 reason: required.to_string(),
6468 }
6469 })?;
6470 }
6471 let state = self
6472 .existing_session_runtime_state(session_id)
6473 .await
6474 .unwrap_or(RuntimeState::Destroyed);
6475 self.reject_unregistration_drain_ingress(session_id, state)
6476 .await?;
6477 Ok(MemberEffectAuthorityGuard {
6478 _slot_guard: slot_guard,
6479 _session_guard: session_guard,
6480 })
6481 }
6482
6483 async fn acquire_member_effect_authority_lease(
6484 &self,
6485 session_id: &SessionId,
6486 expected: Option<&meerkat_contracts::wire::supervisor_bridge::BridgeMemberIncarnation>,
6487 ) -> Result<MemberEffectAuthorityLease, RuntimeDriverError> {
6488 let slot = self.member_incarnation_slot(session_id);
6489 let slot_guard = Arc::clone(&slot.gate).lock_owned().await;
6490 let registered_gate = {
6491 let state = slot
6492 .state
6493 .read()
6494 .unwrap_or_else(std::sync::PoisonError::into_inner);
6495 match (expected, &*state) {
6496 (Some(expected), MemberResidencyState::Placed(registration))
6497 if ®istration.incarnation == expected =>
6498 {
6499 Some(Arc::clone(®istration.session_mutation_gate))
6500 }
6501 (Some(expected), MemberResidencyState::Placed(registration)) => {
6502 return Err(RuntimeDriverError::StaleAuthority {
6503 reason: format!(
6504 "member effect expected incarnation {expected:?}; current is {:?}",
6505 registration.incarnation
6506 ),
6507 });
6508 }
6509 (
6510 Some(expected),
6511 MemberResidencyState::PeerOnly | MemberResidencyState::VacantPlaced,
6512 ) => {
6513 return Err(RuntimeDriverError::StaleAuthority {
6514 reason: format!(
6515 "member effect expected incarnation {expected:?}; current residency is absent"
6516 ),
6517 });
6518 }
6519 (None, MemberResidencyState::Placed(registration)) => {
6520 return Err(RuntimeDriverError::StaleAuthority {
6521 reason: format!(
6522 "peer-only member effect expected no host residency; current is {:?}",
6523 registration.incarnation
6524 ),
6525 });
6526 }
6527 (None, MemberResidencyState::VacantPlaced) => {
6528 return Err(RuntimeDriverError::StaleAuthority {
6529 reason: "peer-only member effect cannot address a vacant placed residency"
6530 .to_string(),
6531 });
6532 }
6533 (None, MemberResidencyState::PeerOnly) => None,
6534 }
6535 };
6536 let registered_gate = match registered_gate {
6537 Some(gate) => gate,
6538 None => self.session_mutation_gate(session_id).await.ok_or(
6539 RuntimeDriverError::NotReady {
6540 state: RuntimeState::Destroyed,
6541 },
6542 )?,
6543 };
6544 {
6545 let sessions = self.sessions.read().await;
6546 let Some(entry) = sessions.get(session_id) else {
6547 return Err(RuntimeDriverError::StaleAuthority {
6548 reason: format!(
6549 "member effect expected incarnation {expected:?}; runtime session is absent"
6550 ),
6551 });
6552 };
6553 if !Arc::ptr_eq(&entry.mutation_gate, ®istered_gate) {
6554 return Err(RuntimeDriverError::StaleAuthority {
6555 reason: format!(
6556 "member effect expected incarnation {expected:?}; runtime session was replaced"
6557 ),
6558 });
6559 }
6560 }
6561 Ok(MemberEffectAuthorityLease {
6562 slot,
6563 slot_guard,
6564 session_mutation_gate: registered_gate,
6565 })
6566 }
6567
6568 fn validate_member_effect_authority_lease_current(
6572 &self,
6573 session_id: &SessionId,
6574 lease: &MemberEffectAuthorityLease,
6575 expected: Option<&meerkat_contracts::wire::supervisor_bridge::BridgeMemberIncarnation>,
6576 ) -> Result<(), RuntimeDriverError> {
6577 let slot_is_current = self
6578 .member_incarnation_slots
6579 .read()
6580 .unwrap_or_else(std::sync::PoisonError::into_inner)
6581 .get(session_id)
6582 .is_some_and(|slot| Arc::ptr_eq(slot, &lease.slot));
6583 if !slot_is_current {
6584 return Err(RuntimeDriverError::StaleAuthority {
6585 reason: "member effect residency slot was replaced".to_string(),
6586 });
6587 }
6588
6589 let state = lease
6590 .slot
6591 .state
6592 .read()
6593 .unwrap_or_else(std::sync::PoisonError::into_inner);
6594 match (expected, &*state) {
6595 (Some(expected), MemberResidencyState::Placed(registration))
6596 if ®istration.incarnation == expected
6597 && Arc::ptr_eq(
6598 ®istration.session_mutation_gate,
6599 &lease.session_mutation_gate,
6600 ) =>
6601 {
6602 Ok(())
6603 }
6604 (Some(expected), MemberResidencyState::Placed(registration)) => {
6605 Err(RuntimeDriverError::StaleAuthority {
6606 reason: format!(
6607 "member effect expected incarnation {expected:?}; current is {:?}",
6608 registration.incarnation
6609 ),
6610 })
6611 }
6612 (
6613 Some(expected),
6614 MemberResidencyState::PeerOnly | MemberResidencyState::VacantPlaced,
6615 ) => Err(RuntimeDriverError::StaleAuthority {
6616 reason: format!(
6617 "member effect expected incarnation {expected:?}; current residency is absent"
6618 ),
6619 }),
6620 (None, MemberResidencyState::PeerOnly) => Ok(()),
6621 (None, MemberResidencyState::Placed(registration)) => {
6622 Err(RuntimeDriverError::StaleAuthority {
6623 reason: format!(
6624 "peer-only member effect expected no host residency; current is {:?}",
6625 registration.incarnation
6626 ),
6627 })
6628 }
6629 (None, MemberResidencyState::VacantPlaced) => Err(RuntimeDriverError::StaleAuthority {
6630 reason: "peer-only member effect cannot address a vacant placed residency"
6631 .to_string(),
6632 }),
6633 }
6634 }
6635
6636 #[must_use]
6641 pub fn shares_runtime_persistence_with(&self, other: &Self) -> bool {
6642 match (&self.store, &other.store) {
6643 (None, None) => true,
6644 (Some(a), Some(b)) => runtime_stores_share_authority(a, b),
6645 _ => false,
6646 }
6647 }
6648
6649 #[must_use]
6652 pub fn shares_runtime_store_authority(&self, store: &Arc<dyn RuntimeStore>) -> bool {
6653 self.store
6654 .as_ref()
6655 .is_some_and(|machine_store| runtime_stores_share_authority(machine_store, store))
6656 }
6657
6658 #[must_use]
6660 pub fn has_runtime_persistence(&self) -> bool {
6661 self.store.is_some()
6662 }
6663
6664 fn normalize_destroyed_error(err: RuntimeDriverError) -> RuntimeDriverError {
6665 match err {
6666 RuntimeDriverError::NotReady {
6667 state: RuntimeState::Destroyed,
6668 } => RuntimeDriverError::Destroyed,
6669 other => other,
6670 }
6671 }
6672
6673 pub fn ephemeral() -> Self {
6675 let auth_lease = Arc::new(crate::handles::RuntimeAuthLeaseHandle::new());
6676 #[cfg(not(target_arch = "wasm32"))]
6677 let oauth_flows = Arc::new(crate::handles::RuntimeOAuthFlowHandle::new_with_auth_lease(
6678 std::time::Duration::from_secs(10 * 60),
6679 Arc::clone(&auth_lease),
6680 ));
6681 let auth_lease = generated_runtime_auth_lease_handle(auth_lease);
6682 Self {
6683 shared: Arc::new(MeerkatMachineShared {
6684 sessions: RwLock::new(HashMap::new()),
6685 boundary_panic_log_gate: meerkat_core::panic_payload::PanicPayloadLogGate::default(
6686 ),
6687 registration_transaction_slots: StdRwLock::new(HashMap::new()),
6688 store: None,
6689 blob_store: None,
6690 llm_reconfigure_host: StdRwLock::new(None),
6691 member_observation_host: StdRwLock::new(None),
6692 member_live_host: StdRwLock::new(None),
6693 live_commands_served: std::sync::atomic::AtomicU64::new(0),
6694 member_incarnation_slots: StdRwLock::new(HashMap::new()),
6695 auth_lease: StdRwLock::new(auth_lease),
6696 #[cfg(not(target_arch = "wasm32"))]
6697 oauth_flows: StdRwLock::new(oauth_flows),
6698 #[cfg(feature = "live")]
6699 live_unbound_rejection_authority: live_unbound_rejection_authority(),
6700 session_claims: Arc::new(crate::handles::RuntimeSessionClaimRegistry::new()),
6701 composition_signal_dispatcher: StdRwLock::new(None),
6702 #[cfg(feature = "test-support")]
6703 test_stop_executor_after_ensure: std::sync::atomic::AtomicBool::new(false),
6704 #[cfg(feature = "test-support")]
6705 test_pause_executor_after_ensure: std::sync::atomic::AtomicBool::new(false),
6706 #[cfg(feature = "test-support")]
6707 test_executor_after_ensure_pause_reached: crate::tokio::sync::Notify::new(),
6708 #[cfg(feature = "test-support")]
6709 test_executor_after_ensure_pause_release: crate::tokio::sync::Notify::new(),
6710 #[cfg(test)]
6711 test_fenced_accept_after_lease: StdMutex::new(None),
6712 #[cfg(test)]
6713 test_registration_transaction_contention_probe: StdMutex::new(None),
6714 #[cfg(test)]
6715 test_fail_post_stop_unregister_after_fence: StdMutex::new(None),
6716 #[cfg(any(test, feature = "test-support"))]
6717 test_runtime_loop_before_queue_authority: StdMutex::new(None),
6718 #[cfg(test)]
6719 test_control_command_after_logical_lookup: StdMutex::new(None),
6720 }),
6721 }
6722 }
6723
6724 pub fn persistent(store: Arc<dyn RuntimeStore>, blob_store: Arc<dyn BlobStore>) -> Self {
6731 #[cfg(not(target_arch = "wasm32"))]
6732 let (auth_lease, oauth_flows) = {
6733 let authorities = persistent_auth_authorities(&store);
6734 (
6735 Arc::clone(&authorities.auth_lease),
6736 Arc::clone(&authorities.oauth_flows),
6737 )
6738 };
6739 #[cfg(target_arch = "wasm32")]
6740 let auth_lease = Arc::new(crate::handles::RuntimeAuthLeaseHandle::new());
6741 let auth_lease = generated_runtime_auth_lease_handle(auth_lease);
6742 Self {
6743 shared: Arc::new(MeerkatMachineShared {
6744 sessions: RwLock::new(HashMap::new()),
6745 boundary_panic_log_gate: meerkat_core::panic_payload::PanicPayloadLogGate::default(
6746 ),
6747 registration_transaction_slots: StdRwLock::new(HashMap::new()),
6748 store: Some(store),
6749 blob_store: Some(blob_store),
6750 llm_reconfigure_host: StdRwLock::new(None),
6751 member_observation_host: StdRwLock::new(None),
6752 member_live_host: StdRwLock::new(None),
6753 live_commands_served: std::sync::atomic::AtomicU64::new(0),
6754 member_incarnation_slots: StdRwLock::new(HashMap::new()),
6755 auth_lease: StdRwLock::new(auth_lease),
6756 #[cfg(not(target_arch = "wasm32"))]
6757 oauth_flows: StdRwLock::new(oauth_flows),
6758 #[cfg(feature = "live")]
6759 live_unbound_rejection_authority: live_unbound_rejection_authority(),
6760 session_claims: Arc::new(crate::handles::RuntimeSessionClaimRegistry::new()),
6761 composition_signal_dispatcher: StdRwLock::new(None),
6762 #[cfg(feature = "test-support")]
6763 test_stop_executor_after_ensure: std::sync::atomic::AtomicBool::new(false),
6764 #[cfg(feature = "test-support")]
6765 test_pause_executor_after_ensure: std::sync::atomic::AtomicBool::new(false),
6766 #[cfg(feature = "test-support")]
6767 test_executor_after_ensure_pause_reached: crate::tokio::sync::Notify::new(),
6768 #[cfg(feature = "test-support")]
6769 test_executor_after_ensure_pause_release: crate::tokio::sync::Notify::new(),
6770 #[cfg(test)]
6771 test_fenced_accept_after_lease: StdMutex::new(None),
6772 #[cfg(test)]
6773 test_registration_transaction_contention_probe: StdMutex::new(None),
6774 #[cfg(test)]
6775 test_fail_post_stop_unregister_after_fence: StdMutex::new(None),
6776 #[cfg(any(test, feature = "test-support"))]
6777 test_runtime_loop_before_queue_authority: StdMutex::new(None),
6778 #[cfg(test)]
6779 test_control_command_after_logical_lookup: StdMutex::new(None),
6780 }),
6781 }
6782 }
6783
6784 pub fn persistent_without_blobs(store: Arc<dyn RuntimeStore>) -> Self {
6791 #[cfg(not(target_arch = "wasm32"))]
6792 let (auth_lease, oauth_flows) = {
6793 let authorities = persistent_auth_authorities(&store);
6794 (
6795 Arc::clone(&authorities.auth_lease),
6796 Arc::clone(&authorities.oauth_flows),
6797 )
6798 };
6799 #[cfg(target_arch = "wasm32")]
6800 let auth_lease = Arc::new(crate::handles::RuntimeAuthLeaseHandle::new());
6801 let auth_lease = generated_runtime_auth_lease_handle(auth_lease);
6802 Self {
6803 shared: Arc::new(MeerkatMachineShared {
6804 sessions: RwLock::new(HashMap::new()),
6805 boundary_panic_log_gate: meerkat_core::panic_payload::PanicPayloadLogGate::default(
6806 ),
6807 registration_transaction_slots: StdRwLock::new(HashMap::new()),
6808 store: Some(store),
6809 blob_store: Some(Arc::new(UnavailableBlobStore)),
6810 llm_reconfigure_host: StdRwLock::new(None),
6811 member_observation_host: StdRwLock::new(None),
6812 member_live_host: StdRwLock::new(None),
6813 live_commands_served: std::sync::atomic::AtomicU64::new(0),
6814 member_incarnation_slots: StdRwLock::new(HashMap::new()),
6815 auth_lease: StdRwLock::new(auth_lease),
6816 #[cfg(not(target_arch = "wasm32"))]
6817 oauth_flows: StdRwLock::new(oauth_flows),
6818 #[cfg(feature = "live")]
6819 live_unbound_rejection_authority: live_unbound_rejection_authority(),
6820 session_claims: Arc::new(crate::handles::RuntimeSessionClaimRegistry::new()),
6821 composition_signal_dispatcher: StdRwLock::new(None),
6822 #[cfg(feature = "test-support")]
6823 test_stop_executor_after_ensure: std::sync::atomic::AtomicBool::new(false),
6824 #[cfg(feature = "test-support")]
6825 test_pause_executor_after_ensure: std::sync::atomic::AtomicBool::new(false),
6826 #[cfg(feature = "test-support")]
6827 test_executor_after_ensure_pause_reached: crate::tokio::sync::Notify::new(),
6828 #[cfg(feature = "test-support")]
6829 test_executor_after_ensure_pause_release: crate::tokio::sync::Notify::new(),
6830 #[cfg(test)]
6831 test_fenced_accept_after_lease: StdMutex::new(None),
6832 #[cfg(test)]
6833 test_registration_transaction_contention_probe: StdMutex::new(None),
6834 #[cfg(test)]
6835 test_fail_post_stop_unregister_after_fence: StdMutex::new(None),
6836 #[cfg(any(test, feature = "test-support"))]
6837 test_runtime_loop_before_queue_authority: StdMutex::new(None),
6838 #[cfg(test)]
6839 test_control_command_after_logical_lookup: StdMutex::new(None),
6840 }),
6841 }
6842 }
6843
6844 pub fn auth_lease_handle(&self) -> Arc<dyn meerkat_core::handles::AuthLeaseHandle> {
6847 self.generated_auth_lease_handle().clone_handle()
6848 }
6849
6850 pub fn generated_auth_lease_handle(&self) -> meerkat_core::handles::GeneratedAuthLeaseHandle {
6853 self.auth_lease
6854 .read()
6855 .unwrap_or_else(std::sync::PoisonError::into_inner)
6856 .clone()
6857 }
6858
6859 #[cfg(not(target_arch = "wasm32"))]
6861 pub fn provider_auth_runtime_authority(&self) -> ProviderAuthRuntimeAuthority {
6862 let auth_lease = self
6863 .auth_lease
6864 .read()
6865 .unwrap_or_else(std::sync::PoisonError::into_inner);
6866 let oauth_flows = self
6867 .oauth_flows
6868 .read()
6869 .unwrap_or_else(std::sync::PoisonError::into_inner);
6870 ProviderAuthRuntimeAuthority {
6871 auth_lease: auth_lease.clone(),
6872 oauth_flows: Arc::clone(&oauth_flows),
6873 }
6874 }
6875
6876 pub fn set_auth_lease_handle(&self, handle: Arc<crate::handles::RuntimeAuthLeaseHandle>) {
6882 self.set_runtime_auth_lease_handle(handle);
6883 }
6884
6885 #[cfg(all(not(target_arch = "wasm32"), any(test, feature = "test-support")))]
6891 pub fn set_auth_lease_handle_with_oauth_flow_authority(
6892 &self,
6893 handle: Arc<crate::handles::RuntimeAuthLeaseHandle>,
6894 oauth_flows: Arc<dyn meerkat_auth_core::oauth_flow::OAuthFlowAuthority>,
6895 ) {
6896 *self
6897 .oauth_flows
6898 .write()
6899 .unwrap_or_else(std::sync::PoisonError::into_inner) = oauth_flows;
6900 let handle = generated_runtime_auth_lease_handle(handle);
6901 *self
6902 .auth_lease
6903 .write()
6904 .unwrap_or_else(std::sync::PoisonError::into_inner) = handle;
6905 }
6906
6907 pub fn set_runtime_auth_lease_handle(
6910 &self,
6911 handle: Arc<crate::handles::RuntimeAuthLeaseHandle>,
6912 ) {
6913 #[cfg(not(target_arch = "wasm32"))]
6914 {
6915 let mut auth_slot = self
6916 .auth_lease
6917 .write()
6918 .unwrap_or_else(std::sync::PoisonError::into_inner);
6919 let mut oauth_slot = self
6920 .oauth_flows
6921 .write()
6922 .unwrap_or_else(std::sync::PoisonError::into_inner);
6923 *oauth_slot = Arc::new(crate::handles::RuntimeOAuthFlowHandle::new_with_auth_lease(
6924 std::time::Duration::from_secs(10 * 60),
6925 Arc::clone(&handle),
6926 ));
6927 *auth_slot = generated_runtime_auth_lease_handle(handle);
6928 }
6929 #[cfg(target_arch = "wasm32")]
6930 let handle = generated_runtime_auth_lease_handle(handle);
6931 #[cfg(target_arch = "wasm32")]
6932 {
6933 *self
6934 .auth_lease
6935 .write()
6936 .unwrap_or_else(std::sync::PoisonError::into_inner) = handle;
6937 }
6938 }
6939
6940 #[cfg(not(target_arch = "wasm32"))]
6943 pub fn oauth_flow_authority(
6944 &self,
6945 ) -> Arc<dyn meerkat_auth_core::oauth_flow::OAuthFlowAuthority> {
6946 Arc::clone(
6947 &self
6948 .oauth_flows
6949 .read()
6950 .unwrap_or_else(std::sync::PoisonError::into_inner),
6951 )
6952 }
6953
6954 pub fn session_claim_handle(&self) -> Arc<dyn meerkat_core::handles::SessionClaimHandle> {
6959 Arc::clone(&self.session_claims) as Arc<dyn meerkat_core::handles::SessionClaimHandle>
6960 }
6961
6962 pub fn set_composition_signal_dispatcher(
6965 &self,
6966 dispatcher: composition::MeerkatCompositionSignalDispatcher,
6967 ) {
6968 let mut slot = self
6969 .composition_signal_dispatcher
6970 .write()
6971 .unwrap_or_else(std::sync::PoisonError::into_inner);
6972 *slot = Some(dispatcher);
6973 }
6974
6975 pub(crate) async fn apply_routed_meerkat_input(
6986 &self,
6987 session_id: &SessionId,
6988 input: dsl::MeerkatMachineInput,
6989 ) -> Result<(), dsl_authority::DslTransitionRefusal> {
6990 let _gate_guard = self
6991 .lock_current_durability_ready_session_mutation_gate(session_id)
6992 .await
6993 .map_err(|error| {
6994 dsl_authority::DslTransitionRefusal::other(
6995 "routed_session_not_durability_ready",
6996 format!(
6997 "session `{session_id}` cannot accept routed input until its persistent \
6998 runtime is cold reloaded: {error}"
6999 ),
7000 )
7001 })?;
7002 self.apply_routed_session_dsl_input(session_id, input, "RoutedMeerkatInput")
7003 .await
7004 .map(|_| ())
7005 }
7006
7007 #[cfg(test)]
7008 pub(crate) async fn debug_shared_ingress_authorities(
7009 &self,
7010 session_id: &SessionId,
7011 ) -> Option<(
7012 Arc<std::sync::Mutex<dsl::MeerkatMachineAuthority>>,
7013 crate::driver::ephemeral::SharedIngressDslAuthority,
7014 )> {
7015 let sessions = self.sessions.read().await;
7016 let entry = sessions.get(session_id)?;
7017 let session_authority = Arc::clone(&entry.dsl_authority);
7018 let driver = entry.driver.lock().await;
7019 Some((session_authority, driver.shared_dsl_authority()))
7020 }
7021
7022 fn make_driver(
7024 &self,
7025 runtime_id: LogicalRuntimeId,
7026 dsl_authority: crate::driver::ephemeral::SharedIngressDslAuthority,
7027 initial_runtime_state: RuntimeState,
7028 durability_health: Option<DurabilityHealthHandle>,
7029 ) -> Result<DriverEntry, RuntimeDriverError> {
7030 let control_projection = Arc::new(StdRwLock::new(
7031 crate::driver::ephemeral::RuntimeControlProjection {
7032 phase: initial_runtime_state,
7033 current_run_id: None,
7034 pre_run_phase: None,
7035 },
7036 ));
7037 match (&self.store, &self.blob_store) {
7038 (Some(store), Some(blob_store)) => {
7039 let durability_health =
7040 durability_health.ok_or_else(|| RuntimeDriverError::Internal(
7041 "persistent runtime driver construction requires the registration cold-install durability handle"
7042 .to_string(),
7043 ))?;
7044 let driver = PersistentRuntimeDriver::new_with_control_and_durability_health(
7045 runtime_id,
7046 store.clone(),
7047 blob_store.clone(),
7048 control_projection,
7049 dsl_authority,
7050 durability_health,
7051 );
7052 Ok(DriverEntry::Persistent(driver))
7053 }
7054 _ if durability_health.is_some() => Err(RuntimeDriverError::Internal(
7055 "storeless runtime driver construction received a persistent durability handle"
7056 .to_string(),
7057 )),
7058 _ => Ok(DriverEntry::Ephemeral(
7059 EphemeralRuntimeDriver::new_with_control_and_dsl(
7060 runtime_id,
7061 control_projection,
7062 dsl_authority,
7063 ),
7064 )),
7065 }
7066 }
7067
7068 async fn recover_or_create_ops_state(
7075 &self,
7076 session_id: &SessionId,
7077 runtime_id: &LogicalRuntimeId,
7078 ) -> Result<
7079 (
7080 Arc<crate::ops_lifecycle::RuntimeOpsLifecycleRegistry>,
7081 meerkat_core::RuntimeEpochId,
7082 Arc<meerkat_core::EpochCursorState>,
7083 ),
7084 RuntimeDriverError,
7085 > {
7086 if let Some(ref store) = self.store {
7087 let (registry, epoch_id, cursor_state) = Self::fresh_ops_state();
7088 let initial_snapshot = registry
7094 .capture_persistence_snapshot(epoch_id.clone(), cursor_state.as_ref())
7095 .map_err(|error| {
7096 RuntimeDriverError::Internal(format!(
7097 "failed to capture initial ops lifecycle authority for session {session_id}: {error}"
7098 ))
7099 })?;
7100 let canonical_snapshot = store
7101 .initialize_ops_lifecycle_if_absent(runtime_id, &initial_snapshot)
7102 .await
7103 .map_err(|error| {
7104 RuntimeDriverError::Internal(format!(
7105 "failed to initialize ops lifecycle authority for session {session_id}: {error}"
7106 ))
7107 })?;
7108 let recovered_epoch = canonical_snapshot.epoch_id.clone();
7109 let initialized = recovered_epoch == epoch_id;
7110 let recovered_ops_count = canonical_snapshot.completion_entries.len();
7111 let registry = crate::ops_lifecycle::RuntimeOpsLifecycleRegistry::from_recovered(
7112 canonical_snapshot,
7113 )
7114 .map_err(|error| {
7115 tracing::error!(
7116 %session_id,
7117 %runtime_id,
7118 error = %error,
7119 "failed to recover ops lifecycle through generated authority"
7120 );
7121 RuntimeDriverError::Internal(format!(
7122 "failed to recover ops lifecycle through generated authority: {error}"
7123 ))
7124 })?;
7125 let recovered_cursor_snapshot = registry.completion_cursor_snapshot();
7126 let recovered_cursors = meerkat_core::EpochCursorState::from_recovered(
7127 recovered_cursor_snapshot.agent_applied_cursor,
7128 recovered_cursor_snapshot.runtime_observed_seq,
7129 recovered_cursor_snapshot.runtime_last_injected_seq,
7130 );
7131 tracing::info!(
7132 %session_id,
7133 %runtime_id,
7134 epoch_id = %recovered_epoch,
7135 recovered_ops = recovered_ops_count,
7136 initialized,
7137 "ops lifecycle authority selected from durable store"
7138 );
7139 Ok((
7140 Arc::new(registry),
7141 recovered_epoch,
7142 Arc::new(recovered_cursors),
7143 ))
7144 } else {
7145 Ok((
7146 Arc::new(crate::ops_lifecycle::RuntimeOpsLifecycleRegistry::new()),
7147 meerkat_core::RuntimeEpochId::new(),
7148 Arc::new(meerkat_core::EpochCursorState::new()),
7149 ))
7150 }
7151 }
7152
7153 fn fresh_ops_state() -> (
7154 Arc<crate::ops_lifecycle::RuntimeOpsLifecycleRegistry>,
7155 meerkat_core::RuntimeEpochId,
7156 Arc<meerkat_core::EpochCursorState>,
7157 ) {
7158 let registry = Arc::new(crate::ops_lifecycle::RuntimeOpsLifecycleRegistry::new());
7159 let epoch = meerkat_core::RuntimeEpochId::new();
7160 let cursors = Arc::new(meerkat_core::EpochCursorState::new());
7161 (registry, epoch, cursors)
7162 }
7163
7164 #[allow(clippy::large_futures)]
7165 fn execute_meerkat_machine_command(
7166 &self,
7167 self_handle: Option<Arc<Self>>,
7168 command: MeerkatMachineCommand,
7169 ) -> MeerkatMachineCommandFuture<'_> {
7170 Box::pin(async move {
7171 match command {
7172 MeerkatMachineCommand::EnsureSessionWithExecutor { .. } => {
7173 let self_handle = self_handle.ok_or_else(|| {
7174 MeerkatMachineCommandError::Driver(RuntimeDriverError::Internal(
7175 "EnsureSessionWithExecutor requires Arc<Self> machine handle".into(),
7176 ))
7177 })?;
7178 self_handle
7179 .execute_meerkat_machine_ensure_session_command(command)
7180 .await
7181 .map_err(Into::into)
7182 }
7183 MeerkatMachineCommand::RegisterSession { .. }
7184 | MeerkatMachineCommand::UnregisterSession { .. }
7185 | MeerkatMachineCommand::SetSilentIntents { .. }
7186 | MeerkatMachineCommand::CancelAfterBoundary { .. }
7187 | MeerkatMachineCommand::StopRuntimeExecutor { .. }
7188 | MeerkatMachineCommand::ContainsSession { .. }
7189 | MeerkatMachineCommand::SessionHasExecutor { .. }
7190 | MeerkatMachineCommand::SessionHasComms { .. }
7191 | MeerkatMachineCommand::OpsLifecycleRegistry { .. }
7192 | MeerkatMachineCommand::PrepareBindings { .. }
7193 | MeerkatMachineCommand::PrepareLocalSessionBindings { .. }
7194 | MeerkatMachineCommand::InputState { .. }
7195 | MeerkatMachineCommand::InputStateByIdempotencyKey { .. }
7196 | MeerkatMachineCommand::InteractionTerminalStatus { .. }
7197 | MeerkatMachineCommand::RunTerminalStatus { .. }
7198 | MeerkatMachineCommand::ListActiveInputs { .. }
7199 | MeerkatMachineCommand::ReconfigureSessionLlmIdentity { .. }
7200 | MeerkatMachineCommand::StagePersistentFilter { .. }
7201 | MeerkatMachineCommand::RequestDeferredTools { .. }
7202 | MeerkatMachineCommand::PublishCommittedVisibleSet { .. } => self
7203 .execute_meerkat_machine_session_command(command)
7204 .await
7205 .map_err(Into::into),
7206 MeerkatMachineCommand::SetPeerIngressContext { .. }
7207 | MeerkatMachineCommand::NotifyDrainExited { .. } => {
7208 let self_handle = self_handle.ok_or_else(|| {
7209 MeerkatMachineCommandError::Driver(RuntimeDriverError::Internal(
7210 "drain command requires Arc<Self> machine handle".into(),
7211 ))
7212 })?;
7213 self_handle
7214 .execute_meerkat_machine_drain_command(command)
7215 .await
7216 .map_err(Into::into)
7217 }
7218 MeerkatMachineCommand::AbortAll
7219 | MeerkatMachineCommand::Abort { .. }
7220 | MeerkatMachineCommand::Wait { .. } => self
7221 .execute_meerkat_machine_drain_local_command(command)
7222 .await
7223 .map_err(Into::into),
7224 MeerkatMachineCommand::Ingest { .. }
7225 | MeerkatMachineCommand::PublishEvent { .. }
7226 | MeerkatMachineCommand::Retire { .. }
7227 | MeerkatMachineCommand::Recycle { .. }
7228 | MeerkatMachineCommand::Reset { .. }
7229 | MeerkatMachineCommand::Recover { .. }
7230 | MeerkatMachineCommand::Destroy { .. }
7231 | MeerkatMachineCommand::RuntimeState { .. }
7232 | MeerkatMachineCommand::ResolvedSessionLlmCapabilities { .. }
7233 | MeerkatMachineCommand::ConfigureModelRoutingBaseline { .. }
7234 | MeerkatMachineCommand::SessionModelRoutingStatus { .. }
7235 | MeerkatMachineCommand::RequestSwitchTurn { .. }
7236 | MeerkatMachineCommand::AdmitModelRoutingAssistantTurn { .. }
7237 | MeerkatMachineCommand::BeginImageOperation { .. }
7238 | MeerkatMachineCommand::DenyImageOperationPlan { .. }
7239 | MeerkatMachineCommand::ActivateImageOperationOverride { .. }
7240 | MeerkatMachineCommand::ClassifyImageOperationTerminal { .. }
7241 | MeerkatMachineCommand::CompleteImageOperation { .. }
7242 | MeerkatMachineCommand::RestoreImageOperationOverride { .. }
7243 | MeerkatMachineCommand::LoadBoundaryReceipt { .. } => self
7244 .execute_meerkat_machine_control_command(command)
7245 .await
7246 .map_err(Into::into),
7247 MeerkatMachineCommand::AcceptWithCompletion { .. }
7248 | MeerkatMachineCommand::AcceptWithoutWake { .. } => self
7249 .execute_meerkat_machine_ingress_command(command)
7250 .await
7251 .map_err(Into::into),
7252 }
7253 })
7254 }
7255
7256 pub async fn register_session(
7263 &self,
7264 session_id: SessionId,
7265 ) -> Result<(), RuntimeControlPlaneError> {
7266 match self
7267 .execute_meerkat_machine_command(
7268 None,
7269 MeerkatMachineCommand::RegisterSession { session_id },
7270 )
7271 .await
7272 .map_err(MeerkatMachine::control_plane_error_from_command_error)?
7273 {
7274 MeerkatMachineCommandResult::Unit => Ok(()),
7275 other => Err(RuntimeControlPlaneError::Internal(format!(
7276 "register_session: unexpected command result variant: {other:?}"
7277 ))),
7278 }
7279 }
7280}
7281
7282#[cfg(test)]
7283#[allow(clippy::expect_used, clippy::panic, clippy::unwrap_used)]
7284#[path = "../meerkat_machine_tests.rs"]
7285mod tests;