1use super::*;
2use crate::input_state::StoredInputState;
3use crate::store::{RuntimeSessionAuthority, RuntimeSessionPersistenceProfile};
4
5#[cfg_attr(not(target_arch = "wasm32"), async_trait::async_trait)]
6#[cfg_attr(target_arch = "wasm32", async_trait::async_trait(?Send))]
7impl SessionServiceRuntimeExt for MeerkatMachine {
8 async fn accept_input(
9 &self,
10 session_id: &SessionId,
11 input: Input,
12 ) -> Result<AcceptOutcome, RuntimeDriverError> {
13 match self
14 .execute_meerkat_machine_command(
15 None,
16 MeerkatMachineCommand::AcceptWithCompletion {
17 session_id: session_id.clone(),
18 input,
19 register_completion: false,
20 member_residency: MemberResidencyExpectation::Unfenced,
21 expected_attachment: None,
22 },
23 )
24 .await
25 .map_err(MeerkatMachine::driver_error_from_command_error)?
26 {
27 MeerkatMachineCommandResult::AcceptWithCompletion {
28 outcome,
29 handle: _,
30 admission_signal: _,
31 } => Ok(outcome),
32 other => Err(RuntimeDriverError::Internal(format!(
33 "unexpected MeerkatMachineCommandResult for SessionServiceRuntimeExt::accept_input: {other:?}"
34 ))),
35 }
36 }
37
38 async fn accept_input_with_completion(
39 &self,
40 session_id: &SessionId,
41 input: Input,
42 ) -> Result<(AcceptOutcome, Option<crate::completion::CompletionHandle>), RuntimeDriverError>
43 {
44 tracing::debug!(
45 session_id = %session_id,
46 input_id = %input.id(),
47 "SessionServiceRuntimeExt::accept_input_with_completion entered"
48 );
49 self.accept_input_with_completion_boxed(session_id, input)
50 .await
51 }
52
53 async fn runtime_state(
54 &self,
55 session_id: &SessionId,
56 ) -> Result<RuntimeState, RuntimeDriverError> {
57 let runtime_id = MeerkatMachine::logical_runtime_id(session_id);
58 match self
59 .execute_meerkat_machine_command(
60 None,
61 MeerkatMachineCommand::RuntimeState { runtime_id },
62 )
63 .await
64 .map_err(MeerkatMachine::driver_error_from_command_error)?
65 {
66 MeerkatMachineCommandResult::RuntimeState(state) => Ok(state),
67 other => Err(RuntimeDriverError::Internal(format!(
68 "unexpected MeerkatMachineCommandResult for SessionServiceRuntimeExt::runtime_state: {other:?}"
69 ))),
70 }
71 }
72
73 async fn retire_runtime(
74 &self,
75 session_id: &SessionId,
76 ) -> Result<RetireReport, RuntimeDriverError> {
77 let runtime_id = MeerkatMachine::logical_runtime_id(session_id);
78 match self
79 .execute_meerkat_machine_command(None, MeerkatMachineCommand::Retire { runtime_id })
80 .await
81 .map_err(MeerkatMachine::driver_error_from_command_error)?
82 {
83 MeerkatMachineCommandResult::RetireReport(report) => Ok(report),
84 other => Err(RuntimeDriverError::Internal(format!(
85 "unexpected MeerkatMachineCommandResult for SessionServiceRuntimeExt::retire_runtime: {other:?}"
86 ))),
87 }
88 }
89
90 async fn reset_runtime(
91 &self,
92 session_id: &SessionId,
93 ) -> Result<ResetReport, RuntimeDriverError> {
94 let runtime_id = MeerkatMachine::logical_runtime_id(session_id);
95 match self
96 .execute_meerkat_machine_command(None, MeerkatMachineCommand::Reset { runtime_id })
97 .await
98 .map_err(MeerkatMachine::driver_error_from_command_error)?
99 {
100 MeerkatMachineCommandResult::ResetReport(report) => Ok(report),
101 other => Err(RuntimeDriverError::Internal(format!(
102 "unexpected MeerkatMachineCommandResult for SessionServiceRuntimeExt::reset_runtime: {other:?}"
103 ))),
104 }
105 }
106
107 async fn input_state(
108 &self,
109 session_id: &SessionId,
110 input_id: &InputId,
111 ) -> Result<Option<StoredInputState>, RuntimeDriverError> {
112 match self
113 .execute_meerkat_machine_command(
114 None,
115 MeerkatMachineCommand::InputState {
116 session_id: session_id.clone(),
117 input_id: input_id.clone(),
118 },
119 )
120 .await
121 .map_err(MeerkatMachine::driver_error_from_command_error)?
122 {
123 MeerkatMachineCommandResult::InputState(state) => Ok(state),
124 other => Err(RuntimeDriverError::Internal(format!(
125 "unexpected MeerkatMachineCommandResult for SessionServiceRuntimeExt::input_state: {other:?}"
126 ))),
127 }
128 }
129
130 async fn input_terminal_completion(
131 &self,
132 session_id: &SessionId,
133 input_id: &InputId,
134 ) -> Result<Option<crate::completion::CompletionOutcome>, RuntimeDriverError> {
135 let driver = {
136 let sessions = self.sessions.read().await;
137 sessions.get(session_id).map(|entry| entry.driver.clone())
138 };
139 if let Some(driver) = driver {
140 return driver
141 .lock()
142 .await
143 .exact_input_terminal_completion_outcome(input_id);
144 }
145
146 let Some(store) = self.store.as_ref() else {
147 return Err(RuntimeDriverError::NotReady {
148 state: RuntimeState::Destroyed,
149 });
150 };
151 let runtime_id = Self::logical_runtime_id(session_id);
152 let load = |error: crate::store::RuntimeStoreError| match error {
153 crate::store::RuntimeStoreError::Unsupported(reason) => {
154 RuntimeDriverError::RecoveryRepairBlocked {
155 evidence_digest: None,
156 reason: format!(
157 "runtime store cannot load one exact terminal completion batch: {reason}"
158 ),
159 }
160 }
161 error => RuntimeDriverError::Internal(format!(
162 "exact terminal completion witness read failed for {runtime_id}: {error}"
163 )),
164 };
165 let mut target_rows = store
166 .load_input_states_by_ids(&runtime_id, std::slice::from_ref(input_id))
167 .await
168 .map_err(load)?;
169 let Some(target) = target_rows.pop().ok_or_else(|| {
170 RuntimeDriverError::Internal(
171 "exact terminal completion target read returned the wrong cardinality".to_string(),
172 )
173 })?
174 else {
175 let lifecycle = store
176 .load_machine_lifecycle_record(&runtime_id)
177 .await
178 .map_err(load)?;
179 return if lifecycle.is_some() {
180 Ok(None)
181 } else {
182 Err(RuntimeDriverError::NotFound {
183 runtime_id: runtime_id.clone(),
184 })
185 };
186 };
187 let Some(target_completion) = target.state.terminal_completion.as_ref() else {
188 return crate::input_state::input_terminal_completion_outcome(&[target], input_id)
189 .map_err(|error| match error {
190 error @ crate::input_state::InputTerminalCompletionReadError::MigratedReceiptUnavailable => {
191 RuntimeDriverError::RecoveryRepairBlocked {
192 evidence_digest: None,
193 reason: error.to_string(),
194 }
195 }
196 crate::input_state::InputTerminalCompletionReadError::Corrupt(reason) => {
197 RuntimeDriverError::RecoveryCorruption { reason }
198 }
199 });
200 };
201 let owner_input_id = target_completion.owner_input_id.clone();
202 let owner = if owner_input_id == *input_id {
203 target
204 } else {
205 let mut owner_rows = store
206 .load_input_states_by_ids(&runtime_id, std::slice::from_ref(&owner_input_id))
207 .await
208 .map_err(load)?;
209 owner_rows
210 .pop()
211 .ok_or_else(|| {
212 RuntimeDriverError::Internal(
213 "exact terminal completion owner read returned the wrong cardinality"
214 .to_string(),
215 )
216 })?
217 .ok_or_else(|| RuntimeDriverError::RecoveryCorruption {
218 reason: "terminal completion target lost its canonical durable owner row"
219 .to_string(),
220 })?
221 };
222 let recipient_ids = owner
223 .state
224 .terminal_completion
225 .as_ref()
226 .and_then(|completion| completion.completion_input_ids.clone())
227 .ok_or_else(|| RuntimeDriverError::RecoveryCorruption {
228 reason: "terminal completion durable owner lost its recipient set".to_string(),
229 })?;
230 let recipient_rows = store
231 .load_input_states_by_ids(&runtime_id, &recipient_ids)
232 .await
233 .map_err(load)?;
234 if recipient_rows.len() != recipient_ids.len() {
235 return Err(RuntimeDriverError::Internal(
236 "exact terminal completion batch read returned the wrong cardinality".to_string(),
237 ));
238 }
239 let witnesses = recipient_rows
240 .into_iter()
241 .zip(recipient_ids)
242 .map(|(stored, recipient_id)| {
243 stored.ok_or_else(|| RuntimeDriverError::RecoveryCorruption {
244 reason: format!(
245 "terminal completion durable batch lost recipient row {recipient_id}"
246 ),
247 })
248 })
249 .collect::<Result<Vec<_>, _>>()?;
250 crate::input_state::input_terminal_completion_outcome(&witnesses, input_id).map_err(
251 |error| match error {
252 error @ crate::input_state::InputTerminalCompletionReadError::MigratedReceiptUnavailable => {
253 RuntimeDriverError::RecoveryRepairBlocked {
254 evidence_digest: None,
255 reason: error.to_string(),
256 }
257 }
258 crate::input_state::InputTerminalCompletionReadError::Corrupt(reason) => {
259 RuntimeDriverError::RecoveryCorruption { reason }
260 }
261 },
262 )
263 }
264
265 async fn input_state_by_idempotency_key(
266 &self,
267 session_id: &SessionId,
268 idempotency_key: &str,
269 ) -> Result<Option<StoredInputState>, RuntimeDriverError> {
270 match self
271 .execute_meerkat_machine_command(
272 None,
273 MeerkatMachineCommand::InputStateByIdempotencyKey {
274 session_id: session_id.clone(),
275 idempotency_key: idempotency_key.to_string(),
276 },
277 )
278 .await
279 .map_err(MeerkatMachine::driver_error_from_command_error)?
280 {
281 MeerkatMachineCommandResult::InputState(state) => Ok(state),
282 other => Err(RuntimeDriverError::Internal(format!(
283 "unexpected MeerkatMachineCommandResult for SessionServiceRuntimeExt::input_state_by_idempotency_key: {other:?}"
284 ))),
285 }
286 }
287
288 async fn interaction_terminal_status(
289 &self,
290 session_id: &SessionId,
291 selector: crate::terminal_status::InteractionSelector,
292 ) -> Result<
293 Option<crate::terminal_status::Sourced<crate::terminal_status::InteractionTerminalReport>>,
294 RuntimeDriverError,
295 > {
296 match self
297 .execute_meerkat_machine_command(
298 None,
299 MeerkatMachineCommand::InteractionTerminalStatus {
300 session_id: session_id.clone(),
301 selector,
302 },
303 )
304 .await
305 .map_err(MeerkatMachine::driver_error_from_command_error)?
306 {
307 MeerkatMachineCommandResult::InteractionTerminalStatus(report) => Ok(report),
308 other => Err(RuntimeDriverError::Internal(format!(
309 "unexpected MeerkatMachineCommandResult for SessionServiceRuntimeExt::interaction_terminal_status: {other:?}"
310 ))),
311 }
312 }
313
314 async fn run_terminal_status(
315 &self,
316 session_id: &SessionId,
317 run_id: &meerkat_core::lifecycle::RunId,
318 ) -> Result<
319 crate::terminal_status::Sourced<crate::terminal_status::RunTerminalReport>,
320 RuntimeDriverError,
321 > {
322 match self
323 .execute_meerkat_machine_command(
324 None,
325 MeerkatMachineCommand::RunTerminalStatus {
326 session_id: session_id.clone(),
327 run_id: run_id.clone(),
328 },
329 )
330 .await
331 .map_err(MeerkatMachine::driver_error_from_command_error)?
332 {
333 MeerkatMachineCommandResult::RunTerminalStatus(report) => Ok(report),
334 other => Err(RuntimeDriverError::Internal(format!(
335 "unexpected MeerkatMachineCommandResult for SessionServiceRuntimeExt::run_terminal_status: {other:?}"
336 ))),
337 }
338 }
339
340 async fn list_active_inputs(
341 &self,
342 session_id: &SessionId,
343 ) -> Result<Vec<InputId>, RuntimeDriverError> {
344 match self
345 .execute_meerkat_machine_command(
346 None,
347 MeerkatMachineCommand::ListActiveInputs {
348 session_id: session_id.clone(),
349 },
350 )
351 .await
352 .map_err(MeerkatMachine::driver_error_from_command_error)?
353 {
354 MeerkatMachineCommandResult::ActiveInputs(inputs) => Ok(inputs),
355 other => Err(RuntimeDriverError::Internal(format!(
356 "unexpected MeerkatMachineCommandResult for SessionServiceRuntimeExt::list_active_inputs: {other:?}"
357 ))),
358 }
359 }
360
361 async fn reconfigure_session_llm_identity(
362 &self,
363 session_id: &SessionId,
364 request: SessionLlmReconfigureRequest,
365 ) -> Result<SessionLlmReconfigureReport, RuntimeDriverError> {
366 let host = self.llm_reconfigure_host()?;
367 let _turn_finalization_guard = host.acquire_turn_finalization_boundary(session_id).await?;
368 self.reconfigure_session_llm_identity_under_turn_finalization_boundary(session_id, request)
369 .await
370 }
371
372 async fn resolved_session_llm_capabilities(
373 &self,
374 session_id: &SessionId,
375 ) -> Result<Option<SessionLlmCapabilitySurface>, RuntimeDriverError> {
376 match self
377 .execute_meerkat_machine_command(
378 None,
379 MeerkatMachineCommand::ResolvedSessionLlmCapabilities {
380 session_id: session_id.clone(),
381 },
382 )
383 .await
384 .map_err(MeerkatMachine::driver_error_from_command_error)?
385 {
386 MeerkatMachineCommandResult::ResolvedSessionLlmCapabilities(capabilities) => {
387 Ok(capabilities)
388 }
389 other => Err(RuntimeDriverError::Internal(format!(
390 "unexpected MeerkatMachineCommandResult for SessionServiceRuntimeExt::resolved_session_llm_capabilities: {other:?}"
391 ))),
392 }
393 }
394
395 async fn configure_model_routing_baseline(
396 &self,
397 session_id: &SessionId,
398 baseline_model: meerkat_core::lifecycle::run_primitive::ModelId,
399 realtime_capable: bool,
400 ) -> Result<(), RuntimeDriverError> {
401 match self
402 .execute_meerkat_machine_command(
403 None,
404 MeerkatMachineCommand::ConfigureModelRoutingBaseline {
405 session_id: session_id.clone(),
406 baseline_model,
407 realtime_capable,
408 },
409 )
410 .await
411 .map_err(MeerkatMachine::driver_error_from_command_error)?
412 {
413 MeerkatMachineCommandResult::Unit => Ok(()),
414 other => Err(RuntimeDriverError::Internal(format!(
415 "unexpected MeerkatMachineCommandResult for SessionServiceRuntimeExt::configure_model_routing_baseline: {other:?}"
416 ))),
417 }
418 }
419
420 async fn session_model_routing_status(
421 &self,
422 session_id: &SessionId,
423 ) -> Result<meerkat_core::image_generation::SessionModelRoutingStatus, RuntimeDriverError> {
424 match self
425 .execute_meerkat_machine_command(
426 None,
427 MeerkatMachineCommand::SessionModelRoutingStatus {
428 session_id: session_id.clone(),
429 },
430 )
431 .await
432 .map_err(MeerkatMachine::driver_error_from_command_error)?
433 {
434 MeerkatMachineCommandResult::SessionModelRoutingStatus(status) => Ok(status),
435 other => Err(RuntimeDriverError::Internal(format!(
436 "unexpected MeerkatMachineCommandResult for SessionServiceRuntimeExt::session_model_routing_status: {other:?}"
437 ))),
438 }
439 }
440
441 async fn request_switch_turn(
442 &self,
443 session_id: &SessionId,
444 request: crate::meerkat_machine_types::SwitchTurnRequest,
445 ) -> Result<meerkat_core::image_generation::SwitchTurnControlResult, RuntimeDriverError> {
446 let _turn_finalization_guard = if matches!(
452 &request.intent.duration,
453 meerkat_core::image_generation::SwitchTurnDuration::UntilChanged
454 ) {
455 Some(
456 self.llm_reconfigure_host()?
457 .acquire_turn_finalization_boundary(session_id)
458 .await?,
459 )
460 } else {
461 None
462 };
463 match self
464 .execute_meerkat_machine_command(
465 None,
466 MeerkatMachineCommand::RequestSwitchTurn {
467 session_id: session_id.clone(),
468 request: Box::new(request),
469 },
470 )
471 .await
472 .map_err(MeerkatMachine::driver_error_from_command_error)?
473 {
474 MeerkatMachineCommandResult::SwitchTurnControlResult(result) => Ok(result),
475 other => Err(RuntimeDriverError::Internal(format!(
476 "unexpected MeerkatMachineCommandResult for SessionServiceRuntimeExt::request_switch_turn: {other:?}"
477 ))),
478 }
479 }
480
481 async fn admit_model_routing_assistant_turn(
482 &self,
483 session_id: &SessionId,
484 ) -> Result<(), RuntimeDriverError> {
485 match self
486 .execute_meerkat_machine_command(
487 None,
488 MeerkatMachineCommand::AdmitModelRoutingAssistantTurn {
489 session_id: session_id.clone(),
490 },
491 )
492 .await
493 .map_err(MeerkatMachine::driver_error_from_command_error)?
494 {
495 MeerkatMachineCommandResult::Unit => Ok(()),
496 other => Err(RuntimeDriverError::Internal(format!(
497 "unexpected MeerkatMachineCommandResult for SessionServiceRuntimeExt::admit_model_routing_assistant_turn: {other:?}"
498 ))),
499 }
500 }
501
502 async fn begin_image_operation(
503 &self,
504 session_id: &SessionId,
505 request: crate::meerkat_machine_types::ImageOperationRoutingRequest,
506 ) -> Result<crate::meerkat_machine_types::ImageOperationRoutingResult, RuntimeDriverError> {
507 match self
508 .execute_meerkat_machine_command(
509 None,
510 MeerkatMachineCommand::BeginImageOperation {
511 session_id: session_id.clone(),
512 request: Box::new(request),
513 },
514 )
515 .await
516 .map_err(MeerkatMachine::driver_error_from_command_error)?
517 {
518 MeerkatMachineCommandResult::ImageOperationRoutingResult(result) => Ok(result),
519 other => Err(RuntimeDriverError::Internal(format!(
520 "unexpected MeerkatMachineCommandResult for SessionServiceRuntimeExt::begin_image_operation: {other:?}"
521 ))),
522 }
523 }
524
525 async fn deny_image_operation_plan(
526 &self,
527 session_id: &SessionId,
528 operation_id: meerkat_core::image_generation::ImageOperationId,
529 reason: meerkat_core::image_generation::ImageOperationDenialReason,
530 ) -> Result<meerkat_core::image_generation::ImageOperationPhase, RuntimeDriverError> {
531 match self
532 .execute_meerkat_machine_command(
533 None,
534 MeerkatMachineCommand::DenyImageOperationPlan {
535 session_id: session_id.clone(),
536 operation_id,
537 reason,
538 },
539 )
540 .await
541 .map_err(MeerkatMachine::driver_error_from_command_error)?
542 {
543 MeerkatMachineCommandResult::ImageOperationPhase(phase) => Ok(phase),
544 other => Err(RuntimeDriverError::Internal(format!(
545 "unexpected MeerkatMachineCommandResult for SessionServiceRuntimeExt::deny_image_operation_plan: {other:?}"
546 ))),
547 }
548 }
549
550 async fn activate_image_operation_override(
551 &self,
552 session_id: &SessionId,
553 operation_id: meerkat_core::image_generation::ImageOperationId,
554 ) -> Result<meerkat_core::image_generation::ImageOperationPhase, RuntimeDriverError> {
555 match self
556 .execute_meerkat_machine_command(
557 None,
558 MeerkatMachineCommand::ActivateImageOperationOverride {
559 session_id: session_id.clone(),
560 operation_id,
561 },
562 )
563 .await
564 .map_err(MeerkatMachine::driver_error_from_command_error)?
565 {
566 MeerkatMachineCommandResult::ImageOperationPhase(phase) => Ok(phase),
567 other => Err(RuntimeDriverError::Internal(format!(
568 "unexpected MeerkatMachineCommandResult for SessionServiceRuntimeExt::activate_image_operation_override: {other:?}"
569 ))),
570 }
571 }
572
573 async fn complete_image_operation(
574 &self,
575 session_id: &SessionId,
576 operation_id: meerkat_core::image_generation::ImageOperationId,
577 terminal: meerkat_core::image_generation::ImageOperationTerminalClass,
578 ) -> Result<meerkat_core::image_generation::ImageOperationPhase, RuntimeDriverError> {
579 match self
580 .execute_meerkat_machine_command(
581 None,
582 MeerkatMachineCommand::CompleteImageOperation {
583 session_id: session_id.clone(),
584 operation_id,
585 terminal,
586 },
587 )
588 .await
589 .map_err(MeerkatMachine::driver_error_from_command_error)?
590 {
591 MeerkatMachineCommandResult::ImageOperationPhase(phase) => Ok(phase),
592 other => Err(RuntimeDriverError::Internal(format!(
593 "unexpected MeerkatMachineCommandResult for SessionServiceRuntimeExt::complete_image_operation: {other:?}"
594 ))),
595 }
596 }
597
598 async fn classify_image_operation_terminal(
599 &self,
600 session_id: &SessionId,
601 operation_id: meerkat_core::image_generation::ImageOperationId,
602 observation: meerkat_core::image_generation::ImageProviderTerminalObservation,
603 provider_text: meerkat_core::image_generation::ProviderTextDisposition,
604 ) -> Result<meerkat_core::image_generation::ImageOperationTerminalClass, RuntimeDriverError>
605 {
606 match self
607 .execute_meerkat_machine_command(
608 None,
609 MeerkatMachineCommand::ClassifyImageOperationTerminal {
610 session_id: session_id.clone(),
611 operation_id,
612 observation,
613 provider_text,
614 },
615 )
616 .await
617 .map_err(MeerkatMachine::driver_error_from_command_error)?
618 {
619 MeerkatMachineCommandResult::ImageOperationTerminalClass(terminal) => Ok(terminal),
620 other => Err(RuntimeDriverError::Internal(format!(
621 "unexpected MeerkatMachineCommandResult for SessionServiceRuntimeExt::classify_image_operation_terminal: {other:?}"
622 ))),
623 }
624 }
625
626 async fn restore_image_operation_override(
627 &self,
628 session_id: &SessionId,
629 operation_id: meerkat_core::image_generation::ImageOperationId,
630 ) -> Result<meerkat_core::image_generation::ImageOperationPhase, RuntimeDriverError> {
631 match self
632 .execute_meerkat_machine_command(
633 None,
634 MeerkatMachineCommand::RestoreImageOperationOverride {
635 session_id: session_id.clone(),
636 operation_id,
637 },
638 )
639 .await
640 .map_err(MeerkatMachine::driver_error_from_command_error)?
641 {
642 MeerkatMachineCommandResult::ImageOperationPhase(phase) => Ok(phase),
643 other => Err(RuntimeDriverError::Internal(format!(
644 "unexpected MeerkatMachineCommandResult for SessionServiceRuntimeExt::restore_image_operation_override: {other:?}"
645 ))),
646 }
647 }
648}
649
650impl MeerkatMachine {
655 pub(crate) fn logical_runtime_id(session_id: &SessionId) -> LogicalRuntimeId {
656 LogicalRuntimeId::for_session(session_id)
657 }
658
659 pub(super) fn post_admission_signal_from_effects(
660 effects: &[crate::meerkat_machine::dsl::MeerkatMachineEffect],
661 ) -> crate::driver::ephemeral::PostAdmissionSignal {
662 effects
663 .iter()
664 .find_map(|effect| match effect {
665 crate::meerkat_machine::dsl::MeerkatMachineEffect::PostAdmissionSignal {
666 signal,
667 } => Some(match signal {
668 crate::meerkat_machine::dsl::PostAdmissionSignalKind::WakeLoop => {
669 crate::driver::ephemeral::PostAdmissionSignal::WakeLoop
670 }
671 crate::meerkat_machine::dsl::PostAdmissionSignalKind::InterruptYielding => {
672 crate::driver::ephemeral::PostAdmissionSignal::InterruptYielding
673 }
674 crate::meerkat_machine::dsl::PostAdmissionSignalKind::RequestImmediateProcessing => {
675 crate::driver::ephemeral::PostAdmissionSignal::RequestImmediateProcessing
676 }
677 }),
678 _ => None,
679 })
680 .unwrap_or(crate::driver::ephemeral::PostAdmissionSignal::None)
681 }
682
683 pub(super) fn driver_error_from_command_error(
684 err: MeerkatMachineCommandError,
685 ) -> RuntimeDriverError {
686 match err {
687 MeerkatMachineCommandError::Driver(err) => err,
688 MeerkatMachineCommandError::Control(err) => {
689 Self::driver_error_from_control_plane_error(err)
690 }
691 }
692 }
693
694 pub(super) fn control_plane_error_from_command_error(
695 err: MeerkatMachineCommandError,
696 ) -> RuntimeControlPlaneError {
697 match err {
698 MeerkatMachineCommandError::Control(err) => err,
699 MeerkatMachineCommandError::Driver(err) => {
700 RuntimeControlPlaneError::Internal(err.to_string())
701 }
702 }
703 }
704
705 pub(super) fn driver_error_from_control_plane_error(
706 err: RuntimeControlPlaneError,
707 ) -> RuntimeDriverError {
708 match err {
709 RuntimeControlPlaneError::NotFound(runtime_id) => {
710 RuntimeDriverError::NotFound { runtime_id }
711 }
712 RuntimeControlPlaneError::InvalidState { state } => {
713 RuntimeDriverError::NotReady { state }
714 }
715 RuntimeControlPlaneError::StoreError(message)
716 | RuntimeControlPlaneError::Internal(message) => RuntimeDriverError::Internal(message),
717 }
718 }
719
720 pub(super) async fn resolve_session_id(
722 &self,
723 runtime_id: &LogicalRuntimeId,
724 ) -> Result<SessionId, RuntimeControlPlaneError> {
725 let sessions = self.sessions.read().await;
726 sessions
727 .iter()
728 .find_map(|(session_id, entry)| {
729 (&entry.runtime_id == runtime_id).then(|| session_id.clone())
730 })
731 .ok_or_else(|| RuntimeControlPlaneError::NotFound(runtime_id.clone()))
732 }
733
734 pub(super) async fn existing_session_runtime_state(
735 &self,
736 session_id: &SessionId,
737 ) -> Option<RuntimeState> {
738 let sessions = self.sessions.read().await;
739 let entry = sessions.get(session_id)?;
740 let control = entry.control_snapshot();
745 let authority = entry
746 .dsl_authority
747 .lock()
748 .unwrap_or_else(std::sync::PoisonError::into_inner);
749 let dsl_phase = dsl_authority::runtime_phase_from_authority(&authority);
750 let dsl_pre_run_phase = dsl_authority::pre_run_phase_from_authority(&authority);
751 match crate::meerkat_machine::resolve_visible_runtime_phase(
758 dsl_phase,
759 dsl_pre_run_phase,
760 control.phase,
761 control.pre_run_phase,
762 self.has_runtime_persistence(),
763 ) {
764 Ok(plan) => Some(plan.selected_raw_phase),
765 Err(reason) => {
766 tracing::error!(%session_id, %reason, "MeerkatMachine visible runtime phase resolution failed; failing closed to Destroyed");
767 Some(RuntimeState::Destroyed)
768 }
769 }
770 }
771
772 pub(super) async fn existing_session_visible_runtime_state(
773 &self,
774 session_id: &SessionId,
775 ) -> Option<RuntimeState> {
776 let sessions = self.sessions.read().await;
777 let entry = sessions.get(session_id)?;
778 let control = entry.control_snapshot();
779 let authority = entry
780 .dsl_authority
781 .lock()
782 .unwrap_or_else(std::sync::PoisonError::into_inner);
783 let dsl_phase = dsl_authority::runtime_phase_from_authority(&authority);
784 let dsl_pre_run_phase = dsl_authority::pre_run_phase_from_authority(&authority);
785 match crate::meerkat_machine::resolve_visible_runtime_phase(
791 dsl_phase,
792 dsl_pre_run_phase,
793 control.phase,
794 control.pre_run_phase,
795 self.has_runtime_persistence(),
796 ) {
797 Ok(plan) => Some(plan.visible_phase),
798 Err(reason) => {
799 tracing::error!(%session_id, %reason, "MeerkatMachine visible runtime phase resolution failed; failing closed to Destroyed");
800 Some(RuntimeState::Destroyed)
801 }
802 }
803 }
804
805 pub(super) async fn lookup_entry(
808 &self,
809 runtime_id: &LogicalRuntimeId,
810 ) -> Result<
811 (
812 SessionId,
813 SharedDriver,
814 SharedCompletionRegistry,
815 Option<mpsc::Sender<()>>,
816 ),
817 RuntimeControlPlaneError,
818 > {
819 let sessions = self.sessions.read().await;
820 let (session_id, entry) = sessions
821 .iter()
822 .find(|(_, entry)| &entry.runtime_id == runtime_id)
823 .ok_or_else(|| RuntimeControlPlaneError::NotFound(runtime_id.clone()))?;
824 Ok((
825 session_id.clone(),
826 entry.driver.clone(),
827 entry.completions.clone(),
828 entry.wake_sender(),
829 ))
830 }
831
832 async fn capture_archive_lease_entry_under_mutation_guard(
837 &self,
838 runtime_id: &LogicalRuntimeId,
839 session_id: &SessionId,
840 expected_driver: &SharedDriver,
841 _mutation_guard: &crate::tokio::sync::OwnedMutexGuard<()>,
842 ) -> Result<
843 (
844 SharedDriver,
845 SharedCompletionRegistry,
846 Option<mpsc::Sender<()>>,
847 Option<Arc<dyn meerkat_core::lifecycle::CoreExecutorPublicationHandle>>,
848 ),
849 RuntimeControlPlaneError,
850 > {
851 let sessions = self.sessions.read().await;
852 let entry = sessions.get(session_id).ok_or_else(|| {
853 RuntimeControlPlaneError::Internal(format!(
854 "runtime {runtime_id} disappeared while its archive/retire mutation gate was held"
855 ))
856 })?;
857 if &entry.runtime_id != runtime_id || !Arc::ptr_eq(&entry.driver, expected_driver) {
858 return Err(RuntimeControlPlaneError::Internal(format!(
859 "runtime {runtime_id} changed authority while its archive/retire mutation gate was held"
860 )));
861 }
862 Ok((
863 Arc::clone(&entry.driver),
864 Arc::clone(&entry.completions),
865 entry.wake_sender(),
866 entry.publication_handle(),
867 ))
868 }
869
870 async fn reject_unregister_overlap_under_registration_transaction(
880 &self,
881 session_id: &SessionId,
882 ) -> Result<(), RuntimeControlPlaneError> {
883 let (blocked, coordinator_present, pending_finalization, runtime_state) = {
884 let sessions = self.sessions.read().await;
885 let entry = sessions.get(session_id).ok_or_else(|| {
886 RuntimeControlPlaneError::NotFound(LogicalRuntimeId::for_session(session_id))
887 })?;
888 let registration_phase = entry
889 .dsl_authority
890 .lock()
891 .unwrap_or_else(std::sync::PoisonError::into_inner)
892 .state()
893 .registration_phase;
894 let coordinator_present = entry.unregister_coordinator.is_some();
895 let pending_finalization = entry.pending_unregister_finalization.is_some();
896 (
897 coordinator_present
898 || pending_finalization
899 || registration_phase
900 == crate::meerkat_machine::dsl::RegistrationPhase::Draining,
901 coordinator_present,
902 pending_finalization,
903 entry.control_snapshot().phase,
904 )
905 };
906 if !blocked {
907 return Ok(());
908 }
909 if !coordinator_present && !pending_finalization {
910 let cleanup_spawner = super::MachineCleanupTaskSpawner::acquire()
911 .map_err(|error| RuntimeControlPlaneError::Internal(error.to_string()))?;
912 let machine = self.clone();
913 let retry_session_id = session_id.clone();
914 drop(cleanup_spawner.spawn(async move {
915 if let Err(error) = machine.try_unregister_session(&retry_session_id).await {
916 tracing::warn!(
917 session_id = %retry_session_id,
918 %error,
919 "cold unregister retry started by lifecycle overlap failed"
920 );
921 }
922 }));
923 }
924 if pending_finalization {
925 return Err(RuntimeControlPlaneError::Internal(
926 RuntimeDriverError::UnregisterFinalizationOutcomeUnknown {
927 reason: format!(
928 "session {session_id} retains an ambiguous unregister finalization; retry unregister before applying any other lifecycle mutation"
929 ),
930 }
931 .to_string(),
932 ));
933 }
934 Err(RuntimeControlPlaneError::InvalidState {
935 state: runtime_state,
936 })
937 }
938
939 pub async fn prepare_session_archive_lease(
942 &self,
943 session_id: &SessionId,
944 ) -> Result<Option<super::MachineSessionArchiveLease>, RuntimeControlPlaneError> {
945 let runtime_id = LogicalRuntimeId::for_session(session_id);
946 let registration_transaction_guard =
951 self.lock_session_registration_transaction(session_id).await;
952 let mut recovered_registration_for_archive = false;
953 let (resolved_session_id, driver, _, _) = match self.lookup_entry(&runtime_id).await {
954 Ok(parts) => parts,
955 Err(RuntimeControlPlaneError::NotFound(_)) => {
956 let Some(store) = self.store.as_ref() else {
965 return Ok(None);
966 };
967 let durable_lifecycle =
968 crate::store::load_machine_lifecycle(store.as_ref(), &runtime_id)
969 .await
970 .map_err(|error| RuntimeControlPlaneError::Internal(error.to_string()))?;
971 let lifecycle_requires_archive = durable_lifecycle.as_ref().is_some_and(
972 super::session_management::machine_lifecycle_has_runtime_archive_residue,
973 );
974 let lifecycle_is_quiescent = durable_lifecycle.as_ref().is_some_and(|lifecycle| {
975 matches!(
976 lifecycle.runtime_state(),
977 RuntimeState::Retired | RuntimeState::Destroyed
978 )
979 });
980 let durable_session_authority = if lifecycle_is_quiescent {
981 None
982 } else {
983 match store.session_persistence_profile() {
984 RuntimeSessionPersistenceProfile::WholeBlobV1 => store
985 .load_whole_blob_store_authority(&runtime_id)
986 .await
987 .map(|authority| authority.map(RuntimeSessionAuthority::WholeBlob))
988 .map_err(|error| {
989 RuntimeControlPlaneError::Internal(error.to_string())
990 })?,
991 RuntimeSessionPersistenceProfile::HeadCanonicalV1 => store
992 .load_session_boundary_authority(&runtime_id)
993 .await
994 .map_err(|error| {
995 RuntimeControlPlaneError::Internal(error.to_string())
996 })?,
997 }
998 };
999 if let Some(authority) = durable_session_authority.as_ref()
1000 && (authority.session_id() != session_id
1001 || authority.profile() != store.session_persistence_profile())
1002 {
1003 return Err(RuntimeControlPlaneError::Internal(format!(
1004 "runtime {runtime_id} returned mismatched session-boundary authority while archiving {session_id}"
1005 )));
1006 }
1007 if !lifecycle_requires_archive && durable_session_authority.is_none() {
1008 return Ok(None);
1009 }
1010 recovered_registration_for_archive = self
1019 .register_session_inner_under_registration_transaction(session_id.clone(), None)
1020 .await
1021 .map_err(|error| RuntimeControlPlaneError::Internal(error.to_string()))?
1022 .inserted();
1023 self.lookup_entry(&runtime_id).await?
1024 }
1025 Err(error) => return Err(error),
1026 };
1027 if &resolved_session_id != session_id {
1028 return Err(RuntimeControlPlaneError::Internal(format!(
1029 "runtime {runtime_id} resolved to unexpected session {resolved_session_id} while archiving {session_id}"
1030 )));
1031 }
1032 self.reject_unregister_overlap_under_registration_transaction(&resolved_session_id)
1033 .await?;
1034 #[cfg(test)]
1035 self.run_control_command_after_logical_lookup_test_hook(
1036 ControlCommandLookupTestKind::Retire,
1037 &resolved_session_id,
1038 )
1039 .await;
1040 #[cfg(feature = "live")]
1041 let live_lifecycle_lease = Some(
1042 self.acquire_member_live_disposal_lease(&resolved_session_id)
1043 .await
1044 .map_err(|error| RuntimeControlPlaneError::Internal(error.to_string()))?,
1045 );
1046 #[cfg(not(feature = "live"))]
1047 let live_lifecycle_lease = None;
1048 let mutation_guard = self
1049 .lock_current_session_driver_gate(&resolved_session_id, &driver)
1050 .await
1051 .map_err(|error| RuntimeControlPlaneError::Internal(error.to_string()))?;
1052 let (driver, completions, wake_tx, publication_handle) = self
1053 .capture_archive_lease_entry_under_mutation_guard(
1054 &runtime_id,
1055 &resolved_session_id,
1056 &driver,
1057 &mutation_guard,
1058 )
1059 .await?;
1060 Ok(Some(super::MachineSessionArchiveLease {
1061 session_id: resolved_session_id,
1062 runtime_id,
1063 driver,
1064 completions,
1065 wake_tx,
1066 publication_handle,
1067 recovered_registration_for_archive,
1068 _registration_transaction_guard: registration_transaction_guard,
1069 _live_lifecycle_lease: live_lifecycle_lease,
1070 _mutation_guard: mutation_guard,
1071 }))
1072 }
1073
1074 pub async fn capture_service_turn_identity(
1078 &self,
1079 session_id: &SessionId,
1080 ) -> Result<super::MachineServiceTurnIdentity, RuntimeDriverError> {
1081 let driver = {
1082 let sessions = self.sessions.read().await;
1083 let entry = sessions
1084 .get(session_id)
1085 .ok_or(RuntimeDriverError::NotReady {
1086 state: RuntimeState::Destroyed,
1087 })?;
1088 if !entry.generated_service_turn_binding_open(session_id) {
1089 return Err(RuntimeDriverError::NotReady {
1090 state: RuntimeState::Destroyed,
1091 });
1092 }
1093 Arc::clone(&entry.driver)
1094 };
1095 Ok(super::MachineServiceTurnIdentity {
1096 session_id: session_id.clone(),
1097 driver,
1098 })
1099 }
1100
1101 pub async fn prepare_service_turn_commit_lease(
1108 &self,
1109 turn_identity: &super::MachineServiceTurnIdentity,
1110 ) -> Result<super::MachineServiceTurnCommitLease, RuntimeDriverError> {
1111 let session_id = &turn_identity.session_id;
1112 let driver = Arc::clone(&turn_identity.driver);
1113 let mutation_guard = self
1114 .lock_current_session_driver_gate(session_id, &driver)
1115 .await?;
1116 let registration_open = {
1117 let sessions = self.sessions.read().await;
1118 sessions.get(session_id).is_some_and(|entry| {
1119 Arc::ptr_eq(&entry.driver, &driver)
1120 && entry.generated_service_turn_binding_open(session_id)
1121 })
1122 };
1123 if !registration_open {
1124 return Err(RuntimeDriverError::NotReady {
1125 state: RuntimeState::Destroyed,
1126 });
1127 }
1128 let run_id = driver.lock().await.current_run_id().ok_or_else(|| {
1129 RuntimeDriverError::Internal(
1130 "service-turn terminal commit lease requires a machine-owned current_run_id"
1131 .to_string(),
1132 )
1133 })?;
1134 Ok(super::MachineServiceTurnCommitLease {
1135 session_id: session_id.clone(),
1136 run_id,
1137 driver,
1138 _mutation_guard: mutation_guard,
1139 })
1140 }
1141
1142 pub async fn commit_service_turn_terminal_receipt_with_lease(
1147 &self,
1148 lease: &mut super::MachineServiceTurnCommitLease,
1149 session: meerkat_core::lifecycle::core_executor::BoundSessionCommit,
1150 ) -> Result<Option<crate::store::PreparedRuntimeSessionCommitResult>, RuntimeDriverError> {
1151 let still_current = {
1152 let sessions = self.sessions.read().await;
1153 sessions.get(&lease.session_id).is_some_and(|entry| {
1154 Arc::ptr_eq(&entry.driver, &lease.driver)
1155 && entry.generated_service_turn_binding_open(&lease.session_id)
1156 })
1157 };
1158 if !still_current {
1159 return Err(RuntimeDriverError::NotReady {
1160 state: RuntimeState::Destroyed,
1161 });
1162 }
1163 let receipt_result = {
1164 let mut driver = lease.driver.lock().await;
1165 machine_commit_service_turn_terminal_receipt(&mut driver, session).await
1166 };
1167 match receipt_result {
1168 Ok(result) => Ok(result),
1169 Err(error) => Err(self
1170 .classify_session_driver_rejection(&lease.session_id, error)
1171 .await),
1172 }
1173 }
1174
1175 async fn remove_archive_recovered_registration_exact(
1184 &self,
1185 session_id: &SessionId,
1186 runtime_id: &LogicalRuntimeId,
1187 driver: &SharedDriver,
1188 ) -> Result<(), RuntimeControlPlaneError> {
1189 let state = driver.lock().await.runtime_state();
1190 if !matches!(state, RuntimeState::Retired | RuntimeState::Destroyed) {
1191 return Err(RuntimeControlPlaneError::InvalidState { state });
1192 }
1193
1194 let removed = {
1195 let mut sessions = self.sessions.write().await;
1196 let Some(entry) = sessions.get(session_id) else {
1197 return Ok(());
1200 };
1201 if &entry.runtime_id != runtime_id || !Arc::ptr_eq(&entry.driver, driver) {
1202 return Err(RuntimeControlPlaneError::Internal(format!(
1203 "archive-recovered runtime {runtime_id} was replaced before quiescent cleanup"
1204 )));
1205 }
1206 if entry.wake_sender().is_some() || entry.publication_handle().is_some() {
1207 return Err(RuntimeControlPlaneError::Internal(format!(
1208 "archive-recovered quiescent runtime {runtime_id} acquired a live attachment before cleanup"
1209 )));
1210 }
1211 sessions.remove(session_id)
1212 };
1213 drop(removed);
1214 Ok(())
1215 }
1216
1217 pub async fn release_quiescent_session_archive_lease(
1220 &self,
1221 lease: super::MachineSessionArchiveLease,
1222 ) -> Result<(), RuntimeControlPlaneError> {
1223 let super::MachineSessionArchiveLease {
1224 session_id,
1225 runtime_id,
1226 driver,
1227 completions: _,
1228 wake_tx,
1229 publication_handle,
1230 recovered_registration_for_archive,
1231 _registration_transaction_guard,
1232 _live_lifecycle_lease,
1233 _mutation_guard,
1234 } = lease;
1235
1236 if !recovered_registration_for_archive {
1237 return Ok(());
1238 }
1239
1240 if wake_tx.is_some() || publication_handle.is_some() {
1241 return Err(RuntimeControlPlaneError::Internal(format!(
1242 "archive-recovered quiescent runtime {runtime_id} acquired a live attachment before cleanup"
1243 )));
1244 }
1245 self.remove_archive_recovered_registration_exact(&session_id, &runtime_id, &driver)
1246 .await
1247 }
1248
1249 pub async fn retire_session_with_archive_lease(
1252 &self,
1253 lease: super::MachineSessionArchiveLease,
1254 ) -> Result<RetireReport, RuntimeControlPlaneError> {
1255 self.realize_retire_with_archive_lease(lease, None).await
1256 }
1257
1258 pub async fn drain_session_archive_lease_terminals(
1263 &self,
1264 lease: &super::MachineSessionArchiveLease,
1265 archive_publication_handle: Option<
1266 &dyn meerkat_core::lifecycle::CoreExecutorPublicationHandle,
1267 >,
1268 ) -> Result<(), RuntimeControlPlaneError> {
1269 let publication_handle = lease
1270 .publication_handle
1271 .as_deref()
1272 .or(archive_publication_handle);
1273 crate::control_plane::drain_recovered_runless_runtime_terminations(
1274 &lease.driver,
1275 Some(&lease.completions),
1276 publication_handle,
1277 )
1278 .await
1279 .map_err(|error| RuntimeControlPlaneError::Internal(error.to_string()))
1280 }
1281
1282 pub async fn retire_session_with_archive_lease_and_publication_handle(
1286 &self,
1287 lease: super::MachineSessionArchiveLease,
1288 publication_handle: &dyn meerkat_core::lifecycle::CoreExecutorPublicationHandle,
1289 ) -> Result<RetireReport, RuntimeControlPlaneError> {
1290 self.realize_retire_with_archive_lease(lease, Some(publication_handle))
1291 .await
1292 }
1293
1294 pub async fn retire_runtime_control_plane(
1295 &self,
1296 runtime_id: &LogicalRuntimeId,
1297 ) -> Result<RetireReport, RuntimeControlPlaneError> {
1298 let (session_id, _, _, _) = self.lookup_entry(runtime_id).await?;
1302 let registration_transaction_guard = self
1303 .lock_session_registration_transaction(&session_id)
1304 .await;
1305 let (resolved_session_id, driver, _, _) = self.lookup_entry(runtime_id).await?;
1306 if resolved_session_id != session_id {
1307 return Err(RuntimeControlPlaneError::Internal(format!(
1308 "runtime {runtime_id} changed session identity from {session_id} to {resolved_session_id} during retirement"
1309 )));
1310 }
1311 self.reject_unregister_overlap_under_registration_transaction(&resolved_session_id)
1312 .await?;
1313 #[cfg(test)]
1314 self.run_control_command_after_logical_lookup_test_hook(
1315 ControlCommandLookupTestKind::Retire,
1316 &resolved_session_id,
1317 )
1318 .await;
1319 #[cfg(feature = "live")]
1320 let live_lifecycle_lease = Some(
1321 self.acquire_member_live_disposal_lease(&session_id)
1322 .await
1323 .map_err(|error| RuntimeControlPlaneError::Internal(error.to_string()))?,
1324 );
1325 #[cfg(not(feature = "live"))]
1326 let live_lifecycle_lease = None;
1327 let mutation_guard = self
1328 .lock_current_session_driver_gate(&session_id, &driver)
1329 .await
1330 .map_err(|error| RuntimeControlPlaneError::Internal(error.to_string()))?;
1331 let (driver, completions, wake_tx, publication_handle) = self
1332 .capture_archive_lease_entry_under_mutation_guard(
1333 runtime_id,
1334 &resolved_session_id,
1335 &driver,
1336 &mutation_guard,
1337 )
1338 .await?;
1339 let lease = super::MachineSessionArchiveLease {
1340 session_id: resolved_session_id,
1341 runtime_id: runtime_id.clone(),
1342 driver,
1343 completions,
1344 wake_tx,
1345 publication_handle,
1346 recovered_registration_for_archive: false,
1347 _registration_transaction_guard: registration_transaction_guard,
1348 _live_lifecycle_lease: live_lifecycle_lease,
1349 _mutation_guard: mutation_guard,
1350 };
1351 self.realize_retire_with_archive_lease(lease, None).await
1352 }
1353
1354 async fn realize_retire_with_archive_lease(
1355 &self,
1356 lease: super::MachineSessionArchiveLease,
1357 archive_publication_handle: Option<
1358 &dyn meerkat_core::lifecycle::CoreExecutorPublicationHandle,
1359 >,
1360 ) -> Result<RetireReport, RuntimeControlPlaneError> {
1361 let super::MachineSessionArchiveLease {
1362 session_id,
1363 runtime_id,
1364 driver,
1365 completions,
1366 wake_tx,
1367 publication_handle,
1368 recovered_registration_for_archive,
1369 _registration_transaction_guard,
1370 _live_lifecycle_lease,
1371 _mutation_guard,
1372 } = lease;
1373 let retained_publication_handle = publication_handle;
1374 let publication_handle = retained_publication_handle
1375 .as_deref()
1376 .or(archive_publication_handle);
1377 tracing::info!(
1378 runtime_id = %runtime_id,
1379 "MeerkatMachine::retire_runtime_control_plane start"
1380 );
1381
1382 let staged_dsl = self
1383 .stage_session_dsl_transition(
1384 &session_id,
1385 crate::meerkat_machine::dsl::MeerkatMachineInput::Retire {
1386 session_id: crate::meerkat_machine::dsl::SessionId::from_domain(&session_id),
1387 },
1388 "Retire",
1389 )
1390 .await
1391 .map_err(RuntimeControlPlaneError::Internal)?;
1392
1393 let mut drv = driver.lock().await;
1394 let mut report = match Box::pin(machine_retire(&mut drv)).await {
1395 Ok(report) => report,
1396 Err(err) => {
1397 drop(drv);
1398 let restored = self
1399 .restore_session_dsl_state_if_current(
1400 &session_id,
1401 staged_dsl.committed_snapshot.clone(),
1402 staged_dsl.previous_snapshot.clone(),
1403 )
1404 .await;
1405 driver
1406 .lock()
1407 .await
1408 .sync_control_projection_from_dsl_authority();
1409 let detail = if restored {
1410 err.to_string()
1411 } else {
1412 format!(
1413 "{err}; archive retire realization failed to restore the staged runtime authority"
1414 )
1415 };
1416 return Err(RuntimeControlPlaneError::Internal(detail));
1417 }
1418 };
1419 drop(drv);
1420
1421 let mut commit_error = None;
1422 if let Err(reason) = self
1423 .commit_session_dsl_transition_preserving_committed_state(
1424 &session_id,
1425 staged_dsl,
1426 "Retire",
1427 )
1428 .await
1429 {
1430 driver
1431 .lock()
1432 .await
1433 .sync_control_projection_from_dsl_authority();
1434 commit_error = Some(reason);
1435 }
1436
1437 crate::control_plane::drain_recovered_runless_runtime_terminations(
1438 &driver,
1439 Some(&completions),
1440 publication_handle,
1441 )
1442 .await
1443 .map_err(|error| RuntimeControlPlaneError::Internal(error.to_string()))?;
1444
1445 if report.inputs_pending_drain > 0 {
1446 if let Some(ref tx) = wake_tx
1447 && tx.send(()).await.is_ok()
1448 {
1449 if let Some(reason) = commit_error {
1450 return Err(RuntimeControlPlaneError::Internal(reason));
1451 }
1452 return Ok(report);
1453 }
1454
1455 let reason = "retired without runtime loop";
1456 let (abandoned, completion_input_ids, candidate_owner_input_id) = {
1457 let mut drv = driver.lock().await;
1458 let completion_input_ids = drv.as_driver().active_input_ids();
1459 let prepared = drv
1460 .prepare_runless_runtime_terminated_interaction_outboxes(
1461 &completion_input_ids,
1462 reason.to_string(),
1463 )
1464 .map_err(|error| RuntimeControlPlaneError::Internal(error.to_string()))?;
1465 let abandoned = match drv
1466 .abandon_pending_inputs(crate::input_state::InputAbandonReason::Retired)
1467 .await
1468 {
1469 Ok(abandoned) => abandoned,
1470 Err(error) => {
1471 drv.rollback_prepared_runless_interaction_terminal_outboxes(prepared);
1472 return Err(RuntimeControlPlaneError::Internal(error.to_string()));
1473 }
1474 };
1475 let candidate_owner_input_id =
1476 crate::meerkat_machine::driver::DriverEntry::commit_prepared_runless_interaction_terminal_outboxes(prepared);
1477 (abandoned, completion_input_ids, candidate_owner_input_id)
1478 };
1479 crate::control_plane::publish_and_resolve_runless_runtime_termination(
1480 &driver,
1481 Some(&completions),
1482 publication_handle,
1483 &completion_input_ids,
1484 candidate_owner_input_id.as_ref(),
1485 reason,
1486 )
1487 .await
1488 .map_err(|error| RuntimeControlPlaneError::Internal(error.to_string()))?;
1489 report.inputs_abandoned += abandoned;
1490 report.inputs_pending_drain = 0;
1491 }
1492 if let Some(reason) = commit_error {
1493 return Err(RuntimeControlPlaneError::Internal(reason));
1494 }
1495 if recovered_registration_for_archive {
1496 self.remove_archive_recovered_registration_exact(&session_id, &runtime_id, &driver)
1497 .await?;
1498 }
1499 Ok(report)
1500 }
1501}
1502
1503#[cfg_attr(not(target_arch = "wasm32"), async_trait::async_trait)]
1504#[cfg_attr(target_arch = "wasm32", async_trait::async_trait(?Send))]
1505impl crate::traits::RuntimeControlPlane for MeerkatMachine {
1506 async fn ingest(
1507 &self,
1508 runtime_id: &LogicalRuntimeId,
1509 input: Input,
1510 ) -> Result<AcceptOutcome, RuntimeControlPlaneError> {
1511 match self
1512 .execute_meerkat_machine_command(
1513 None,
1514 MeerkatMachineCommand::Ingest {
1515 runtime_id: runtime_id.clone(),
1516 input,
1517 },
1518 )
1519 .await
1520 .map_err(MeerkatMachine::control_plane_error_from_command_error)?
1521 {
1522 MeerkatMachineCommandResult::AcceptOutcome(outcome) => Ok(outcome),
1523 other => Err(RuntimeControlPlaneError::Internal(format!(
1524 "unexpected MeerkatMachineCommandResult for ingest: {other:?}"
1525 ))),
1526 }
1527 }
1528
1529 async fn publish_event(
1530 &self,
1531 event: crate::runtime_event::RuntimeEventEnvelope,
1532 ) -> Result<(), RuntimeControlPlaneError> {
1533 match self
1534 .execute_meerkat_machine_command(None, MeerkatMachineCommand::PublishEvent { event })
1535 .await
1536 .map_err(MeerkatMachine::control_plane_error_from_command_error)?
1537 {
1538 MeerkatMachineCommandResult::Unit => Ok(()),
1539 other => Err(RuntimeControlPlaneError::Internal(format!(
1540 "unexpected MeerkatMachineCommandResult for publish_event: {other:?}"
1541 ))),
1542 }
1543 }
1544
1545 async fn retire(
1546 &self,
1547 runtime_id: &LogicalRuntimeId,
1548 ) -> Result<RetireReport, RuntimeControlPlaneError> {
1549 self.retire_runtime_control_plane(runtime_id).await
1550 }
1551
1552 async fn recycle(
1553 &self,
1554 runtime_id: &LogicalRuntimeId,
1555 ) -> Result<RecycleReport, RuntimeControlPlaneError> {
1556 match self
1557 .execute_meerkat_machine_command(
1558 None,
1559 MeerkatMachineCommand::Recycle {
1560 runtime_id: runtime_id.clone(),
1561 },
1562 )
1563 .await
1564 .map_err(MeerkatMachine::control_plane_error_from_command_error)?
1565 {
1566 MeerkatMachineCommandResult::RecycleReport(report) => Ok(report),
1567 other => Err(RuntimeControlPlaneError::Internal(format!(
1568 "unexpected MeerkatMachineCommandResult for recycle: {other:?}"
1569 ))),
1570 }
1571 }
1572
1573 async fn reset(
1574 &self,
1575 runtime_id: &LogicalRuntimeId,
1576 ) -> Result<crate::traits::ResetReport, RuntimeControlPlaneError> {
1577 match self
1578 .execute_meerkat_machine_command(
1579 None,
1580 MeerkatMachineCommand::Reset {
1581 runtime_id: runtime_id.clone(),
1582 },
1583 )
1584 .await
1585 .map_err(MeerkatMachine::control_plane_error_from_command_error)?
1586 {
1587 MeerkatMachineCommandResult::ResetReport(report) => Ok(report),
1588 other => Err(RuntimeControlPlaneError::Internal(format!(
1589 "unexpected MeerkatMachineCommandResult for reset: {other:?}"
1590 ))),
1591 }
1592 }
1593
1594 async fn recover(
1595 &self,
1596 runtime_id: &LogicalRuntimeId,
1597 ) -> Result<RecoveryReport, RuntimeControlPlaneError> {
1598 match self
1599 .execute_meerkat_machine_command(
1600 None,
1601 MeerkatMachineCommand::Recover {
1602 runtime_id: runtime_id.clone(),
1603 },
1604 )
1605 .await
1606 .map_err(MeerkatMachine::control_plane_error_from_command_error)?
1607 {
1608 MeerkatMachineCommandResult::RecoveryReport(report) => Ok(report),
1609 other => Err(RuntimeControlPlaneError::Internal(format!(
1610 "unexpected MeerkatMachineCommandResult for recover: {other:?}"
1611 ))),
1612 }
1613 }
1614
1615 async fn destroy(
1616 &self,
1617 runtime_id: &LogicalRuntimeId,
1618 ) -> Result<DestroyReport, RuntimeControlPlaneError> {
1619 match self
1620 .execute_meerkat_machine_command(
1621 None,
1622 MeerkatMachineCommand::Destroy {
1623 runtime_id: runtime_id.clone(),
1624 },
1625 )
1626 .await
1627 .map_err(MeerkatMachine::control_plane_error_from_command_error)?
1628 {
1629 MeerkatMachineCommandResult::DestroyReport(report) => Ok(report),
1630 other => Err(RuntimeControlPlaneError::Internal(format!(
1631 "unexpected MeerkatMachineCommandResult for destroy: {other:?}"
1632 ))),
1633 }
1634 }
1635
1636 async fn runtime_state(
1637 &self,
1638 runtime_id: &LogicalRuntimeId,
1639 ) -> Result<RuntimeState, RuntimeControlPlaneError> {
1640 match self
1641 .execute_meerkat_machine_command(
1642 None,
1643 MeerkatMachineCommand::RuntimeState {
1644 runtime_id: runtime_id.clone(),
1645 },
1646 )
1647 .await
1648 .map_err(MeerkatMachine::control_plane_error_from_command_error)?
1649 {
1650 MeerkatMachineCommandResult::RuntimeState(state) => Ok(state),
1651 other => Err(RuntimeControlPlaneError::Internal(format!(
1652 "unexpected MeerkatMachineCommandResult for runtime_state: {other:?}"
1653 ))),
1654 }
1655 }
1656
1657 async fn load_boundary_receipt(
1658 &self,
1659 runtime_id: &LogicalRuntimeId,
1660 run_id: &RunId,
1661 sequence: u64,
1662 ) -> Result<Option<meerkat_core::lifecycle::RunBoundaryReceipt>, RuntimeControlPlaneError> {
1663 match self
1664 .execute_meerkat_machine_command(
1665 None,
1666 MeerkatMachineCommand::LoadBoundaryReceipt {
1667 runtime_id: runtime_id.clone(),
1668 run_id: run_id.clone(),
1669 sequence,
1670 },
1671 )
1672 .await
1673 .map_err(MeerkatMachine::control_plane_error_from_command_error)?
1674 {
1675 MeerkatMachineCommandResult::BoundaryReceipt(receipt) => Ok(receipt),
1676 other => Err(RuntimeControlPlaneError::Internal(format!(
1677 "unexpected MeerkatMachineCommandResult for load_boundary_receipt: {other:?}"
1678 ))),
1679 }
1680 }
1681}
1682
1683#[cfg(test)]
1684#[allow(clippy::expect_used, clippy::panic, clippy::unwrap_used)]
1685mod tests {
1686 use super::*;
1687
1688 #[test]
1693 fn control_plane_not_found_maps_to_driver_not_found() {
1694 let runtime_id = LogicalRuntimeId("missing-runtime".to_string());
1695 let mapped = MeerkatMachine::driver_error_from_control_plane_error(
1696 RuntimeControlPlaneError::NotFound(runtime_id.clone()),
1697 );
1698
1699 match mapped {
1700 RuntimeDriverError::NotFound {
1701 runtime_id: mapped_id,
1702 } => assert_eq!(mapped_id, runtime_id),
1703 other => panic!(
1704 "expected RuntimeDriverError::NotFound, got {other:?} (must not collapse absence into NotReady/Destroyed)"
1705 ),
1706 }
1707 }
1708
1709 #[test]
1712 fn control_plane_not_found_is_not_destroyed_not_ready() {
1713 let mapped = MeerkatMachine::driver_error_from_control_plane_error(
1714 RuntimeControlPlaneError::NotFound(LogicalRuntimeId("missing-runtime".to_string())),
1715 );
1716
1717 assert!(
1718 !matches!(
1719 mapped,
1720 RuntimeDriverError::NotReady {
1721 state: RuntimeState::Destroyed
1722 }
1723 ),
1724 "not-found must not be laundered into NotReady{{Destroyed}}"
1725 );
1726 }
1727}