1mod destination;
4mod handoff;
5#[cfg(test)]
6mod tests;
7mod transfer;
8
9use anyhow::{Context, Result, bail, ensure};
10use mj_core::hex::lower_hex;
11use sha2::{Digest, Sha256};
12
13use super::lifecycle::SourceTargetDisposition;
14use super::{Controller, SessionResumeOptions, now};
15
16#[derive(Clone, Copy)]
18enum MoveOwnership {
19 Executing,
20 ExecutingQueue,
21 PendingQueue,
22}
23
24pub(in crate::controller) enum SubagentMutationDrain {
25 Drained,
26 UnsupportedWorkerProtocol(u32),
27}
28
29fn move_ownership() -> &'static std::sync::Mutex<std::collections::BTreeMap<String, MoveOwnership>>
30{
31 static OWNER: std::sync::OnceLock<
32 std::sync::Mutex<std::collections::BTreeMap<String, MoveOwnership>>,
33 > = std::sync::OnceLock::new();
34 OWNER.get_or_init(Default::default)
35}
36
37pub fn move_owns_session(session_id: &str) -> bool {
38 move_ownership()
39 .lock()
40 .unwrap_or_else(std::sync::PoisonError::into_inner)
41 .contains_key(session_id)
42}
43
44pub fn move_has_pending_queue(session_id: &str) -> bool {
45 matches!(
46 move_ownership()
47 .lock()
48 .unwrap_or_else(std::sync::PoisonError::into_inner)
49 .get(session_id),
50 Some(MoveOwnership::ExecutingQueue | MoveOwnership::PendingQueue)
51 )
52}
53
54pub fn release_move_queue_hold(session_id: &str) {
55 set_move_queue_hold(session_id, false);
56}
57
58fn set_move_queue_hold(session_id: &str, pending: bool) {
59 let mut owner = move_ownership()
60 .lock()
61 .unwrap_or_else(std::sync::PoisonError::into_inner);
62 let next = match (owner.get(session_id), pending) {
63 (Some(MoveOwnership::Executing | MoveOwnership::ExecutingQueue), true) => {
64 Some(MoveOwnership::ExecutingQueue)
65 }
66 (Some(MoveOwnership::Executing | MoveOwnership::ExecutingQueue), false) => {
67 Some(MoveOwnership::Executing)
68 }
69 (_, true) => Some(MoveOwnership::PendingQueue),
70 (_, false) => None,
71 };
72 if let Some(next) = next {
73 owner.insert(session_id.to_owned(), next);
74 } else {
75 owner.remove(session_id);
76 }
77}
78
79pub fn restore_move_queue_hold(operation: &MoveOperation) {
80 set_move_queue_hold(
81 &operation.selection.session_id,
82 operation.queue_admission_started && !operation.queue_admission_finished,
83 );
84}
85
86fn interrupted_source_stop_message(recovered: &str, in_place: bool) -> String {
90 if in_place {
91 format!("{recovered}; the in-place swap was interrupted; the environment was retained")
92 } else {
93 recovered.to_owned()
94 }
95}
96
97fn source_stopped_with_verified_checkpoint(record: &mj_core::state::SessionRecord) -> bool {
103 record.state == SessionState::Stopped && record.checkpoint.is_some()
104}
105
106fn stopped_source_recovery(
112 session_id: &str,
113 destination_profile: Option<&str>,
114 destination_target: Option<&str>,
115) -> String {
116 let flag = |name: &str, value: Option<&str>| {
117 value
118 .filter(|value| !value.is_empty())
119 .map(|value| format!(" --{name} {value}"))
120 .unwrap_or_default()
121 };
122 format!(
123 "Source is stopped with a verified checkpoint. Bring it back with \
124 `mj resume --session {session_id}{}{} --queue start`.",
125 flag("profile", destination_profile),
126 flag("target", destination_target),
127 )
128}
129
130fn failed_move_recovery(operation: &MoveOperation, state: &mj_core::state::State) -> String {
139 let record = state.sessions.get(&operation.selection.session_id);
140 if let Some(destination) = &operation.prepared_destination
141 && matches!(
142 destination.state,
143 PreparedDestinationState::CleanupPending { .. }
144 )
145 {
146 return format!(
147 "Source and recovery data retained. Automatic EC2 cleanup is pending for {}; charges may continue until termination is confirmed.",
148 destination
149 .instance_id()
150 .unwrap_or("the recorded launch token")
151 );
152 }
153 if operation.prepared_destination.is_some()
154 && record.is_some_and(|r| {
155 matches!(r.state, SessionState::Running | SessionState::Disconnected)
156 && r.target == operation.source_target
157 })
158 {
159 return "Source retained and still running. EC2 destination cleaned up; prepare Move again when ready.".into();
160 }
161 if operation.queue_admission_started {
162 return "Destination is live; retry queue admission on this same destination. Already accepted work may have effects.".to_owned();
163 }
164 let source_live = record.is_some_and(|record| {
165 matches!(
166 record.state,
167 SessionState::Running | SessionState::Disconnected
168 )
169 });
170 let environment_held = operation.holds_source_environment()
171 || (operation.in_place
172 && !source_live
173 && record.is_some_and(|record| record.target.is_some()));
174 if environment_held && !operation.checkpoint_retained() {
175 return missing_move_checkpoint_recovery(state, &operation.selection.session_id);
176 }
177 if operation.in_place && record.is_some_and(|record| record.target.is_some()) {
178 return if source_live {
179 "Source retained and still running. Retry Move when ready.".to_owned()
180 } else {
181 "Environment and checkpoint retained. Retry Move on the same target; the checkout will not be recreated.".to_owned()
182 };
183 }
184 match record {
185 Some(record) if source_stopped_with_verified_checkpoint(record) => {
186 stopped_source_recovery(
187 &operation.selection.session_id,
188 operation.selection.profile_id.as_deref(),
189 operation.selection.target_template_id.as_deref(),
190 )
191 }
192 _ => "Source or partial destination is retained. Retry move after resolving the reported error.".to_owned(),
193 }
194}
195
196fn missing_move_checkpoint_recovery(state: &mj_core::state::State, session_id: &str) -> String {
201 let checkout_path = state
202 .checkout(session_id)
203 .ok()
204 .and_then(|checkout| match &checkout {
205 mj_core::state::Checkout::ManagedWorktree { worktree, .. } => {
206 Some(worktree.worktree_root.clone())
207 }
208 mj_core::state::Checkout::Attached { path } => Some((*path).to_path_buf()),
209 mj_core::state::Checkout::Borrowed { .. } => {
210 checkout.project_directory().map(|path| path.to_path_buf())
211 }
212 mj_core::state::Checkout::ManagedWorkspace => None,
213 });
214 let checkout = checkout_path
215 .map(|path| format!(" in {}", path.display()))
216 .unwrap_or_default();
217 format!(
218 "The Move's checkpoint archive is missing, so the Move cannot be retried and Resume cannot restore the session. \
219 The environment and checkout{checkout} are retained; copy out anything you need, then destroy the session."
220 )
221}
222
223fn failed_move_message(
226 phase: Option<&str>,
227 cancelled: bool,
228 recovery: &str,
229 operation_id: &str,
230) -> String {
231 format!(
232 "{}{}{}. {recovery} The daemon log records the reason under reference {operation_id}",
233 mj_core::state::MOVE_FAILURE_PREFIX,
234 phase
235 .map(|phase| format!(" while {phase}"))
236 .unwrap_or_default(),
237 if cancelled { " (cancelled)" } else { "" },
238 )
239}
240
241pub(crate) fn forget_missing_move_archives(operation: &mut MoveOperation) -> bool {
249 let missing = |checkpoint: &Option<mj_core::state::CheckpointMetadata>| {
250 checkpoint.as_ref().is_some_and(|checkpoint| {
251 matches!(
252 std::fs::symlink_metadata(&checkpoint.archive_path),
253 Err(error) if error.kind() == std::io::ErrorKind::NotFound
254 )
255 })
256 };
257 let lost = if missing(&operation.handoff) {
258 [operation.handoff.take(), operation.checkpoint.take()]
259 } else if operation.handoff.is_none() && missing(&operation.checkpoint) {
260 [None, operation.checkpoint.take()]
261 } else {
262 return false;
263 };
264 for checkpoint in lost.iter().flatten() {
265 tracing::warn!(
266 session_id = %operation.selection.session_id,
267 reference = %operation.operation_id,
268 path = %checkpoint.archive_path.display(),
269 "a Move's checkpoint archive is missing; recording that the Move cannot restore it"
270 );
271 }
272 true
273}
274
275pub(crate) fn record_missing_move_archives(
279 state: &mj_core::state::State,
280 operations: Vec<MoveOperation>,
281) -> Result<()> {
282 for mut operation in operations {
283 if !operation.retains_checkpoint() || !forget_missing_move_archives(&mut operation) {
284 continue;
285 }
286 let record = state.sessions.get(&operation.selection.session_id);
287 let published = record.and_then(|record| record.last_error.as_deref());
288 if operation.is_active()
289 || !published
290 .is_some_and(|error| error.starts_with(mj_core::state::MOVE_FAILURE_PREFIX))
291 {
292 crate::database::save_move_operation(&operation)?;
293 continue;
294 }
295 record_finished_move_recovery(state, &mut operation)?;
296 }
297 Ok(())
298}
299
300pub(crate) fn record_finished_move_recovery(
302 state: &mj_core::state::State,
303 operation: &mut MoveOperation,
304) -> Result<()> {
305 let record = state.sessions.get(&operation.selection.session_id);
306 operation.updated_at = now();
307 if !operation.is_active()
308 && record
309 .and_then(|record| record.last_error.as_deref())
310 .is_some_and(|message| message.starts_with(mj_core::state::MOVE_FAILURE_PREFIX))
311 {
312 let message = failed_move_message(
313 None,
314 operation.phase == MovePhase::Cancelled,
315 &failed_move_recovery(operation, state),
316 &operation.operation_id,
317 );
318 crate::database::save_move_outcome(operation, Some(&message))
319 } else {
320 crate::database::save_move_operation(operation)
321 }
322}
323
324fn sealed_selection_difference(retained: &MoveSelection, requested: &MoveSelection) -> String {
327 let mut parts = Vec::new();
328 for (name, flag, retained, requested) in [
329 (
330 "profile",
331 "--profile",
332 &retained.profile_id,
333 &requested.profile_id,
334 ),
335 (
336 "target",
337 "--target",
338 &retained.target_template_id,
339 &requested.target_template_id,
340 ),
341 ] {
342 if retained != requested {
343 parts.push(format!(
344 "{name} (sealed with {0}; pass {flag} {0})",
345 retained.as_deref().unwrap_or_default()
346 ));
347 }
348 }
349 if retained.workspace.acknowledge_large_transfer
350 != requested.workspace.acknowledge_large_transfer
351 {
352 parts.push(if retained.workspace.acknowledge_large_transfer {
353 "large-transfer acknowledgement (the Move was sealed with it; pass --allow-large-transfer)".to_owned()
354 } else {
355 "large-transfer acknowledgement (the Move was sealed without it; omit --allow-large-transfer)".to_owned()
356 });
357 }
358 if retained.workspace.exclusions != requested.workspace.exclusions {
359 let excluded = retained
360 .workspace
361 .exclusions
362 .iter()
363 .map(|path| format!("{}:{}", path.repository, path.path.display()))
364 .collect::<Vec<_>>();
365 parts.push(if excluded.is_empty() {
366 "excluded files (the Move was sealed excluding none)".to_owned()
367 } else {
368 format!(
369 "excluded files (the Move was sealed excluding exactly {})",
370 excluded.join(", ")
371 )
372 });
373 }
374 if retained.additional_mounts != requested.additional_mounts {
375 parts.push("attached directories (send the ones the Move was sealed with)".to_owned());
376 }
377 if retained.resource_allocation != requested.resource_allocation
378 || retained.clear_resource_allocation != requested.clear_resource_allocation
379 {
380 parts.push("resource allocation (send the one the Move was sealed with)".to_owned());
381 }
382 if retained.subagents != requested.subagents {
383 parts.push("sub-agent policy (send the one the Move was sealed with)".to_owned());
384 }
385 if retained.session_id != requested.session_id || parts.is_empty() {
386 parts.push("the session".to_owned());
387 }
388 format!(
389 "a sealed Move must be retried with the selection it was sealed with; this request differs in its {}",
390 parts.join("; ")
391 )
392}
393
394pub struct MoveMutationGuard(String);
395
396impl MoveMutationGuard {
397 pub fn reserve(session_id: &str) -> Result<Self> {
398 let mut owner = move_ownership()
399 .lock()
400 .unwrap_or_else(std::sync::PoisonError::into_inner);
401 ensure!(
402 !matches!(
403 owner.get(session_id),
404 Some(MoveOwnership::Executing | MoveOwnership::ExecutingQueue)
405 ),
406 "session already has a move owner"
407 );
408 let next = if owner.contains_key(session_id) {
409 MoveOwnership::ExecutingQueue
410 } else {
411 MoveOwnership::Executing
412 };
413 owner.insert(session_id.to_owned(), next);
414 Ok(Self(session_id.to_owned()))
415 }
416}
417
418impl Drop for MoveMutationGuard {
419 fn drop(&mut self) {
420 let mut owner = move_ownership()
421 .lock()
422 .unwrap_or_else(std::sync::PoisonError::into_inner);
423 match owner.get(&self.0) {
424 Some(MoveOwnership::ExecutingQueue) => {
425 owner.insert(self.0.clone(), MoveOwnership::PendingQueue);
426 }
427 Some(MoveOwnership::Executing) => {
428 owner.remove(&self.0);
429 }
430 _ => {}
431 }
432 }
433}
434
435pub(crate) fn move_refuses_command(session_id: &str, command: &RelayCommand) -> bool {
436 move_owns_session(session_id)
437 && matches!(
438 command,
439 RelayCommand::Prompt { .. }
440 | RelayCommand::SetConfig { .. }
441 | RelayCommand::SetSessionMode { .. }
442 | RelayCommand::RestoreExecutionMode
443 | RelayCommand::RunUserShell { .. }
444 | RelayCommand::CancelUserShell { .. }
445 | RelayCommand::Cancel
446 | RelayCommand::RemoveQueuedPrompt { .. }
447 | RelayCommand::ClearQueuedPrompts
448 )
449}
450use crate::session_manager::{SessionManagerControl, StandaloneSession, new_command_id};
451use mj_checkpoint::archive::{CanonicalQueuedCommandKind, verify_archive_streaming};
452use mj_core::state::{
453 DestinationChecks, MoveOperation, MovePhase, PreparedDestinationState, ResumeQueueDisposition,
454 SessionState,
455};
456
457pub use mj_core::state::{MoveOutcome, MovePreparation, MoveSelection, MoveSessionRequest};
458
459use crate::targets::{CommandExecutor, ProvisionStage, ProvisionStageGuard};
460use mj_core::relay::RelayCommand;
461
462pub async fn refresh_move_source(
464 manager: &SessionManagerControl,
465 id: &str,
466) -> Result<Option<mj_core::state::ManagedSessionSnapshot>> {
467 Ok(MoveSourceRelay::lease(manager, id).await?.snapshot())
468}
469
470#[derive(Default)]
476pub(in crate::controller) struct MoveSourceRelay(
477 Option<super::checkpoint::ControllerRelayLease>,
478 Option<crate::worker_lifecycle::WorkerPermit>,
479);
480
481impl MoveSourceRelay {
482 pub(in crate::controller) fn owner(&self) -> Option<crate::worker_lifecycle::WorkerPermit> {
483 self.1.clone()
484 }
485
486 pub(in crate::controller) async fn lease(
491 manager: &SessionManagerControl,
492 id: &str,
493 ) -> Result<Self> {
494 let owner = crate::worker_lifecycle::WorkerPermit::acquire(
495 id,
496 "Move source relay",
497 &crate::targets::ProcessExecutor,
498 )
499 .await?;
500 let handle = manager
501 .wait_for_session(id, std::time::Duration::from_secs(5))
502 .await?;
503 match handle.lease_connection().await {
504 Ok(lease) => Ok(Self(
505 Some(super::checkpoint::ControllerRelayLease::Managed {
506 handle,
507 lease: Some(lease),
508 }),
509 Some(owner),
510 )),
511 Err(error) if crate::worker_client::RelayTransportDead::marks(&error) => {
512 tracing::warn!(session_id = id, error = %error, "Move will recover the unavailable source without its harness");
513 Ok(Self(None, Some(owner)))
514 }
515 Err(error) => Err(error),
516 }
517 }
518
519 async fn set_subagent_admission_via_manager(
520 manager: &SessionManagerControl,
521 id: &str,
522 open: bool,
523 ) -> Result<bool> {
524 let handle = manager
525 .wait_for_session(id, std::time::Duration::from_secs(5))
526 .await?;
527 let mut lease = handle.lease_connection().await?;
528 if !(mj_core::relay::RelayRequest::SetSubagentAdmission { open })
529 .supported_at(lease.connection_mut().protocol_version())
530 {
531 lease.release();
532 return Ok(false);
533 }
534 let result = lease.connection_mut().set_subagent_admission(open).await;
535 lease.release();
536 result.map(|()| true)
537 }
538
539 pub(in crate::controller) fn snapshot(
541 &mut self,
542 ) -> Option<mj_core::state::ManagedSessionSnapshot> {
543 self.0
544 .as_mut()
545 .map(|relay| relay.connection_mut().snapshot())
546 }
547
548 pub(in crate::controller) async fn sync(
551 &mut self,
552 id: &str,
553 ) -> Result<Option<mj_core::state::ManagedSessionSnapshot>> {
554 let Some(relay) = self.0.as_mut() else {
555 return Ok(None);
556 };
557 match relay.sync_snapshot().await {
558 Ok(snapshot) => Ok(Some(snapshot)),
559 Err(error) if crate::worker_client::RelayTransportDead::marks(&error) => {
560 tracing::warn!(session_id = id, error = %error, "Move will recover the unavailable source without its harness");
561 self.0 = None;
562 Ok(None)
563 }
564 Err(error) => Err(error),
565 }
566 }
567
568 pub(in crate::controller) async fn drain_subagent_mutations(
572 &mut self,
573 id: &str,
574 executor: &(impl CommandExecutor + Sync),
575 ) -> Result<SubagentMutationDrain> {
576 ensure!(
577 self.0.is_some(),
578 "cannot safely drain sub-agent requests because the source worker is unavailable"
579 );
580 let protocol_version = self
581 .0
582 .as_mut()
583 .expect("held source relay checked")
584 .connection_mut()
585 .protocol_version();
586 if !(mj_core::relay::RelayRequest::SetSubagentAdmission { open: false })
587 .supported_at(protocol_version)
588 {
589 return Ok(SubagentMutationDrain::UnsupportedWorkerProtocol(
590 protocol_version,
591 ));
592 }
593 if let Err(error) = self
594 .0
595 .as_mut()
596 .expect("held source relay checked")
597 .connection_mut()
598 .set_subagent_admission(false)
599 .await
600 {
601 self.release_managed_connection();
605 if let Err(reopen) = self.reopen_subagent_mutations_with_retry(id).await {
606 return Err(error.context(format!(
607 "could not reopen sub-agent requests after an ambiguous admission close: {reopen:#}"
608 )));
609 }
610 return Err(error);
611 }
612 self.release_managed_connection();
613
614 let drain = async {
615 loop {
616 ensure!(
617 !executor.cancellation_requested(),
618 "Move cancelled while waiting for sub-agent requests"
619 );
620 let snapshot = self.sync(id).await?.context(
621 "source worker became unavailable while draining sub-agent requests",
622 )?;
623 let parent = id.to_owned();
624 let effects_pending = tokio::task::spawn_blocking(move || {
625 crate::database::has_pending_mutating_delegations(&parent)
626 })
627 .await??;
628 if !subagent_mutations_pending(&snapshot.subagent_requests, effects_pending) {
629 break;
630 }
631 executor.notify_notice("Waiting for subagent requests to finish");
632 tokio::time::sleep(std::time::Duration::from_millis(100)).await;
633 }
634 Ok::<(), anyhow::Error>(())
635 }
636 .await;
637
638 if let Err(error) = drain {
639 if let Err(reopen) = self.reopen_subagent_mutations_with_retry(id).await {
640 return Err(error.context(format!(
641 "could not reopen sub-agent requests after Move stopped waiting: {reopen:#}"
642 )));
643 }
644 return Err(error);
645 }
646 self.reacquire_managed_connection().await?;
647 if executor.cancellation_requested() {
648 self.reopen_subagent_mutations_with_retry(id).await?;
649 bail!("Move cancelled before source interruption; sub-agent requests reopened");
650 }
651 Ok(SubagentMutationDrain::Drained)
652 }
653
654 pub(in crate::controller) async fn reopen_subagent_mutations(
655 &mut self,
656 id: &str,
657 ) -> Result<()> {
658 self.reopen_subagent_mutations_with_retry(id).await
659 }
660
661 async fn open_subagent_mutations(&mut self) -> Result<()> {
662 self.reacquire_managed_connection().await?;
663 let result = self
664 .0
665 .as_mut()
666 .context("source worker is unavailable")?
667 .connection_mut()
668 .set_subagent_admission(true)
669 .await;
670 self.release_managed_connection();
671 result
672 }
673
674 async fn reopen_subagent_mutations_with_retry(&mut self, id: &str) -> Result<()> {
675 let first_error = match self.open_subagent_mutations().await {
676 Ok(()) => return Ok(()),
677 Err(error) => error,
678 };
679 tracing::warn!(
680 session_id = id,
681 error = %first_error,
682 "sub-agent admission reopen failed; retrying on a fresh relay connection"
683 );
684 tokio::task::yield_now().await;
685 match self.open_subagent_mutations().await {
686 Ok(()) => {
687 tracing::info!(
688 session_id = id,
689 "reopened sub-agent requests on a replacement relay connection"
690 );
691 Ok(())
692 }
693 Err(retry_error) => {
694 tracing::warn!(
695 session_id = id,
696 error = %retry_error,
697 "could not reopen sub-agent requests on the replacement relay connection"
698 );
699 Err(first_error.context(format!("relay reconnect retry failed: {retry_error:#}")))
700 }
701 }
702 }
703
704 async fn reacquire_managed_connection(&mut self) -> Result<()> {
705 if let Some(super::checkpoint::ControllerRelayLease::Managed { handle, lease }) =
706 self.0.as_mut()
707 && lease.is_none()
708 {
709 *lease = Some(handle.lease_connection().await?);
710 }
711 Ok(())
712 }
713
714 fn release_managed_connection(&mut self) {
715 if let Some(super::checkpoint::ControllerRelayLease::Managed { lease, .. }) =
716 self.0.as_mut()
717 && let Some(lease) = lease.take()
718 {
719 lease.release();
720 }
721 }
722
723 pub(in crate::controller) fn is_held(&self) -> bool {
724 self.0.is_some()
725 }
726
727 pub(in crate::controller) fn take(
729 &mut self,
730 ) -> Option<super::checkpoint::ControllerRelayLease> {
731 self.0.take()
732 }
733
734 pub(in crate::controller) fn replace_connection(
737 &mut self,
738 connection: crate::session_manager::StandaloneSession,
739 ) {
740 if let Some(relay) = self.0.as_mut() {
741 relay.replace_connection(connection);
742 }
743 }
744}
745
746impl Drop for MoveSourceRelay {
747 fn drop(&mut self) {
748 if let Some(relay) = self.0.take() {
749 relay.release();
750 }
751 self.1.take();
752 }
753}
754
755fn digest(value: &impl serde::Serialize) -> Result<String> {
756 Ok(lower_hex(Sha256::digest(serde_json::to_vec(value)?)))
757}
758
759struct MovePhaseTimer<'a> {
760 session_id: &'a str,
761 phase: &'static str,
762 started: std::time::Instant,
763}
764
765impl<'a> MovePhaseTimer<'a> {
766 fn new(session_id: &'a str, phase: &'static str) -> Self {
767 Self {
768 session_id,
769 phase,
770 started: std::time::Instant::now(),
771 }
772 }
773}
774
775impl Drop for MovePhaseTimer<'_> {
776 fn drop(&mut self) {
777 tracing::info!(
778 session_id = self.session_id,
779 phase = self.phase,
780 elapsed_ms = self.started.elapsed().as_millis() as u64,
781 "move phase finished"
782 );
783 }
784}
785
786fn replace_queued_images_with_placeholders(
790 queued_commands: &mut [mj_core::state::MaterializedQueuedPrompt],
791) {
792 for block in queued_commands
793 .iter_mut()
794 .flat_map(|command| command.content.iter_mut())
795 {
796 if block.get("type").and_then(serde_json::Value::as_str) != Some("image") {
797 continue;
798 }
799 let mime = block
800 .get("mimeType")
801 .or_else(|| block.get("mime_type"))
802 .and_then(serde_json::Value::as_str)
803 .unwrap_or("image");
804 *block = serde_json::json!({"type": "text", "text": format!("[Image attachment: {mime}]")});
805 }
806}
807
808impl Controller {
809 fn validate_move_destination_paths(
813 &self,
814 source: &mj_core::state::SessionRecord,
815 target_id: &str,
816 executor: &(impl CommandExecutor + Sync),
817 ) -> Result<Option<super::worktree::RawToWorkspaceConversion>> {
818 use super::worktree::ResumePlan;
819 let checkout = self.state.checkout(&source.id)?;
820 match super::worktree::resume_compatibility_with_checkout(
821 source,
822 &checkout,
823 &self.config,
824 target_id,
825 )
826 .map_err(anyhow::Error::msg)?
827 {
828 ResumePlan::RawToWorkspace => {
829 return Ok(Some(super::worktree::plan_raw_to_workspace_for_session(
830 source,
831 &checkout,
832 &self.config,
833 executor,
834 )?));
835 }
836 ResumePlan::WorkspaceToRaw => {
837 self.plan_workspace_to_raw(source, target_id, executor)?;
838 }
839 ResumePlan::InPlace
840 if matches!(checkout, mj_core::state::Checkout::Attached { .. }) =>
841 {
842 if let Some(path) = checkout.project_directory() {
843 self.validate_project_directory(target_id, path, executor)?;
844 }
845 }
846 ResumePlan::InPlace => {}
847 }
848 Ok(None)
849 }
850 fn move_confirmation(
851 &self,
852 selection: &MoveSelection,
853 conversion: Option<&mj_core::state::RawConversionPreview>,
854 ) -> Result<(bool, Vec<mj_core::state::MaterializedQueuedPrompt>, String)> {
855 let source = self
856 .state
857 .sessions
858 .get(&selection.session_id)
859 .context("unknown move session")?;
860 let (mut active, mut queued) = crate::database::move_pending_work(&source.id)?;
861 if let Some(operation) = crate::database::load_move_operation(&source.id)?
862 && operation.queue_admission_started
863 && !operation.queue_admission_finished
864 {
865 ensure!(
866 operation.selection == *selection,
867 "queue admission is incomplete on the live destination; retry that move before selecting another destination"
868 );
869 let checkpoint = operation
870 .restore_artifact()
871 .context("retained queue checkpoint is missing")?;
872 let verified = verify_archive_streaming(&checkpoint.archive_path)?;
873 ensure!(
874 verified.archive_sha256 == checkpoint.sha256
875 && verified.manifest.session.id == source.id,
876 "retained move checkpoint verification failed"
877 );
878 queued = verified
879 .canonical_session
880 .queued_prompts
881 .into_iter()
882 .map(|entry| mj_core::state::MaterializedQueuedPrompt {
883 accepted_ordinal: None,
884 command_id: entry.command_id,
885 kind: match entry.kind {
886 CanonicalQueuedCommandKind::Prompt => {
887 mj_core::state::QueuedCommandKind::Prompt
888 }
889 CanonicalQueuedCommandKind::SetConfig { key, value } => {
890 mj_core::state::QueuedCommandKind::SetConfig { key, value }
891 }
892 },
893 content: entry.content,
894 queued_at_ms: entry.queued_at_ms,
895 })
896 .collect();
897 active = false;
898 }
899 let fingerprint = digest(&(
900 &source.last_profile,
901 &source.target_template_id,
902 &source.target,
903 &source.target_runtime,
904 &source.native_session_id,
905 &source.resource_allocation,
906 &source.additional_mounts,
907 &source.container_cpus,
908 &source.container_memory,
909 selection,
910 self.move_configuration_fingerprint(selection)?,
911 &queued,
912 conversion.map(|preview| {
916 (
917 &preview.fetch_url,
918 &preview.push_urls,
919 &preview.branch,
920 &preview.destination,
921 )
922 }),
923 ))?;
924 Ok((active, queued, fingerprint))
925 }
926 fn verify_current_move_configuration(&self, operation: &MoveOperation) -> Result<()> {
927 let current = Controller {
928 config: mj_core::config::Config::load()?,
929 state: self.state.clone(),
930 };
931 ensure!(
932 current.move_configuration_fingerprint(&operation.selection)?
933 == operation.configuration_fingerprint,
934 "destination configuration changed during Move preparation; source retained, prepare and confirm Move again"
935 );
936 Ok(())
937 }
938
939 pub(super) fn move_configuration_fingerprint(
940 &self,
941 selection: &MoveSelection,
942 ) -> Result<String> {
943 let profile = self
944 .config
945 .profiles
946 .get(
947 selection
948 .profile_id
949 .as_deref()
950 .context("move profile is unresolved")?,
951 )
952 .context("move profile no longer exists")?;
953 let target = self
954 .config
955 .targets
956 .get(
957 selection
958 .target_template_id
959 .as_deref()
960 .context("move target is unresolved")?,
961 )
962 .context("move target no longer exists")?;
963 let source = self
966 .state
967 .sessions
968 .get(&selection.session_id)
969 .context("unknown move session")?;
970 digest(&(
971 profile,
972 target,
973 &self.config.bundles,
974 self.config.targets.get(&source.target_template_id),
975 self.config.profiles.get(&source.last_profile),
976 ))
977 }
978
979 async fn validate_move_destination_configuration(
980 &self,
981 selection: &MoveSelection,
982 source_harness: mj_core::config::HarnessKind,
983 operational: &mj_core::relay::RelayOperationalState,
984 ) -> Result<()> {
985 let profile_id = selection
986 .profile_id
987 .as_deref()
988 .context("move profile is unresolved")?;
989 let profile = self
990 .config
991 .profiles
992 .get(profile_id)
993 .context("move profile no longer exists")?;
994 let source = self
995 .state
996 .sessions
997 .get(&selection.session_id)
998 .context("unknown move session")?;
999 if profile.kind != source_harness || profile_id == source.last_profile {
1000 return Ok(());
1001 }
1002 let accepted = mj_core::acp::AcceptedSessionConfig::from_configuration(
1003 &operational.config,
1004 &operational.config_options,
1005 );
1006 if accepted.model.is_none() && accepted.effort.is_none() {
1007 return Ok(());
1008 }
1009 let choices =
1012 super::profile_config::discover(profile_id.to_owned(), accepted.model.clone(), true)
1013 .await
1014 .with_context(|| {
1015 format!("discover destination profile {profile_id:?} configuration")
1016 })?;
1017 validate_preserved_configuration(profile_id, &accepted, &choices)
1018 }
1019
1020 pub async fn prepare_move_session_controlled(
1021 &self,
1022 mut selection: MoveSelection,
1023 executor: &(impl CommandExecutor + Sync),
1024 ) -> Result<MovePreparation> {
1025 let source = self
1026 .state
1027 .sessions
1028 .get(&selection.session_id)
1029 .context("unknown session")?;
1030 ensure!(
1031 !self.state.subagents.contains_key(&source.id),
1032 "sub-agent sessions cannot move independently of their parent"
1033 );
1034 let previous = crate::database::load_move_operation(&source.id)?;
1035 if let Some(retained) = previous.as_ref().filter(|op| op.holds_source_environment()) {
1038 selection.profile_id = selection
1039 .profile_id
1040 .or(retained.selection.profile_id.clone());
1041 selection.target_template_id = selection
1042 .target_template_id
1043 .or(retained.selection.target_template_id.clone());
1044 selection.subagents = selection.subagents.or(retained.selection.subagents.clone());
1045 if retained.selection.subagents.is_none()
1048 && selection.subagents.as_ref()
1049 == Some(&source.subagents.clone().unwrap_or_default())
1050 {
1051 selection.subagents = None;
1052 }
1053 }
1054 ensure!(
1055 selection.profile_id.is_some() || selection.target_template_id.is_some(),
1056 "move requires a target or profile selection"
1057 );
1058 ensure!(
1059 previous
1060 .as_ref()
1061 .and_then(|op| op.prepared_destination.as_ref())
1062 .is_none_or(|d| !matches!(
1063 d.state,
1064 PreparedDestinationState::CleanupPending { .. }
1065 )),
1066 "EC2 destination cleanup is pending; source retained. Automatic cleanup will retry before another Move"
1067 );
1068
1069 if previous
1070 .as_ref()
1071 .is_some_and(|op| op.holds_source_environment() && !op.checkpoint_retained())
1072 {
1073 bail!(
1074 "this Move cannot be retried. {}",
1075 missing_move_checkpoint_recovery(&self.state, &source.id)
1076 );
1077 }
1078 let retry = previous.as_ref().is_some_and(|op| {
1079 !matches!(op.phase, MovePhase::Completed)
1080 && (op.restore_artifact().is_some() || source.checkpoint.is_some())
1081 });
1082 ensure!(
1083 matches!(
1084 source.state,
1085 SessionState::Running | SessionState::Disconnected
1086 ) || retry,
1087 "only active sessions can move; run `mj resume` (or POST /api/v1/sessions/{}/resume) for a stopped or lost session",
1088 source.id
1089 );
1090 selection
1091 .profile_id
1092 .get_or_insert_with(|| source.last_profile.clone());
1093 selection
1094 .target_template_id
1095 .get_or_insert_with(|| source.target_template_id.clone());
1096 selection
1097 .additional_mounts
1098 .get_or_insert_with(|| source.additional_mounts.clone());
1099 ensure!(
1100 !selection.clear_resource_allocation || selection.resource_allocation.is_none(),
1101 "select resource allocation or explicitly clear it, not both"
1102 );
1103 if selection.resource_allocation.is_none()
1104 && matches!(
1105 self.config
1106 .targets
1107 .get(selection.target_template_id.as_deref().unwrap()),
1108 Some(mj_core::config::TargetTemplate::AwsEc2 { .. })
1109 )
1110 && (matches!(
1111 source.resource_allocation,
1112 Some(mj_core::state::SessionResourceAllocation::Container { .. })
1113 ) || source.container_cpus.is_some()
1114 || source.container_memory.is_some())
1115 {
1116 selection.clear_resource_allocation = true;
1117 }
1118 if !selection.clear_resource_allocation {
1119 selection.resource_allocation = selection
1120 .resource_allocation
1121 .or_else(|| source.resource_allocation.clone());
1122 }
1123 if selection.clear_resource_allocation
1124 && source.resource_allocation.is_none()
1125 && source.container_cpus.is_none()
1126 && source.container_memory.is_none()
1127 {
1128 selection.clear_resource_allocation = false;
1129 }
1130 if let Some(retained) = previous.as_ref().filter(|op| op.holds_source_environment()) {
1131 if retained.in_place {
1135 selection.workspace = retained.selection.workspace.clone();
1136 }
1137 ensure!(
1138 retained.selection == selection,
1139 "{}",
1140 sealed_selection_difference(&retained.selection, &selection)
1141 );
1142 ensure!(
1143 !retained.in_place
1144 || (retained.source_target.is_some()
1145 && source.target == retained.source_target),
1146 "retained Move target is missing or changed; refusing to recreate it"
1147 );
1148 }
1149 let profile_id = selection.profile_id.as_deref().unwrap();
1150 let target_id = selection.target_template_id.as_deref().unwrap();
1151 let profile = self
1152 .config
1153 .profiles
1154 .get(profile_id)
1155 .context("unknown destination profile")?;
1156 ensure!(
1157 profile.enabled,
1158 "destination profile {profile_id:?} is disabled"
1159 );
1160 let policy = move_subagent_policy(source, &selection);
1161 if policy.uses_mjolnir() || policy != source.subagents.clone().unwrap_or_default() {
1162 super::profile_config::validate_session_subagent_policy(
1163 &self.config,
1164 profile_id,
1165 &policy,
1166 )
1167 .await?;
1168 }
1169 let target = self
1170 .config
1171 .targets
1172 .get(target_id)
1173 .context("unknown destination target")?;
1174 self.validate_muse_resume_destination(source, profile.kind, target_id)?;
1175 let checkout = self.state.checkout(&source.id)?;
1176 super::worktree::resume_compatibility_with_checkout(
1177 source,
1178 &checkout,
1179 &self.config,
1180 target_id,
1181 )
1182 .map_err(anyhow::Error::msg)?;
1183 super::backend::validate_resource_allocation(
1184 target,
1185 selection.resource_allocation.as_ref(),
1186 )?;
1187 let mounts = selection.additional_mounts.as_deref().unwrap_or_default();
1188 ensure!(
1189 profile.kind != mj_core::config::HarnessKind::Muse || mounts.is_empty(),
1190 "Muse Code ACP supports one workspace root; attached directories are unsupported"
1191 );
1192 ensure!(
1193 mounts.is_empty() || mj_core::config::mount_history_host(target).is_some(),
1194 "attached resources are unsupported for this target; select compatible resources explicitly"
1195 );
1196 crate::targets::validate_additional_mounts(mounts)?;
1197 for mount in mounts {
1198 self.validate_mount_source(target_id, &mount.source, executor)?;
1199 }
1200 let planned_conversion =
1201 self.validate_move_destination_paths(source, target_id, executor)?;
1202 ensure!(
1203 profile.home.is_dir(),
1204 "destination profile home is unavailable; configure the profile before moving"
1205 );
1206 super::worker_binary::preflight_worker_binary(target, executor)?;
1207 super::backend::preflight_target(target, executor, super::backend::TargetCheck::Launch)?;
1208 let source_harness = previous
1209 .as_ref()
1210 .filter(|operation| {
1211 !operation.queue_admission_started && operation.phase != MovePhase::Completed
1212 })
1213 .and_then(|operation| operation.recovery_session.as_ref())
1214 .map_or(source.harness_kind, |record| record.harness_kind);
1215 let cross_harness = profile.kind != source_harness;
1216 if cross_harness {
1217 let cancel = tokio_util::sync::CancellationToken::new();
1218 let resolve =
1219 crate::utility_llm::UtilityLlmRuntime::shared().resolve(&self.config, &cancel);
1220 tokio::pin!(resolve);
1221 loop {
1222 tokio::select! {
1223 result = &mut resolve => { result.context("cross-harness move needs an available utility model")?; break; }
1224 _ = tokio::time::sleep(std::time::Duration::from_millis(50)) => {
1225 if executor.cancellation_requested() { cancel.cancel(); bail!("move preparation cancelled"); }
1226 }
1227 }
1228 }
1229 }
1230 let conversion = planned_conversion
1233 .map(|conversion| {
1234 super::worktree::raw_conversion_preview_with_checkout(source, &conversion, executor)
1235 .context("describe the move of this checkout into the target")
1236 })
1237 .transpose()?
1238 .map(Box::new);
1239 let (active, mut queued_commands, fingerprint) =
1240 self.move_confirmation(&selection, conversion.as_deref())?;
1241 replace_queued_images_with_placeholders(&mut queued_commands);
1242 let operation_id = previous
1243 .as_ref()
1244 .filter(|operation| {
1245 operation.selection == selection
1246 && operation.phase != MovePhase::Completed
1247 && operation.restore_artifact().is_some()
1248 })
1249 .map(|operation| operation.operation_id.clone())
1250 .unwrap_or(new_command_id("move")?);
1251 let in_place = previous
1252 .as_ref()
1253 .is_some_and(|op| retry && op.in_place && op.selection == selection)
1254 || in_place_move_eligible(
1255 source,
1256 &selection,
1257 &self.config.targets,
1258 self.state.subagents.contains_key(&source.id),
1259 retry,
1260 );
1261 let destination_has_parent_role =
1262 parent_tools_enabled(&move_subagent_policy(source, &selection), profile.kind);
1263 if in_place
1264 && !destination_has_parent_role
1265 && let Some(error) =
1266 roleless_move_children_error(&live_move_children(&self.state, &source.id))
1267 {
1268 bail!("{error}; close or finish those children before moving to this harness");
1269 }
1270 let workspace = if in_place {
1271 None
1272 } else {
1273 Some(self.assess_move_workspace(&selection, executor)?)
1274 };
1275 Ok(MovePreparation {
1276 destination_checks: if !in_place
1277 && matches!(target, mj_core::config::TargetTemplate::AwsEc2 { .. })
1278 {
1279 DestinationChecks::AfterProvisioning
1280 } else {
1281 DestinationChecks::Checked
1282 },
1283 workspace,
1284 source_unavailable: false,
1285 in_place,
1286 conversion,
1287 selection,
1288 source_profile_id: source.last_profile.clone(),
1289 source_target_template_id: source.target_template_id.clone(),
1290 cross_harness,
1291 active,
1292 queued_commands,
1293 fingerprint,
1294 operation_id,
1295 })
1296 }
1297
1298 pub async fn move_session_managed_controlled(
1300 &mut self,
1301 request: MoveSessionRequest,
1302 executor: &(impl CommandExecutor + Sync),
1303 manager: &SessionManagerControl,
1304 ) -> Result<MoveOutcome> {
1305 crate::worker_lifecycle::run(&request.preparation.selection.session_id.clone(), "move session managed controlled", executor, async {
1306 let prepared = &request.preparation;
1307 let id = prepared.selection.session_id.clone();
1308 let started = std::time::Instant::now();
1309 executor.notify_notice("Checking destination");
1310 let mut source_relay = MoveSourceRelay::default();
1311 let checked = {
1312 let _checking_destination =
1313 ProvisionStageGuard::new(executor, ProvisionStage::Verifying);
1314 let mut checked = {
1315 let _timing = MovePhaseTimer::new(&id, "preflight destination checks");
1316 self.prepare_move_session_controlled(prepared.selection.clone(), executor)
1317 .await?
1318 };
1319 if matches!(
1324 self.state.sessions[&id].state,
1325 SessionState::Running | SessionState::Disconnected
1326 ) {
1327 let source_harness = self.state.sessions[&id].harness_kind;
1328 source_relay = {
1329 let _timing = MovePhaseTimer::new(&id, "preflight source lease");
1330 MoveSourceRelay::lease(manager, &id).await?
1331 };
1332 let snapshot = source_relay.snapshot();
1333 let (active, queue, fingerprint) =
1334 self.move_confirmation(&checked.selection, checked.conversion.as_deref())?;
1335 checked.source_unavailable = snapshot
1336 .as_ref()
1337 .is_none_or(|snapshot| !snapshot.operational.native_session_is_ready());
1338 checked.active = active
1339 || checked.source_unavailable
1340 || snapshot.as_ref().is_some_and(|snapshot| {
1341 let mut operational = snapshot.operational.clone();
1342 operational.queued_prompts.clear();
1343 operational.checkpoint_barrier = None;
1344 !operational.safe_to_replace(source_harness)
1345 });
1346 if let Some(snapshot) = &snapshot {
1347 let _timing = MovePhaseTimer::new(&id, "preflight destination configuration");
1348 self.validate_move_destination_configuration(
1349 &checked.selection,
1350 source_harness,
1351 &snapshot.operational,
1352 )
1353 .await?;
1354 }
1355 checked.queued_commands = queue;
1356 checked.fingerprint = fingerprint;
1357 }
1358 checked
1359 };
1360 ensure!(
1361 checked.fingerprint == prepared.fingerprint,
1362 "session, pending work, or destination configuration changed; prepare and confirm Move again"
1363 );
1364 let queue = request.queue.unwrap_or(ResumeQueueDisposition::Discard);
1365 let source = self.state.sessions[&id].clone();
1366 if let Some(assessment) = &checked.workspace {
1367 checked.selection.workspace.validate(assessment)?;
1368 }
1369 let old_operation = crate::database::load_move_operation(&id)?;
1370 let retry = old_operation.filter(|op| {
1371 op.selection == prepared.selection
1372 && op.phase != MovePhase::Completed
1373 && op.restore_artifact().is_some()
1374 });
1375 if retry.is_none() && move_changes_nothing(&source, &checked.selection) {
1376 return Ok(outcome(
1377 &prepared.operation_id,
1378 &prepared.selection,
1379 "unchanged",
1380 None,
1381 None,
1382 ));
1383 }
1384 ensure!(
1385 !checked.active || request.acknowledge_interruption,
1386 "active work will be interrupted; confirm Move again with interruption acknowledgement"
1387 );
1388 ensure!(
1389 checked.queued_commands.is_empty() || request.queue.is_some(),
1390 "pending work requires an explicit queue choice: discard or start"
1391 );
1392 let timestamp = now();
1393 let mut operation = match retry {
1394 Some(mut op) => {
1395 ensure!(
1396 !op.queue_admission_started || op.queue == queue,
1397 "queued work may already have run; retry with the original queue choice on the same destination"
1398 );
1399 crate::database::clear_move_cancellation_for_retry(&id)?;
1400 op.configuration_fingerprint =
1401 self.move_configuration_fingerprint(&checked.selection)?;
1402 op.queue = queue;
1403 op.cancellation_requested = false;
1404 op.error = None;
1405 op
1406 }
1407 None => MoveOperation {
1408 prepared_destination: None,
1409 accepted_preparation: None,
1410 acknowledge_interruption: false,
1411 workspace_transfer: if checked.in_place {
1412 None
1413 } else {
1414 Some(
1415 self.new_workspace_transfer(
1416 &id,
1417 &prepared.operation_id,
1418 checked
1419 .workspace
1420 .clone()
1421 .context("Move workspace assessment missing")?,
1422 executor,
1423 )?,
1424 )
1425 },
1426 handoff: None,
1427 source_checkpoint_only: false,
1428 in_place: in_place_move_eligible(
1429 &source,
1430 &checked.selection,
1431 &self.config.targets,
1432 self.state.subagents.contains_key(&id),
1433 false,
1434 ),
1435 operation_id: prepared.operation_id.clone(),
1436 selection: checked.selection.clone(),
1437 source_profile_id: source.last_profile.clone(),
1438 source_target_template_id: source.target_template_id.clone(),
1439 source_target: source.target.clone(),
1440 source_native_session_id: source.native_session_id.clone(),
1441 source_additional_mounts: source.additional_mounts.clone(),
1442 source_resource_allocation: source.resource_allocation.clone(),
1443 destination_target: None,
1444 destination_native_session_id: None,
1445 destination_store_id: None,
1446 configuration_fingerprint: self
1447 .move_configuration_fingerprint(&checked.selection)?,
1448 checkpoint: None,
1449 recovery_session: None,
1450 queue,
1451 phase: MovePhase::Preparing,
1452 queue_admission_started: false,
1453 queue_admission_finished: false,
1454 cancellation_requested: false,
1455 created_at: timestamp.clone(),
1456 updated_at: timestamp,
1457 error: None,
1458 },
1459 };
1460 operation.accepted_preparation = Some(Box::new(checked.clone()));
1461 operation.acknowledge_interruption = request.acknowledge_interruption;
1462 crate::database::save_move_operation(&operation)?;
1463 tracing::info!(
1464 session_id = id,
1465 in_place = operation.in_place,
1466 reason = move_environment_change(
1467 &source,
1468 &checked.selection,
1469 bare_targets_share_environment(
1470 &self.config.targets,
1471 &source.target_template_id,
1472 checked
1473 .selection
1474 .target_template_id
1475 .as_deref()
1476 .unwrap_or_default(),
1477 ),
1478 )
1479 .unwrap_or(if operation.in_place {
1480 "environment unchanged"
1481 } else {
1482 "source environment unavailable or previously released"
1483 }),
1484 "move environment decision"
1485 );
1486 tracing::info!(
1487 session_id = id,
1488 phase = "preflight",
1489 elapsed_ms = started.elapsed().as_millis() as u64,
1490 "move phase completed"
1491 );
1492 let result = Box::pin(self.execute_move(
1495 &mut operation,
1496 Some(&checked),
1497 executor,
1498 manager,
1499 source_relay,
1500 ))
1501 .await;
1502 self.finish_move_result_with_subagent_recovery(
1503 &mut operation,
1504 result,
1505 executor,
1506 manager,
1507 )
1508 .await
1509
1510 }).await
1511 }
1512
1513 fn finish_move_result(
1514 &mut self,
1515 operation: &mut MoveOperation,
1516 result: Result<()>,
1517 executor: &impl CommandExecutor,
1518 ) -> Result<MoveOutcome> {
1519 if result.is_err()
1520 && operation.workspace_transfer.is_some()
1521 && !crate::upgrade::gate().is_open()
1522 {
1523 return Ok(outcome(
1526 &operation.operation_id,
1527 &operation.selection,
1528 "interrupted",
1529 None,
1530 Some("Move will continue after the daemon upgrade".into()),
1531 ));
1532 }
1533 if let Some(saved) = crate::database::load_move_operation(&operation.selection.session_id)?
1534 && saved.operation_id == operation.operation_id
1535 {
1536 operation.prepared_destination = saved.prepared_destination;
1537 }
1538 let result = match result {
1539 Err(error)
1540 if !operation.queue_admission_started
1541 && operation
1542 .prepared_destination
1543 .as_ref()
1544 .is_some_and(|d| d.owns_resource()) =>
1545 {
1546 executor.begin_resumable_move_work()?;
1547 let cleaned = self.cleanup_prepared_move_destination(operation, executor);
1548 executor.end_resumable_move_work()?;
1549 Err(match cleaned {
1550 Ok(()) => error,
1551 Err(cleanup_error) => error.context(format!("EC2 destination cleanup is pending and will retry automatically: {cleanup_error:#}")),
1552 })
1553 }
1554 result => result,
1555 };
1556 let session_id = operation.selection.session_id.clone();
1557 let mut last_error = self
1558 .state
1559 .sessions
1560 .get(&session_id)
1561 .context("Move outcome session is missing")?
1562 .last_error
1563 .clone();
1564 let (status, error, recovery) = match result {
1565 Ok(()) => {
1566 operation.phase = MovePhase::Completed;
1567 operation.error = None;
1568 if last_error
1569 .as_deref()
1570 .is_some_and(|error| error.starts_with(mj_core::state::MOVE_FAILURE_PREFIX))
1571 {
1572 last_error = None;
1573 }
1574 ("completed", None, None)
1575 }
1576 Err(error) => {
1577 let phase = match operation.phase {
1578 MovePhase::Preparing => "preparing the destination",
1579 MovePhase::ClosingSource => "checkpointing and suspending the source",
1580 MovePhase::ResumingDestination => "resuming the destination",
1581 MovePhase::StartingQueue => "starting the destination queue",
1582 MovePhase::Completed | MovePhase::Failed | MovePhase::Cancelled => {
1583 "recovering the move"
1584 }
1585 };
1586 let cancelled =
1587 executor.cancellation_requested() || operation.cancellation_requested;
1588 operation.phase = if cancelled {
1589 MovePhase::Cancelled
1590 } else {
1591 MovePhase::Failed
1592 };
1593 operation.cancellation_requested = cancelled;
1594 let recovery = failed_move_recovery(operation, &self.state);
1595 let error = format!("{error:#}");
1596 tracing::warn!(
1597 %session_id,
1598 reference = %operation.operation_id,
1599 phase,
1600 cancelled,
1601 %error,
1602 "session move did not finish"
1603 );
1604 last_error = Some(failed_move_message(
1605 Some(phase),
1606 cancelled,
1607 &recovery,
1608 &operation.operation_id,
1609 ));
1610 operation.error = Some(error.clone());
1611 (
1612 if cancelled { "cancelled" } else { "failed" },
1613 Some(error),
1614 Some(recovery),
1615 )
1616 }
1617 };
1618 operation.updated_at = now();
1619 crate::database::save_move_outcome(operation, last_error.as_deref())?;
1620 let record = self
1621 .state
1622 .sessions
1623 .get_mut(&session_id)
1624 .expect("Move outcome session");
1625 record.last_error = last_error;
1626 record.updated_at = operation.updated_at.clone();
1627 Ok(outcome(
1628 &operation.operation_id,
1629 &operation.selection,
1630 status,
1631 error,
1632 recovery,
1633 ))
1634 }
1635
1636 async fn finish_move_result_with_subagent_recovery(
1637 &mut self,
1638 operation: &mut MoveOperation,
1639 result: Result<()>,
1640 executor: &(impl CommandExecutor + Sync),
1641 manager: &SessionManagerControl,
1642 ) -> Result<MoveOutcome> {
1643 let outcome = self.finish_move_result(operation, result, executor)?;
1644 if !operation.in_place
1645 || !matches!(operation.phase, MovePhase::Failed | MovePhase::Cancelled)
1646 {
1647 return Ok(outcome);
1648 }
1649 let session_id = &operation.selection.session_id;
1650 let Some(session) = self.state.sessions.get(session_id) else {
1651 return Ok(outcome);
1652 };
1653 if !matches!(
1654 session.state,
1655 SessionState::Running | SessionState::Disconnected
1656 ) || !worker_subagent_queue_enabled(
1657 &session.subagents.clone().unwrap_or_default(),
1658 session.harness_kind,
1659 self.state.subagents.get(session_id),
1660 self.config.agent_mailboxes_enabled_for(session),
1661 ) {
1662 return Ok(outcome);
1663 }
1664
1665 for attempt in 1..=2 {
1669 match MoveSourceRelay::set_subagent_admission_via_manager(manager, session_id, true)
1670 .await
1671 {
1672 Ok(true) => {
1673 if attempt > 1 {
1674 tracing::info!(
1675 session_id,
1676 "reopened sub-agent requests on a replacement relay connection"
1677 );
1678 }
1679 break;
1680 }
1681 Ok(false) => break,
1682 Err(error) => {
1683 tracing::warn!(
1684 session_id,
1685 attempt,
1686 error = %error,
1687 "could not reopen sub-agent requests after the in-place Move left its source running"
1688 );
1689 if attempt == 1 {
1690 tokio::task::yield_now().await;
1691 }
1692 }
1693 }
1694 }
1695 Ok(outcome)
1696 }
1697
1698 pub async fn recover_move_managed_controlled(
1699 &mut self,
1700 mut operation: MoveOperation,
1701 executor: &(impl CommandExecutor + Sync),
1702 manager: &SessionManagerControl,
1703 ) -> Result<MoveOutcome> {
1704 crate::worker_lifecycle::run(&operation.selection.session_id.clone(), "recover move managed controlled", executor, async {
1705 let id = operation.selection.session_id.clone();
1706 let result = async {
1707 let session = self.state.sessions.get(&id).context("move session is missing")?.clone();
1708 if operation.phase == MovePhase::Preparing
1709 && operation.accepted_preparation.is_some()
1710 && matches!(self.config.targets.get(operation.selection.target_template_id.as_deref().unwrap_or_default()),
1711 Some(mj_core::config::TargetTemplate::AwsEc2 { .. }))
1712 && !operation.cancellation_requested {
1713 return Box::pin(self.execute_move(&mut operation, None, executor, manager, MoveSourceRelay::default())).await;
1714 }
1715 if operation.queue_admission_started {
1716 ensure!(!operation.cancellation_requested, "Move was cancelled; destination retained without further queue admission");
1717 self.finish_workspace_transfer(&mut operation, executor)?;
1718 return self.admit_move_queue(&mut operation, executor).await;
1719 }
1720 if operation.workspace_transfer.is_some() && operation.recovery_session.is_some() && !operation.cancellation_requested
1721 && !(operation.phase == MovePhase::ResumingDestination && session.state == SessionState::Running) {
1722 if session.state == SessionState::Provisioning || session.target != operation.source_target {
1723 self.rollback_move_destination(&operation, anyhow::anyhow!("resume interrupted Move transfer"), executor)?;
1724 }
1725 return Box::pin(self.execute_move(&mut operation, None, executor, manager, MoveSourceRelay::default())).await;
1726 }
1727 if matches!(session.state, SessionState::Closing | SessionState::Destroying) {
1728 let cleanup = crate::targets::CancellableProcessExecutor::with_timeout(std::time::Duration::from_secs(15));
1729 let sealed = operation.in_place && operation.recovery_session.is_some() && session.state == SessionState::Closing;
1735 if !sealed {
1736 Box::pin(self.recover_move_source_stop(&mut operation, &cleanup, manager)).await?;
1737 }
1738 if !operation.retains_source_environment() {
1742 operation.checkpoint = self.state.sessions[&id].checkpoint.clone();
1743 }
1744 if operation.retains_source_environment() && self.state.sessions[&id].state == SessionState::Closing {
1745 let previous = self.state.sessions[&id].clone();
1746 operation.recovery_session = Some(previous.clone());
1747 let cause = anyhow::anyhow!("Move source sealed; environment retained for explicit retry");
1748 return Err(self.retain_failed_in_place_move(&id, &previous, cause)?);
1749 }
1750 if !operation.retains_source_environment() && self.state.sessions[&id].state == SessionState::Stopped && self.state.sessions[&id].target.is_some() {
1751 self.cleanup_stopped_target(&id, &cleanup)?;
1752 }
1753 bail!("{}", interrupted_source_stop_message(
1754 "Move source stop recovered; no destination work was started. Retry Move or Resume with previous settings",
1755 operation.in_place,
1756 ));
1757 }
1758 match operation.phase {
1759 MovePhase::ClosingSource
1760 if operation.in_place
1761 && matches!(
1762 session.state,
1763 SessionState::Running | SessionState::Disconnected
1764 )
1765 && !operation.cancellation_requested =>
1766 {
1767 let relay = MoveSourceRelay::lease(manager, &id).await?;
1771 return Box::pin(self.execute_move(
1772 &mut operation,
1773 None,
1774 executor,
1775 manager,
1776 relay,
1777 ))
1778 .await;
1779 }
1780 MovePhase::ClosingSource
1781 if operation.in_place
1782 && matches!(
1783 session.state,
1784 SessionState::Running | SessionState::Disconnected
1785 )
1786 && operation.cancellation_requested =>
1787 {
1788 let source = self.state.sessions.get(&id).context("move source is missing")?;
1789 if parent_tools_enabled(
1790 &source.subagents.clone().unwrap_or_default(),
1791 source.harness_kind,
1792 ) {
1793 MoveSourceRelay::set_subagent_admission_via_manager(
1794 manager, &id, true,
1795 )
1796 .await?;
1797 }
1798 bail!("Move was cancelled before source interruption; source retained and sub-agent requests reopened")
1799 }
1800 MovePhase::Preparing => bail!("Move preparation was interrupted; source retained. Prepare Move again."),
1801 MovePhase::ResumingDestination if session.state == SessionState::Running => {
1802 ensure!(Some(&session.last_profile) == operation.selection.profile_id.as_ref()
1806 && Some(&session.target_template_id) == operation.selection.target_template_id.as_ref(),
1807 "ready destination does not match the move intent");
1808 operation.destination_target = session.target.clone();
1809 operation.destination_native_session_id = session.native_session_id.clone();
1810 operation.queue_admission_started = true;
1811 operation.phase = MovePhase::StartingQueue;
1812 crate::database::save_move_operation(&operation)?;
1813 restore_move_queue_hold(&operation);
1814 ensure!(!operation.cancellation_requested, "Move was cancelled; ready destination retained");
1815 self.finish_workspace_transfer(&mut operation, executor)?;
1816 self.admit_move_queue(&mut operation, executor).await
1817 }
1818 MovePhase::ResumingDestination => {
1819 let previous = operation.recovery_session.as_ref().context("move lacks its stopped recovery identity; retain resources for inspection")?;
1820 let cause = anyhow::anyhow!("destination restoration was interrupted; checkpoint retained for an explicit retry");
1823 let error = if operation.in_place {
1824 self.retain_failed_in_place_move(&id, previous, cause)?
1825 } else if operation.workspace_transfer.is_some() {
1826 self.rollback_move_destination(&operation, cause, executor)?
1827 } else {
1828 self.rollback_failed_resume(&id, previous, false, cause, executor)?
1829 };
1830 Err(error)
1831 }
1832 MovePhase::ClosingSource => {
1833 if matches!(session.state, SessionState::Closing | SessionState::Destroying) {
1834 Box::pin(self.recover_move_source_stop(&mut operation, executor, manager)).await?;
1835 }
1836 operation.checkpoint = self.state.sessions[&id].checkpoint.clone();
1837 if !operation.retains_source_environment() && self.state.sessions[&id].state == SessionState::Stopped && self.state.sessions[&id].target.is_some() {
1838 self.cleanup_stopped_target(&id, executor)?;
1839 }
1840 bail!("{}", interrupted_source_stop_message(
1841 "source stop was recovered; verified checkpoint retained. Retry move or Resume with previous settings",
1842 operation.in_place,
1843 ))
1844 }
1845 _ => bail!("Move requires an explicit retry after the daemon restarted"),
1846 }
1847 }.await;
1848 self.finish_move_result_with_subagent_recovery(
1849 &mut operation,
1850 result,
1851 executor,
1852 manager,
1853 )
1854 .await
1855
1856 }).await
1857 }
1858
1859 async fn recover_move_source_stop(
1860 &mut self,
1861 operation: &mut MoveOperation,
1862 executor: &(impl CommandExecutor + Sync),
1863 manager: &SessionManagerControl,
1864 ) -> Result<()> {
1865 let id = operation.selection.session_id.clone();
1866 if self.state.sessions[&id].state == SessionState::Closing {
1867 self.prepare_move_source_checkpoint(
1868 &id,
1869 executor,
1870 manager,
1871 operation,
1872 &mut MoveSourceRelay::default(),
1873 )
1874 .await?;
1875 let handle = manager
1876 .wait_for_session(&id, std::time::Duration::from_secs(5))
1877 .await?;
1878 let mut lease = handle.lease_connection().await?;
1879 let execution = lease.connection_mut().sync().await?.operational.execution;
1880 if operation.retains_source_environment()
1881 && matches!(
1882 execution,
1883 mj_core::relay::RelayExecutionState::Closing
1884 | mj_core::relay::RelayExecutionState::Closed
1885 )
1886 {
1887 super::checkpoint::wait_for_relay_closed(lease.connection_mut()).await?;
1888 lease.release();
1889 return Ok(());
1894 }
1895 lease.release();
1896 if matches!(
1897 execution,
1898 mj_core::relay::RelayExecutionState::Idle
1899 | mj_core::relay::RelayExecutionState::Running
1900 ) {
1901 if operation.cancellation_requested || operation.retains_source_environment() {
1902 let record = self.state.sessions.get_mut(&id).unwrap();
1903 record.state = SessionState::Running;
1904 record.updated_at = now();
1905 record.last_error = Some(
1906 "Move was interrupted before the source was sealed; source retained".into(),
1907 );
1908 crate::database::save_lifecycle_session(record)?;
1909 } else {
1910 Box::pin(self.suspend_session_for_move(
1912 &id,
1913 executor,
1914 manager,
1915 operation,
1916 None,
1917 SourceTargetDisposition::Destroy,
1918 MoveSourceRelay::default(),
1919 ))
1920 .await?;
1921 }
1922 return Ok(());
1923 }
1924 }
1925 ensure!(
1926 !operation.retains_source_environment(),
1927 "retained Move source cannot be proven; refusing target teardown"
1928 );
1929 self.recover_interrupted_close_managed(&id, executor, manager, true, None)
1931 .await?;
1932 Ok(())
1933 }
1934
1935 async fn execute_move(
1938 &mut self,
1939 operation: &mut MoveOperation,
1940 preparation: Option<&MovePreparation>,
1941 executor: &(impl CommandExecutor + Sync),
1942 manager: &SessionManagerControl,
1943 mut source_relay: MoveSourceRelay,
1944 ) -> Result<()> {
1945 let retained = source_relay.owner();
1946 let admission_id = operation.selection.session_id.clone();
1947 crate::worker_lifecycle::run_with_owner(&admission_id, "execute move", executor, retained, async {
1948 let id = operation.selection.session_id.clone();
1949 let mut preparation = preparation.cloned();
1950 if let Some(saved) = crate::database::load_move_operation(&id)?
1951 && saved.operation_id == operation.operation_id
1952 {
1953 operation.prepared_destination = saved.prepared_destination;
1954 }
1955
1956 if !operation.in_place
1957 && !operation.queue_admission_started
1958 && matches!(
1959 self.config.targets.get(
1960 operation
1961 .selection
1962 .target_template_id
1963 .as_deref()
1964 .unwrap_or_default()
1965 ),
1966 Some(mj_core::config::TargetTemplate::AwsEc2 { .. })
1967 )
1968 {
1969 drop(source_relay);
1970 source_relay = MoveSourceRelay::default();
1971 self.verify_current_move_configuration(operation)?;
1972 if self.state.sessions[&id].state == SessionState::Error
1976 && operation.recovery_session.is_some()
1977 {
1978 ensure!(
1979 !forget_missing_move_archives(operation),
1980 "the Move's checkpoint archive is missing; nothing is left to restore"
1981 );
1982 self.rollback_move_destination(
1983 operation,
1984 anyhow::anyhow!("clean up the partial Move destination before retry"),
1985 executor,
1986 )?;
1987 let record = self.state.sessions.get_mut(&id).unwrap();
1988 record.state = SessionState::Closing;
1989 crate::database::save_resumed_session(record, None)?;
1990 operation.prepared_destination = crate::database::load_move_operation(&id)?
1991 .context("Move cleanup intent missing")?
1992 .prepared_destination;
1993 }
1994 self.prepare_ec2_move_destination(operation, executor)?;
1995 self.verify_current_move_configuration(operation)?;
1996 if matches!(
1997 self.state.sessions[&id].state,
1998 SessionState::Running | SessionState::Disconnected
1999 ) && operation.recovery_session.is_none()
2000 {
2001 let accepted = operation
2002 .accepted_preparation
2003 .as_ref()
2004 .context("EC2 Move confirmation missing")?;
2005 let mut checked = self
2006 .prepare_move_session_controlled(operation.selection.clone(), executor)
2007 .await?;
2008 let harness = self.state.sessions[&id].harness_kind;
2009 source_relay = MoveSourceRelay::lease(manager, &id).await?;
2010 let snapshot = source_relay.snapshot();
2011 let (active, queue, fingerprint) =
2012 self.move_confirmation(&checked.selection, checked.conversion.as_deref())?;
2013 let unavailable = snapshot
2014 .as_ref()
2015 .is_none_or(|s| !s.operational.native_session_is_ready());
2016 let active = active
2017 || unavailable
2018 || snapshot.as_ref().is_some_and(|s| {
2019 let mut state = s.operational.clone();
2020 state.queued_prompts.clear();
2021 state.checkpoint_barrier = None;
2022 !state.safe_to_replace(harness)
2023 });
2024 ensure!(
2025 fingerprint == accepted.fingerprint,
2026 "session, pending work, or destination configuration changed; source retained, prepare and confirm Move again"
2027 );
2028 ensure!(
2029 !active || operation.acknowledge_interruption,
2030 "active work will be interrupted; source retained, confirm Move again with interruption acknowledgement"
2031 );
2032 if let Some(snapshot) = snapshot {
2033 self.validate_move_destination_configuration(
2034 &checked.selection,
2035 harness,
2036 &snapshot.operational,
2037 )
2038 .await?;
2039 }
2040 checked.active = active;
2041 checked.queued_commands = queue;
2042 checked.source_unavailable = unavailable;
2043 let assessment = checked
2044 .workspace
2045 .as_mut()
2046 .context("EC2 Move workspace missing")?;
2047 self.assess_prepared_destination(operation, assessment, executor)?;
2048 operation
2049 .workspace_transfer
2050 .as_mut()
2051 .context("EC2 Move transfer missing")?
2052 .assessment = assessment.clone();
2053 crate::database::save_move_operation(operation)?;
2054 preparation = Some(checked);
2055 } else {
2056 let mut assessment = operation
2057 .workspace_transfer
2058 .as_ref()
2059 .context("EC2 Move transfer missing")?
2060 .assessment
2061 .clone();
2062 assessment
2063 .storage
2064 .retain(|s| !s.allocations.contains("destination"));
2065 self.assess_prepared_destination(operation, &mut assessment, executor)?;
2066 }
2067 }
2068 ensure!(
2069 !executor.cancellation_requested(),
2070 "move cancelled before source interruption"
2071 );
2072 if !operation.queue_admission_started {
2073 if forget_missing_move_archives(operation) {
2078 operation.updated_at = now();
2079 crate::database::save_move_operation(operation)?;
2080 bail!("the Move's checkpoint archive is missing; nothing is left to restore");
2081 }
2082 if operation.in_place {
2083 ensure!(
2084 operation.source_target.is_some()
2085 && self.state.sessions[&id].target == operation.source_target,
2086 "retained Move target is missing or changed; refusing to recreate the environment"
2087 );
2088 }
2089 if self.state.sessions[&id].state == SessionState::Error
2090 && let Some(previous) = operation.recovery_session.as_ref()
2091 {
2092 let cause = anyhow::anyhow!("clean up the partial Move destination before retry");
2095 if operation.in_place {
2096 self.retain_failed_in_place_move(&id, previous, cause)?;
2102 } else if operation.workspace_transfer.is_some() {
2103 self.rollback_move_destination(operation, cause, executor)?;
2104 let record = self.state.sessions.get_mut(&id).unwrap();
2105 record.state = SessionState::Closing;
2106 crate::database::save_resumed_session(record, None)?;
2107 } else {
2108 let failure =
2109 self.rollback_failed_resume(&id, previous, false, cause, executor)?;
2110 ensure!(
2111 self.state.sessions[&id].state == SessionState::Stopped,
2112 "{failure:#}"
2113 );
2114 }
2115 }
2116 let state = self.state.sessions[&id].state;
2117 if matches!(state, SessionState::Closing | SessionState::Destroying)
2118 && !(operation.retains_source_environment() && operation.recovery_session.is_some())
2119 {
2120 Box::pin(self.recover_move_source_stop(operation, executor, manager)).await?;
2121 operation.checkpoint = self.state.sessions[&id].checkpoint.clone();
2122 }
2123 if matches!(
2124 self.state.sessions[&id].state,
2125 SessionState::Running | SessionState::Disconnected
2126 ) && operation.destination_target.is_none()
2127 {
2128 let source = self.state.sessions[&id].clone();
2129 let source_has_parent_role = parent_tools_enabled(
2130 &source.subagents.clone().unwrap_or_default(),
2131 source.harness_kind,
2132 );
2133 let destination_has_parent_role = parent_tools_enabled(
2134 &move_subagent_policy(&source, &operation.selection),
2135 operation
2136 .selection
2137 .profile_id
2138 .as_deref()
2139 .and_then(|profile| self.config.profiles.get(profile))
2140 .context("destination profile is missing")?
2141 .kind,
2142 );
2143 if operation.in_place {
2144 operation.phase = MovePhase::ClosingSource;
2148 operation.updated_at = now();
2149 crate::database::save_move_operation(operation)?;
2150 let mut stopped_subagents_for_legacy_worker = false;
2151 if source_has_parent_role {
2152 if !source_relay.is_held() {
2153 source_relay = MoveSourceRelay::lease(manager, &id).await?;
2154 }
2155 match source_relay.drain_subagent_mutations(&id, executor).await? {
2156 SubagentMutationDrain::Drained => {}
2157 SubagentMutationDrain::UnsupportedWorkerProtocol(version) => {
2158 executor.notify_notice(&format!(
2159 "The source worker uses relay protocol {version}; protocol 32 is required to safely drain sub-agent requests, so stopping sub-agents before this in-place Move"
2160 ));
2161 executor.before_move_source_stop().await?;
2162 stopped_subagents_for_legacy_worker = true;
2163 }
2164 }
2165 }
2166 if should_stop_move_subagents(true, destination_has_parent_role)
2167 && !stopped_subagents_for_legacy_worker
2168 {
2169 let current = crate::database::load_state()?;
2170 if let Some(error) =
2171 roleless_move_children_error(&live_move_children(¤t, &id))
2172 {
2173 if source_has_parent_role {
2174 source_relay
2175 .reopen_subagent_mutations(&id)
2176 .await
2177 .context("could not reopen sub-agent requests after refusing Move")?;
2178 }
2179 bail!("{error}; close or finish those children before moving to this harness");
2180 }
2181 if let Err(error) = executor.before_move_source_stop().await {
2182 if source_has_parent_role {
2183 source_relay
2184 .reopen_subagent_mutations(&id)
2185 .await
2186 .context("could not reopen sub-agent requests after stopping children failed")?;
2187 }
2188 return Err(error);
2189 }
2190 }
2191 } else {
2192 executor.before_move_source_stop().await?;
2195 }
2196 if executor.cancellation_requested() {
2197 if operation.in_place && source_has_parent_role {
2198 source_relay.reopen_subagent_mutations(&id).await?;
2199 }
2200 bail!("Move cancelled before source interruption");
2201 }
2202 executor.notify_notice("Stopping source");
2203 let _timing = MovePhaseTimer::new(&id, "checkpoint and source stop");
2204 if !operation.in_place {
2205 operation.phase = MovePhase::ClosingSource;
2206 operation.updated_at = now();
2207 crate::database::save_move_operation(operation)?;
2208 }
2209 let disposition = if operation.in_place {
2213 SourceTargetDisposition::RetainForInPlaceSwap
2214 } else {
2215 SourceTargetDisposition::Destroy
2216 };
2217 let suspended = Box::pin(self.suspend_session_for_move(
2218 &id,
2219 executor,
2220 manager,
2221 operation,
2222 preparation.as_ref(),
2223 disposition,
2224 std::mem::take(&mut source_relay),
2225 ))
2226 .await;
2227 if let Err(error) = suspended {
2228 if operation.in_place
2229 && source_has_parent_role
2230 && matches!(
2231 self.state.sessions[&id].state,
2232 SessionState::Running | SessionState::Disconnected
2233 )
2234 && let Err(reopen) = MoveSourceRelay::set_subagent_admission_via_manager(
2235 manager,
2236 &id,
2237 true,
2238 )
2239 .await
2240 {
2241 return Err(error.context(format!(
2242 "could not reopen sub-agent requests after source checkpoint failed: {reopen:#}"
2243 )));
2244 }
2245 return Err(error);
2246 }
2247 }
2248 drop(source_relay);
2250 if !operation.in_place
2253 && operation.workspace_transfer.is_none()
2254 && self.state.sessions[&id].state == SessionState::Stopped
2255 && self.state.sessions[&id].target.is_some()
2256 {
2257 executor.notify_notice("Cleaning up source");
2258 let _timing = MovePhaseTimer::new(&id, "source storage cleanup");
2259 self.cleanup_stopped_target(&id, executor)?;
2260 }
2261 operation.checkpoint = operation
2262 .checkpoint
2263 .clone()
2264 .or_else(|| self.state.sessions[&id].checkpoint.clone());
2265 ensure!(
2266 operation.restore_artifact().is_some(),
2267 "move has no verified checkpoint"
2268 );
2269 ensure!(
2270 !executor.cancellation_requested(),
2271 "move cancelled after source sealing; checkpoint and remaining environment retained"
2272 );
2273 operation.phase = MovePhase::ResumingDestination;
2274 operation.recovery_session = Some(self.state.sessions[&id].clone());
2275 operation.updated_at = now();
2276 crate::database::save_move_operation(operation)?;
2277 executor.reserve_move_destination();
2278 executor.notify_notice("Preparing destination");
2279 if let Some(policy) = &operation.selection.subagents {
2283 let session = self.state.sessions.get_mut(&id).unwrap();
2284 session.subagents = Some(policy.clone());
2285 crate::database::save_resumed_session(session, None)?;
2286 }
2287 if operation.in_place {
2288 Box::pin(self.restore_session_in_place(
2292 &id,
2293 operation.selection.profile_id.as_deref().unwrap(),
2294 operation.selection.target_template_id.as_deref().unwrap(),
2295 executor,
2296 ))
2297 .await?;
2298 } else {
2299 if operation.selection.clear_resource_allocation {
2300 let session = self.state.sessions.get_mut(&id).unwrap();
2301 session.resource_allocation = None;
2302 session.container_cpus = None;
2303 session.container_memory = None;
2304 crate::database::save_resumed_session(session, None)?;
2305 }
2306 if operation.workspace_transfer.is_some() {
2307 self.capture_move_workspace(operation, executor)?;
2308 Box::pin(self.resume_session_for_move(operation, executor)).await?;
2309 let saved = crate::database::load_move_operation(&id)?
2310 .context("Move intent disappeared during restore")?;
2311 operation.workspace_transfer = saved.workspace_transfer;
2312 operation.prepared_destination = saved.prepared_destination;
2313 } else {
2314 Box::pin(self.resume_session_controlled(
2315 &id,
2316 operation.selection.profile_id.as_deref().unwrap(),
2317 operation.selection.target_template_id.as_deref().unwrap(),
2318 SessionResumeOptions {
2319 additional_mounts: operation.selection.additional_mounts.clone(),
2320 resource_allocation: operation.selection.resource_allocation.clone(),
2321 discard_queue: true,
2322 },
2323 executor,
2324 ))
2325 .await?;
2326 }
2327 }
2328 let destination = &self.state.sessions[&id];
2329 operation.destination_target = destination.target.clone();
2330 operation.destination_native_session_id = destination.native_session_id.clone();
2331 operation.phase = MovePhase::StartingQueue;
2333 operation.queue_admission_started = true;
2334 operation.updated_at = now();
2335 crate::database::save_move_operation(operation)?;
2336 }
2337 restore_move_queue_hold(operation);
2338 self.finish_workspace_transfer(operation, executor)?;
2339 self.admit_move_queue(operation, executor).await
2340
2341 }).await
2342 }
2343
2344 async fn admit_move_queue(
2345 &self,
2346 operation: &mut MoveOperation,
2347 executor: &(impl CommandExecutor + Sync),
2348 ) -> Result<()> {
2349 let timing_id = operation.selection.session_id.clone();
2350 let _timing = MovePhaseTimer::new(&timing_id, "queue admission");
2351 let id = &operation.selection.session_id;
2352 let mut relay = {
2353 let _checking_destination =
2354 ProvisionStageGuard::new(executor, ProvisionStage::Verifying);
2355 let destination = &self.state.sessions[id];
2356 ensure!(
2357 destination.state == SessionState::Running
2358 && destination.target == operation.destination_target
2359 && destination.native_session_id == operation.destination_native_session_id,
2360 "cannot prove the same ready destination; refusing to replay potentially executed work"
2361 );
2362 let spec = self.reconnect_command(id)?;
2363 let relay = StandaloneSession::connect_command(&spec, id).await?;
2364 let store_id = relay.snapshot().operational.store_id.context("destination worker does not expose its durable store identity; upgrade the worker before admitting queued work")?;
2365 if let Some(expected) = &operation.destination_store_id {
2366 ensure!(
2367 *expected == store_id,
2368 "destination relay storage was replaced; refusing to replay potentially executed work"
2369 );
2370 } else {
2371 operation.destination_store_id = Some(store_id);
2373 crate::database::save_move_operation(operation)?;
2374 }
2375 ensure!(
2376 relay.snapshot().operational.native_session_id
2377 == operation.destination_native_session_id,
2378 "destination relay native identity changed; refusing queue replay"
2379 );
2380 relay
2381 };
2382 if operation.queue == ResumeQueueDisposition::Start {
2383 let _starting_queue = ProvisionStageGuard::new(executor, ProvisionStage::Starting);
2384 executor.notify_notice("Starting queued work");
2385 let checkpoint = operation
2386 .restore_artifact()
2387 .context("move queue archive is missing")?;
2388 let verified = verify_archive_streaming(&checkpoint.archive_path)?;
2389 ensure!(
2390 verified.archive_sha256 == checkpoint.sha256 && verified.manifest.session.id == *id,
2391 "move queue checkpoint verification failed"
2392 );
2393 for queued in verified.canonical_session.queued_prompts {
2394 if operation.queue_admission_finished {
2395 continue;
2396 }
2397 ensure!(
2398 !executor.cancellation_requested(),
2399 "move cancelled during queue admission; destination retained"
2400 );
2401 let command = match queued.kind {
2402 CanonicalQueuedCommandKind::Prompt => RelayCommand::Prompt {
2403 prompt: queued
2404 .content
2405 .into_iter()
2406 .map(serde_json::from_value)
2407 .collect::<serde_json::Result<_>>()?,
2408 },
2409 CanonicalQueuedCommandKind::SetConfig { key, value } => {
2410 RelayCommand::SetConfig { key, value }
2411 }
2412 };
2413 relay.submit_accepted(queued.command_id, command).await?;
2414 }
2415 }
2416 operation.queue_admission_finished = true;
2417 crate::database::save_move_operation(operation)?;
2418 restore_move_queue_hold(operation);
2419 let queue_sentence = if operation.queue == ResumeQueueDisposition::Discard {
2420 "Queued work was discarded; ready and idle."
2421 } else {
2422 "Queued work was accepted."
2423 };
2424 let source_profile = &operation.source_profile_id;
2425 let source_target = &operation.source_target_template_id;
2426 let destination_profile = operation.selection.profile_id.as_deref().unwrap();
2427 let destination_target = operation.selection.target_template_id.as_deref().unwrap();
2428 let text = if operation.in_place {
2431 format!(
2432 "Switched from {source_profile} / {source_target} to {destination_profile} / {destination_target} in place; the workspace and environment were kept. {queue_sentence} The interrupted prompt was not replayed."
2433 )
2434 } else {
2435 format!(
2436 "Moved from {source_profile} / {source_target} to {destination_profile} / {destination_target} in a fresh environment. {queue_sentence} The interrupted prompt was not replayed."
2437 )
2438 };
2439 relay
2440 .submit(
2441 format!("{}-notice", operation.operation_id),
2442 RelayCommand::RecordNotice { text },
2443 )
2444 .await?;
2445 Ok(())
2446 }
2447
2448 pub(super) fn validate_move_checkpoint(
2449 &self,
2450 operation: &MoveOperation,
2451 preparation: Option<&MovePreparation>,
2452 github_token: Option<&str>,
2453 executor: &(impl CommandExecutor + Sync),
2454 ) -> Result<()> {
2455 let _verifying = ProvisionStageGuard::new(executor, ProvisionStage::Verifying);
2456 let current = Controller {
2457 config: mj_core::config::Config::load()?,
2458 state: self.state.clone(),
2459 };
2460 ensure!(
2461 current.move_configuration_fingerprint(&operation.selection)?
2462 == operation.configuration_fingerprint,
2463 "destination configuration changed during move"
2464 );
2465 let id = &operation.selection.session_id;
2466 current.validate_move_destination_paths(
2467 &self.state.sessions[id],
2468 operation.selection.target_template_id.as_deref().unwrap(),
2469 executor,
2470 )?;
2471 if !operation.in_place
2472 && operation.workspace_transfer.is_none()
2473 && let super::ResumeRepositorySourcePreflight::RepositoryMoved(mismatch) = self
2474 .preflight_resume_repository_sources_with_token(
2475 id,
2476 operation.selection.target_template_id.as_deref().unwrap(),
2477 github_token,
2478 executor,
2479 )?
2480 {
2481 bail!(
2482 "destination repository source is missing checkpoint commit {}; source retained",
2483 mismatch.missing_commit
2484 );
2485 }
2486 if let Some(prepared) = preparation {
2487 let checkpoint = operation
2488 .restore_artifact()
2489 .or(self.state.sessions[id].checkpoint.as_ref())
2490 .context("no move checkpoint")?;
2491 let verified = verify_archive_streaming(&checkpoint.archive_path)?;
2492 let actual: Vec<_> = verified
2493 .canonical_session
2494 .queued_prompts
2495 .iter()
2496 .map(|p| p.command_id.as_str())
2497 .collect();
2498 let expected: Vec<_> = prepared
2499 .queued_commands
2500 .iter()
2501 .map(|p| p.command_id.as_str())
2502 .collect();
2503 ensure!(
2504 actual == expected,
2505 "pending queue changed before checkpoint capture; source retained, confirm Move again"
2506 );
2507 }
2508 Ok(())
2509 }
2510}
2511
2512fn validate_preserved_configuration(
2513 profile_id: &str,
2514 accepted: &mj_core::acp::AcceptedSessionConfig,
2515 choices: &mj_core::worker_launch::ProfileConfig,
2516) -> Result<()> {
2517 for (key, value, offered) in [
2518 ("model", accepted.model.as_deref(), &choices.models),
2519 ("effort", accepted.effort.as_deref(), &choices.efforts),
2520 ] {
2521 let Some(value) = value else { continue };
2522 ensure!(
2523 offered.iter().any(|choice| choice.value == value),
2524 "destination profile {profile_id:?} does not offer the session's accepted {key} {value:?}; choices: {}",
2525 offered
2526 .iter()
2527 .map(|choice| choice.value.as_str())
2528 .collect::<Vec<_>>()
2529 .join(", ")
2530 );
2531 }
2532 Ok(())
2533}
2534
2535fn move_subagent_policy(
2536 source: &mj_core::state::SessionRecord,
2537 selection: &MoveSelection,
2538) -> mj_core::subagent::SubagentPolicy {
2539 selection
2540 .subagents
2541 .clone()
2542 .or_else(|| source.subagents.clone())
2543 .unwrap_or_default()
2544}
2545
2546fn move_changes_nothing(source: &mj_core::state::SessionRecord, selection: &MoveSelection) -> bool {
2551 source.state == SessionState::Running
2552 && selection.profile_id.as_deref() == Some(source.last_profile.as_str())
2553 && move_environment_change(source, selection, false).is_none()
2554 && move_subagent_policy(source, selection) == source.subagents.clone().unwrap_or_default()
2555}
2556
2557pub(crate) fn parent_tools_enabled(
2558 policy: &mj_core::subagent::SubagentPolicy,
2559 harness: mj_core::config::HarnessKind,
2560) -> bool {
2561 policy.for_launch(harness, false).parent_role().is_some()
2562}
2563
2564pub(crate) fn worker_subagent_queue_enabled(
2568 policy: &mj_core::subagent::SubagentPolicy,
2569 harness: mj_core::config::HarnessKind,
2570 subagent: Option<&mj_core::subagent::SubagentRecord>,
2571 agent_mailboxes_enabled: bool,
2572) -> bool {
2573 let managed_subagent = subagent.is_some();
2574 let launch_policy = policy.for_launch(harness, managed_subagent);
2575 mj_core::worker_launch::resolve_subagent_mcp_role(
2576 &launch_policy,
2577 managed_subagent,
2578 subagent.is_some_and(|record| record.handback_tool),
2579 agent_mailboxes_enabled,
2580 !managed_subagent,
2581 )
2582 .is_some()
2583}
2584
2585pub(in crate::controller) fn move_children(
2586 state: &mj_core::state::State,
2587 parent_session_id: &str,
2588) -> Vec<mj_core::subagent::InPlaceSubagent> {
2589 state
2590 .subagents
2591 .values()
2592 .filter(|relation| relation.parent_session_id == parent_session_id)
2593 .filter_map(|relation| {
2594 let child = state.sessions.get(&relation.child_session_id)?;
2595 let state = if child.state.has_live_worker() {
2596 mj_core::subagent::InPlaceSubagentState::Running
2597 } else if child.state == SessionState::Parked {
2598 mj_core::subagent::InPlaceSubagentState::Parked
2599 } else {
2600 return None;
2601 };
2602 Some(mj_core::subagent::InPlaceSubagent {
2603 child_session_id: relation.child_session_id.clone(),
2604 task_name: relation.task_name.clone(),
2605 state,
2606 })
2607 })
2608 .collect()
2609}
2610
2611fn live_move_children(state: &mj_core::state::State, parent_session_id: &str) -> Vec<String> {
2612 move_children(state, parent_session_id)
2613 .into_iter()
2614 .filter(|child| child.state == mj_core::subagent::InPlaceSubagentState::Running)
2615 .map(|child| {
2616 format!(
2617 "{} (child_session_id {})",
2618 child.task_name, child.child_session_id
2619 )
2620 })
2621 .collect()
2622}
2623
2624fn roleless_move_children_error(children: &[String]) -> Option<String> {
2625 (!children.is_empty()).then(|| {
2626 format!(
2627 "cannot in-place Move to a harness without mj-agents parent tools while live sub-agents exist: {}",
2628 children.join(", ")
2629 )
2630 })
2631}
2632
2633fn should_stop_move_subagents(in_place: bool, destination_has_parent_role: bool) -> bool {
2634 !in_place || !destination_has_parent_role
2635}
2636
2637fn subagent_mutations_pending(
2638 requests: &[mj_core::subagent::SubagentToolRequest],
2639 durable_effect_pending: bool,
2640) -> bool {
2641 durable_effect_pending
2642 || requests
2643 .iter()
2644 .any(|request| request.action.mutates_child_state())
2645}
2646
2647pub(super) fn in_place_move_eligible(
2654 source: &mj_core::state::SessionRecord,
2655 selection: &MoveSelection,
2656 targets: &std::collections::BTreeMap<String, mj_core::config::TargetTemplate>,
2657 is_subagent: bool,
2658 retry: bool,
2659) -> bool {
2660 !retry
2661 && !is_subagent
2662 && source.target.is_some()
2663 && matches!(
2664 source.state,
2665 SessionState::Running | SessionState::Disconnected
2666 )
2667 && move_environment_change(
2668 source,
2669 selection,
2670 bare_targets_share_environment(
2671 targets,
2672 &source.target_template_id,
2673 selection.target_template_id.as_deref().unwrap_or_default(),
2674 ),
2675 )
2676 .is_none()
2677}
2678
2679fn bare_targets_share_environment(
2684 targets: &std::collections::BTreeMap<String, mj_core::config::TargetTemplate>,
2685 source_id: &str,
2686 destination_id: &str,
2687) -> bool {
2688 use mj_core::config::TargetTemplate::{LocalBare, SshBare};
2689 match (targets.get(source_id), targets.get(destination_id)) {
2690 (Some(LocalBare), Some(LocalBare)) => true,
2691 (
2692 Some(SshBare { ssh: source, .. }),
2693 Some(SshBare {
2694 ssh: destination, ..
2695 }),
2696 ) => {
2697 crate::targets::SshTarget::from(source) == crate::targets::SshTarget::from(destination)
2698 }
2699 _ => false,
2700 }
2701}
2702fn outcome(
2703 operation_id: &str,
2704 selection: &MoveSelection,
2705 status: &str,
2706 error: Option<String>,
2707 recovery: Option<String>,
2708) -> MoveOutcome {
2709 MoveOutcome {
2710 operation_id: operation_id.into(),
2711 session_id: selection.session_id.clone(),
2712 profile_id: selection.profile_id.clone().unwrap_or_default(),
2713 target_template_id: selection.target_template_id.clone().unwrap_or_default(),
2714 outcome: status.into(),
2715 error,
2716 recovery,
2717 }
2718}
2719
2720fn move_environment_change(
2724 source: &mj_core::state::SessionRecord,
2725 selection: &MoveSelection,
2726 same_environment: bool,
2727) -> Option<&'static str> {
2728 if Some(&source.target_template_id) != selection.target_template_id.as_ref()
2729 && !same_environment
2730 {
2731 Some("target changed")
2732 } else if Some(&source.additional_mounts) != selection.additional_mounts.as_ref() {
2733 Some("attached mounts changed")
2734 } else if source.resource_allocation != selection.resource_allocation
2735 || (selection.clear_resource_allocation
2736 && (source.container_cpus.is_some() || source.container_memory.is_some()))
2737 {
2738 Some("resource allocation changed")
2739 } else {
2740 None
2741 }
2742}