1use std::path::{Path, PathBuf};
4use std::sync::Arc;
5use std::time::{Duration, Instant};
6
7use agent_client_protocol::schema::v1::ContentBlock;
8use anyhow::{Context, Result, bail, ensure};
9use rayon::prelude::*;
10use tokio_util::sync::CancellationToken;
11
12use crate::checkpoint_transfer::restore_command;
13use crate::session_manager::new_command_id;
14use mj_checkpoint::archive::{
15 CanonicalQueuedCommandKind, CanonicalSessionSnapshot, CheckpointRepositoryBundle, SystemGit,
16 checkpoint_bundle_prerequisites, read_checkpoint_repository_bundles, verify_archive_streaming,
17};
18use mj_checkpoint::checkpoint::CheckpointRestoreSpec;
19use mj_core::config::{Config, HarnessKind, ProjectRepository, mount_history_host};
20use mj_core::state::{MaterializedSession, SessionRecord, SessionResourceAllocation, SessionState};
21use mj_transcript::projection::materialized_session_from_canonical;
22
23use crate::targets::{
24 self, AdditionalMount, CancellableProcessExecutor, CommandExecutor, CommandOutput, CommandSpec,
25 ProcessExecutor, ProvisionStage, ProvisionStageGuard,
26};
27use mj_core::relay::RelayCommand;
28
29use super::backend::{backend_locator, controller_github_token, validate_resource_allocation};
30use super::checkpoint::upload_checkpoint_spec;
31use super::provisioning::{
32 ProvisioningFailureDisposition, StagedExecutor, execute_concurrent_lanes,
33 install_attached_resources,
34};
35use super::readiness::{connect_started_worker, wait_for_native_session_in_stage};
36use super::worker_binary::{bridge_readiness_stage, start_worker, worker_probe_diagnosis};
37use super::worktree::{
38 PrimaryCheckoutRequirement, ResumeConversion, ResumePlan, apply_raw_to_workspace,
39 apply_workspace_to_raw, cleanup_managed_worktree, create_managed_worktree,
40 managed_worktree_checkout_exists, plan_raw_to_workspace,
41 preserve_retained_managed_worktree_branch, raw_checkout_divergence_notice,
42 raw_checkout_position, raw_checkout_snapshot, raw_conversion_preview, restore_managed_worktree,
43 resume_compatibility, retire_managed_worktree,
44};
45use super::{
46 Controller, SessionResumeOptions, execute_checked, now, selected_host_container_size,
47 target_profile_home,
48};
49
50#[derive(Debug, Clone, PartialEq, Eq)]
51pub struct ResumeRepositorySourceMismatch {
52 pub session_id: String,
53 pub bundle_id: String,
54 pub repository_id: String,
55 pub missing_commit: String,
56 pub archived_origin: String,
57 pub configured_origin: String,
58}
59
60pub use mj_core::state::ResumeRepositorySourceReceipt;
61
62#[derive(Debug, Clone, PartialEq, Eq)]
63pub enum ResumeRepositorySourcePreflight {
64 Ready(ResumeRepositorySourceReceipt),
65 RepositoryMoved(ResumeRepositorySourceMismatch),
66 ConvertingRawCheckout {
71 receipt: ResumeRepositorySourceReceipt,
72 preview: Box<mj_core::state::RawConversionPreview>,
73 },
74}
75
76struct ResumeRepositoryBundles {
77 checkpoint_sha256: String,
78 repositories: Vec<CheckpointRepositoryBundle>,
79}
80
81struct ResumePhaseTimer<'a> {
85 session_id: &'a str,
86 phase: &'static str,
87 started: Instant,
88}
89
90impl<'a> ResumePhaseTimer<'a> {
91 fn new(session_id: &'a str, phase: &'static str) -> Self {
92 Self {
93 session_id,
94 phase,
95 started: Instant::now(),
96 }
97 }
98}
99
100impl Drop for ResumePhaseTimer<'_> {
101 fn drop(&mut self) {
102 tracing::debug!(
103 session_id = self.session_id,
104 phase = self.phase,
105 elapsed_ms = self.started.elapsed().as_millis(),
106 "resume phase completed"
107 );
108 }
109}
110
111impl Controller {
112 pub(super) fn validate_muse_resume_destination(
114 &self,
115 source: &SessionRecord,
116 destination_harness: HarnessKind,
117 _target_id: &str,
118 ) -> Result<()> {
119 if destination_harness != HarnessKind::Muse {
120 return Ok(());
121 }
122 ensure!(
123 source.project_directory.is_some()
124 || self
125 .config
126 .bundles
127 .get(&source.bundle_id)
128 .is_none_or(|bundle| bundle.repositories.len() == 1),
129 "Muse Code ACP supports one workspace root; use a single-repository bundle"
130 );
131 Ok(())
132 }
133 pub fn preflight_resume_repository_sources(
137 &self,
138 session_id: &str,
139 target_id: &str,
140 executor: &(impl CommandExecutor + Sync),
141 ) -> Result<ResumeRepositorySourcePreflight> {
142 self.preflight_repository_sources(session_id, target_id, true, executor)
143 }
144
145 fn preflight_repository_sources(
150 &self,
151 session_id: &str,
152 target_id: &str,
153 describe_conversion: bool,
154 executor: &(impl CommandExecutor + Sync),
155 ) -> Result<ResumeRepositorySourcePreflight> {
156 let session = self
157 .state
158 .sessions
159 .get(session_id)
160 .with_context(|| format!("unknown session {session_id}"))?;
161 let checkpoint = session
162 .checkpoint
163 .as_ref()
164 .context("session has no checkpoint")?;
165 let plan = resume_compatibility(session, &self.config, target_id)
166 .map_err(|reason| anyhow::anyhow!(reason))?;
167 if session.project_directory.is_some() {
168 debug_assert!(matches!(
169 plan,
170 ResumePlan::InPlace | ResumePlan::RawToWorkspace
171 ));
172 let receipt = ResumeRepositorySourceReceipt {
177 session_id: session_id.to_owned(),
178 bundle_id: session.bundle_id.clone(),
179 checkpoint_sha256: checkpoint.sha256.clone(),
180 repositories: Vec::new(),
181 };
182 if describe_conversion && plan == ResumePlan::RawToWorkspace {
187 let preview = raw_conversion_preview_for(session, &self.config, executor)?;
188 return Ok(ResumeRepositorySourcePreflight::ConvertingRawCheckout {
189 receipt,
190 preview: Box::new(preview),
191 });
192 }
193 return Ok(ResumeRepositorySourcePreflight::Ready(receipt));
194 }
195 let repositories = read_checkpoint_repository_bundles(&checkpoint.archive_path)?;
196 self.preflight_verified_repository_sources(
197 session_id,
198 ResumeRepositoryBundles {
199 checkpoint_sha256: checkpoint.sha256.clone(),
200 repositories,
201 },
202 None,
203 plan != ResumePlan::WorkspaceToRaw,
204 executor,
205 )
206 }
207
208 fn preflight_verified_repository_sources(
209 &self,
210 session_id: &str,
211 verified: ResumeRepositoryBundles,
212 skip_repository_id: Option<&str>,
213 use_archived_network_sources: bool,
214 executor: &(impl CommandExecutor + Sync),
215 ) -> Result<ResumeRepositorySourcePreflight> {
216 let session = self
217 .state
218 .sessions
219 .get(session_id)
220 .with_context(|| format!("unknown session {session_id}"))?;
221 ensure!(
222 verified.repositories.iter().all(|repository| {
223 !repository.metadata.origin.starts_with("mj-local:")
224 && !repository.metadata.origin.starts_with("ext::")
225 }),
226 "resuming legacy host-bridge sessions is not supported; start a new network-backed session"
227 );
228 if use_archived_network_sources
231 && verified
232 .repositories
233 .iter()
234 .all(|repository| repository.metadata.remote_workspace)
235 {
236 return Ok(ResumeRepositorySourcePreflight::Ready(
237 ResumeRepositorySourceReceipt {
238 session_id: session_id.to_owned(),
239 bundle_id: session.bundle_id.clone(),
240 checkpoint_sha256: verified.checkpoint_sha256,
241 repositories: Vec::new(),
242 },
243 ));
244 }
245 if verified.repositories.is_empty() {
246 return Ok(ResumeRepositorySourcePreflight::Ready(
247 ResumeRepositorySourceReceipt {
248 session_id: session_id.to_owned(),
249 bundle_id: session.bundle_id.clone(),
250 checkpoint_sha256: verified.checkpoint_sha256,
251 repositories: Vec::new(),
252 },
253 ));
254 }
255 let bundle = self
256 .config
257 .bundles
258 .get(&session.bundle_id)
259 .with_context(|| format!("session bundle {:?} is missing", session.bundle_id))?;
260 let configured = verified
261 .repositories
262 .iter()
263 .map(|archived| {
264 bundle
265 .repositories
266 .iter()
267 .find(|repository| repository.id == archived.metadata.id)
268 .cloned()
269 .with_context(|| {
270 format!(
271 "session bundle {:?} no longer contains repository {:?}",
272 session.bundle_id, archived.metadata.id
273 )
274 })
275 })
276 .collect::<Result<Vec<_>>>()?;
277 let github_token = configured
278 .iter()
279 .any(|repository| repository.github.is_some())
280 .then(controller_github_token)
281 .flatten();
282 let outcomes = verified
283 .repositories
284 .par_iter()
285 .zip(configured.par_iter())
286 .map(|(archived, configured)| {
287 if skip_repository_id == Some(configured.id.as_str()) {
288 return Ok(None);
289 }
290 checkpoint_source_missing_commit(
291 configured,
292 archived,
293 executor,
294 github_token.as_deref(),
295 )
296 .map(|missing_commit| {
297 missing_commit.map(|missing_commit| ResumeRepositorySourceMismatch {
298 session_id: session_id.to_owned(),
299 bundle_id: session.bundle_id.clone(),
300 repository_id: configured.id.clone(),
301 missing_commit,
302 archived_origin: archived.metadata.origin.clone(),
303 configured_origin: configured.source_label(),
304 })
305 })
306 })
307 .collect::<Vec<Result<Option<ResumeRepositorySourceMismatch>>>>();
308 for outcome in outcomes {
309 if let Some(mismatch) = outcome? {
310 return Ok(ResumeRepositorySourcePreflight::RepositoryMoved(mismatch));
311 }
312 }
313 Ok(ResumeRepositorySourcePreflight::Ready(
314 ResumeRepositorySourceReceipt {
315 session_id: session_id.to_owned(),
316 bundle_id: session.bundle_id.clone(),
317 checkpoint_sha256: verified.checkpoint_sha256,
318 repositories: configured,
319 },
320 ))
321 }
322
323 fn repository_source_receipt_is_current(
324 &self,
325 session_id: &str,
326 receipt: &ResumeRepositorySourceReceipt,
327 ) -> bool {
328 let Some(session) = self.state.sessions.get(session_id) else {
329 return false;
330 };
331 if receipt.session_id != session_id
332 || receipt.bundle_id != session.bundle_id
333 || session
334 .checkpoint
335 .as_ref()
336 .map(|checkpoint| &checkpoint.sha256)
337 != Some(&receipt.checkpoint_sha256)
338 {
339 return false;
340 }
341 if receipt.repositories.is_empty() {
342 return true;
343 }
344 let Some(bundle) = self.config.bundles.get(&session.bundle_id) else {
345 return false;
346 };
347 receipt.repositories.iter().all(|expected| {
348 bundle
349 .repositories
350 .iter()
351 .any(|configured| configured == expected)
352 })
353 }
354
355 pub fn replace_resume_repository_origin(
359 &mut self,
360 session_id: &str,
361 repository_id: &str,
362 replacement: &str,
363 executor: &(impl CommandExecutor + Sync),
364 ) -> Result<ResumeRepositorySourcePreflight> {
365 let session = self
366 .state
367 .sessions
368 .get(session_id)
369 .with_context(|| format!("unknown session {session_id}"))?;
370 let bundle_id = session.bundle_id.clone();
371 let checkpoint = session
372 .checkpoint
373 .as_ref()
374 .context("session has no checkpoint")?;
375 let replacement = replacement_repository_source(repository_id, replacement)?;
376 let repositories = read_checkpoint_repository_bundles(&checkpoint.archive_path)?;
377 let verified = ResumeRepositoryBundles {
378 checkpoint_sha256: checkpoint.sha256.clone(),
379 repositories,
380 };
381 let archived = verified
382 .repositories
383 .iter()
384 .find(|repository| repository.metadata.id == repository_id)
385 .with_context(|| format!("checkpoint does not contain repository {repository_id:?}"))?;
386 if let Some(missing_commit) = checkpoint_source_missing_commit(
387 &replacement,
388 archived,
389 executor,
390 controller_github_token().as_deref(),
391 )? {
392 return Ok(ResumeRepositorySourcePreflight::RepositoryMoved(
393 ResumeRepositorySourceMismatch {
394 session_id: session_id.to_owned(),
395 bundle_id,
396 repository_id: repository_id.to_owned(),
397 missing_commit,
398 archived_origin: archived.metadata.origin.clone(),
399 configured_origin: replacement.source_label(),
400 },
401 ));
402 }
403 let (config, ()) = Config::update(|config| {
404 let bundle = config
405 .bundles
406 .get_mut(&bundle_id)
407 .with_context(|| format!("session bundle {bundle_id:?} is missing"))?;
408 let repository = bundle
409 .repositories
410 .iter_mut()
411 .find(|repository| repository.id == repository_id)
412 .with_context(|| {
413 format!(
414 "session bundle {:?} no longer contains repository {repository_id:?}",
415 bundle_id
416 )
417 })?;
418 repository.github = replacement.github.clone();
419 repository.local = replacement.local.clone();
420 Ok(())
421 })?;
422 self.config = config;
423 self.preflight_verified_repository_sources(
424 session_id,
425 verified,
426 Some(repository_id),
427 false,
428 executor,
429 )
430 }
431}
432
433pub fn raw_conversion_preview_for(
440 session: &SessionRecord,
441 config: &Config,
442 executor: &(impl CommandExecutor + Sync),
443) -> Result<mj_core::state::RawConversionPreview> {
444 let conversion = plan_raw_to_workspace(session, config, executor)?;
445 raw_conversion_preview(session, &conversion, executor)
446}
447
448fn replacement_repository_source(id: &str, replacement: &str) -> Result<ProjectRepository> {
449 let replacement = replacement.trim();
450 ensure!(!replacement.is_empty(), "enter the repository's new origin");
451 let expanded = mj_core::path_input::expand_local(Path::new(replacement))?;
452 let path = expanded.as_path();
453 let (github, local) = if path.is_absolute() {
454 ensure!(
455 path.is_dir(),
456 "local repository {replacement:?} is not a directory"
457 );
458 (None, Some(mj_core::local_git::canonical_repository(path)?))
459 } else {
460 let github = crate::setup::github_repository_from_origin(replacement)
461 .context("origin must be a GitHub repository or an absolute local repository path")?;
462 (
463 Some(format!("{}/{}", github.owner, github.repository)),
464 None,
465 )
466 };
467 Ok(ProjectRepository {
468 id: id.to_owned(),
469 github,
470 local,
471 destination: PathBuf::from(id),
472 git_ref: None,
473 })
474}
475
476fn checkpoint_source_missing_commit(
477 configured: &ProjectRepository,
478 archived: &CheckpointRepositoryBundle,
479 executor: &impl CommandExecutor,
480 github_token: Option<&str>,
481) -> Result<Option<String>> {
482 let staging = tempfile::tempdir().context("create repository source preflight")?;
483 let repository = staging.path().join("repository.git");
484 checked_preflight_git(
485 executor,
486 CommandSpec::new(
487 "git",
488 [
489 "init".to_owned(),
490 "--bare".to_owned(),
491 "--quiet".to_owned(),
492 repository.to_string_lossy().into_owned(),
493 ],
494 )
495 .purpose("initialize repository source preflight"),
496 )?;
497 let missing = checkpoint_bundle_prerequisites(archived)?;
498 if missing.is_empty() {
499 let bundle = staging.path().join("checkpoint.bundle");
500 std::fs::write(&bundle, &archived.committed_bundle)
501 .context("write self-contained checkpoint bundle for source preflight")?;
502 checked_preflight_git(
503 executor,
504 checkpoint_bundle_import_command(&repository, &bundle),
505 )?;
506 return Ok(None);
507 }
508 for commit in missing {
512 let output = fetch_source_commit(executor, &repository, configured, &commit, github_token)?;
513 if output.status != 0 {
514 let stderr = String::from_utf8_lossy(&output.stderr);
515 if source_does_not_have_commit(&stderr) {
516 return Ok(Some(commit));
517 }
518 bail!(
519 "could not check configured source {:?}: {}",
520 configured.source_label(),
521 stderr.trim()
522 );
523 }
524 }
525 Ok(None)
529}
530
531fn checkpoint_bundle_import_command(repository: &Path, bundle: &Path) -> CommandSpec {
532 let mut command = CommandSpec::new(
533 "git",
534 [
535 "-C".to_owned(),
536 repository.to_string_lossy().into_owned(),
537 "fetch".to_owned(),
538 "--no-tags".to_owned(),
539 bundle.to_string_lossy().into_owned(),
540 "HEAD".to_owned(),
541 ],
542 )
543 .purpose("validate self-contained checkpoint bundle");
544 command
545 .env
546 .insert("GIT_NO_LAZY_FETCH".to_owned(), "1".to_owned());
547 command
548 .env
549 .insert("GIT_TERMINAL_PROMPT".to_owned(), "0".to_owned());
550 command
551}
552
553fn fetch_source_commit(
554 executor: &impl CommandExecutor,
555 repository: &Path,
556 configured: &ProjectRepository,
557 commit: &str,
558 github_token: Option<&str>,
559) -> Result<CommandOutput> {
560 let mut arguments = Vec::new();
561 let mut token_auth = false;
562 let mut ssh_transport = false;
563 let source = if let Some(local) = &configured.local {
564 local.to_string_lossy().into_owned()
565 } else {
566 let source = configured
567 .github
568 .as_deref()
569 .context("repository source is missing")?;
570 let github = crate::setup::github_repository_from_origin(source)
571 .context("configured repository is not a GitHub source")?;
572 if github_token.is_some() {
573 token_auth = true;
574 arguments.extend([
575 "-c".to_owned(),
576 "credential.helper=".to_owned(),
577 "-c".to_owned(),
578 "credential.helper=!f() { if [ \"$1\" = get ]; then echo username=x-access-token; echo \"password=$GH_TOKEN\"; fi; }; f".to_owned(),
579 ]);
580 format!(
581 "https://github.com/{}/{}.git",
582 github.owner, github.repository
583 )
584 } else {
585 ssh_transport = true;
586 format!("git@github.com:{}/{}.git", github.owner, github.repository)
587 }
588 };
589 arguments.extend([
590 "-C".to_owned(),
591 repository.to_string_lossy().into_owned(),
592 "fetch".to_owned(),
593 "--no-tags".to_owned(),
594 "--depth=1".to_owned(),
595 "--filter=blob:none".to_owned(),
596 source,
597 commit.to_owned(),
598 ]);
599 let mut command = CommandSpec::new("git", arguments).purpose("check checkpoint base commit");
600 command
601 .env
602 .insert("GIT_NO_LAZY_FETCH".to_owned(), "1".to_owned());
603 command
604 .env
605 .insert("GIT_TERMINAL_PROMPT".to_owned(), "0".to_owned());
606 if token_auth {
607 let token = github_token.expect("token authentication requires a GitHub token");
608 command.env.insert("GH_TOKEN".to_owned(), token.to_owned());
609 }
610 if ssh_transport {
611 command.env.insert(
612 "GIT_SSH_COMMAND".to_owned(),
613 "ssh -o BatchMode=yes -o StrictHostKeyChecking=accept-new -o ConnectTimeout=15"
614 .to_owned(),
615 );
616 }
617 executor.execute(&command)
618}
619
620fn source_does_not_have_commit(stderr: &str) -> bool {
621 let stderr = stderr.to_ascii_lowercase();
622 [
623 "not our ref",
624 "couldn't find remote ref",
625 "not a valid object name",
626 "no such ref was fetched",
627 ]
628 .iter()
629 .any(|needle| stderr.contains(needle))
630}
631
632fn checked_preflight_git(
633 executor: &impl CommandExecutor,
634 command: CommandSpec,
635) -> Result<CommandOutput> {
636 let output = executor.execute(&command)?;
637 ensure!(
638 output.status == 0,
639 "{}: {}",
640 command.purpose,
641 String::from_utf8_lossy(&output.stderr).trim()
642 );
643 Ok(output)
644}
645
646impl Controller {
647 pub async fn resume_session_with_options(
652 &mut self,
653 session_id: &str,
654 profile_id: &str,
655 target_id: &str,
656 additional_mounts: Option<Vec<AdditionalMount>>,
657 resource_allocation: Option<SessionResourceAllocation>,
658 ) -> Result<MaterializedSession> {
659 self.resume_session_with_options_and_queue_disposition(
660 session_id,
661 profile_id,
662 target_id,
663 additional_mounts,
664 resource_allocation,
665 false,
666 )
667 .await
668 }
669
670 pub async fn resume_session_with_options_and_queue_disposition(
671 &mut self,
672 session_id: &str,
673 profile_id: &str,
674 target_id: &str,
675 additional_mounts: Option<Vec<AdditionalMount>>,
676 resource_allocation: Option<SessionResourceAllocation>,
677 discard_queue: bool,
678 ) -> Result<MaterializedSession> {
679 self.resume_session_controlled(
680 session_id,
681 profile_id,
682 target_id,
683 SessionResumeOptions {
684 additional_mounts,
685 resource_allocation,
686 discard_queue,
687 },
688 &ProcessExecutor,
689 )
690 .await
691 }
692
693 pub async fn resume_session_controlled(
694 &mut self,
695 session_id: &str,
696 profile_id: &str,
697 target_id: &str,
698 options: SessionResumeOptions,
699 executor: &(impl CommandExecutor + Sync),
700 ) -> Result<MaterializedSession> {
701 self.resume_session_controlled_with_repository_preflight(
702 session_id, profile_id, target_id, options, None, executor,
703 )
704 .await
705 }
706
707 pub async fn resume_session_controlled_with_repository_preflight(
708 &mut self,
709 session_id: &str,
710 profile_id: &str,
711 target_id: &str,
712 options: SessionResumeOptions,
713 repository_preflight: Option<ResumeRepositorySourceReceipt>,
714 executor: &(impl CommandExecutor + Sync),
715 ) -> Result<MaterializedSession> {
716 let SessionResumeOptions {
717 additional_mounts,
718 resource_allocation,
719 discard_queue,
720 } = options;
721 let previous = self
722 .state
723 .sessions
724 .get(session_id)
725 .with_context(|| format!("unknown session {session_id}"))?
726 .clone();
727 if !matches!(
728 previous.state,
729 SessionState::Stopped | SessionState::Lost | SessionState::Error
730 ) {
731 bail!("session {session_id} is not stopped, lost, or retryable");
732 }
733 let checkpoint = previous
734 .checkpoint
735 .as_ref()
736 .context("session has no checkpoint")?;
737 let moving_to_raw = previous.project_directory.is_none()
740 && self
741 .config
742 .targets
743 .get(target_id)
744 .is_some_and(mj_core::config::is_bare_project_target);
745 if moving_to_raw
746 || !repository_preflight.as_ref().is_some_and(|receipt| {
747 self.repository_source_receipt_is_current(session_id, receipt)
748 })
749 {
750 let _phase = ResumePhaseTimer::new(session_id, "preflight repository sources");
751 if let ResumeRepositorySourcePreflight::RepositoryMoved(mismatch) =
752 self.preflight_repository_sources(session_id, target_id, false, executor)?
753 {
754 bail!(
755 "checkpoint base commit {} is missing from configured source {:?} for repository {:?}; the repository may have moved (archived origin: {:?})",
756 mismatch.missing_commit,
757 mismatch.configured_origin,
758 mismatch.repository_id,
759 mismatch.archived_origin,
760 );
761 }
762 }
763 let archive_path = {
768 let _phase = ResumePhaseTimer::new(session_id, "verify checkpoint archive");
769 checkpoint.archive_path.canonicalize().with_context(|| {
770 format!(
771 "resolve checkpoint archive {}",
772 checkpoint.archive_path.display()
773 )
774 })?
775 };
776 ensure!(
777 archive_path.is_absolute() && archive_path.is_file(),
778 "checkpoint archive path is not an absolute regular file: {}",
779 archive_path.display()
780 );
781 let mj_checkpoint::archive::VerifiedArchiveMetadata {
785 manifest: archive_manifest,
786 canonical_session,
787 archive_sha256,
788 } = {
789 let _phase = ResumePhaseTimer::new(session_id, "verify checkpoint archive contents");
790 verify_archive_streaming(&archive_path)?
791 };
792 if archive_sha256 != checkpoint.sha256 || archive_manifest.session.id != session_id {
793 bail!("persisted checkpoint verification failed");
794 }
795 ensure!(
796 archive_manifest.repositories.iter().all(|repository| {
797 !repository.metadata.origin.starts_with("mj-local:")
798 && !repository.metadata.origin.starts_with("ext::")
799 }),
800 "resuming legacy host-bridge sessions is not supported; start a new network-backed session"
801 );
802 let canonical_session = Arc::new(canonical_session);
803 let profile = self
804 .config
805 .profiles
806 .get(profile_id)
807 .with_context(|| format!("unknown profile {profile_id:?}"))?
808 .clone();
809 ensure!(profile.enabled, "profile {profile_id:?} is disabled");
810 let target_template = self
811 .config
812 .targets
813 .get(target_id)
814 .with_context(|| format!("unknown target template {target_id:?}"))?
815 .clone();
816 self.validate_muse_resume_destination(&previous, profile.kind, target_id)?;
819 ensure!(
820 profile.kind != HarnessKind::Muse || previous.additional_mounts.is_empty(),
821 "Muse Code ACP supports one workspace root; attached directories are unsupported"
822 );
823 let plan = resume_compatibility(&previous, &self.config, target_id)
824 .map_err(|reason| anyhow::anyhow!("{reason}"))?;
825 if !mj_core::config::is_bare_project_target(&target_template)
829 && plan != ResumePlan::RawToWorkspace
830 {
831 super::network_git::bundle_from_manifest(&archive_manifest)?;
832 }
833 if plan == ResumePlan::InPlace
834 && previous.managed_worktree.is_none()
835 && let Some(project_directory) = &previous.project_directory
836 {
837 self.validate_project_directory(target_id, project_directory, executor)
838 .context("raw project is unavailable for resume")?;
839 }
840 let conversion = match plan {
841 ResumePlan::InPlace => None,
842 ResumePlan::RawToWorkspace => Some(ResumeConversion::RawToWorkspace(
843 plan_raw_to_workspace(&previous, &self.config, executor)
844 .context("prepare the raw checkout for its new target")?,
845 )),
846 ResumePlan::WorkspaceToRaw => Some(ResumeConversion::WorkspaceToRaw(
847 self.plan_workspace_to_raw(&previous, target_id, executor)
848 .context("prepare a checkout for this session")?,
849 )),
850 };
851 let resource_allocation =
852 resource_allocation.or_else(|| previous.resource_allocation.clone());
853 let additional_mounts =
854 additional_mounts.unwrap_or_else(|| previous.additional_mounts.clone());
855 validate_resource_allocation(&target_template, resource_allocation.as_ref())?;
856 let selected_container_size =
857 selected_host_container_size(&target_template, resource_allocation.as_ref());
858 if !additional_mounts.is_empty() && mount_history_host(&target_template).is_none() {
859 bail!("attached resources are unsupported for this target");
860 }
861 targets::validate_additional_mounts(&additional_mounts)?;
862 let history_host = mount_history_host(&target_template);
863 let history_mounts = additional_mounts.clone();
864 if previous.state == SessionState::Error
865 && let Some(locator) = &previous.target
866 {
867 let backend = backend_locator(locator, &previous, &self.config)?;
868 targets::close_plan(&backend, session_id)?
869 .execute(executor)
870 .context("clean up target from failed resume")?;
871 }
872 let mut resume_notices = Vec::new();
873 if let Some(conversion) = conversion
876 .as_ref()
877 .and_then(ResumeConversion::workspace_to_raw)
878 {
879 resume_notices.push(format!(
880 "This session moved out of its {} target and into {}. Its branch {} is now {}.",
881 previous.target_template_id,
882 conversion.worktree.worktree_root.display(),
883 archive_manifest
884 .repositories
885 .first()
886 .and_then(|repository| repository.metadata.branch.as_deref())
887 .unwrap_or("a detached head"),
888 conversion.worktree.branch,
889 ));
890 }
891 let managed_checkout_present = previous
892 .managed_worktree
893 .as_ref()
894 .map(|worktree| managed_worktree_checkout_exists(executor, worktree))
895 .transpose()?
896 .unwrap_or(true);
897 if managed_checkout_present && let Some(project_directory) = &previous.project_directory {
900 match raw_checkout_position(&previous, &self.config, project_directory, executor) {
901 Ok(live) => resume_notices.extend(raw_checkout_divergence_notice(
902 project_directory,
903 archive_manifest
904 .repositories
905 .first()
906 .map(|repository| &repository.metadata),
907 &live,
908 )),
909 Err(error) => tracing::warn!(
912 session_id,
913 error = format!("{error:#}"),
914 "could not read the raw checkout position for a resume notice"
915 ),
916 }
917 }
918 super::worker_binary::preflight_worker_binary(&target_template)?;
923 let same_harness = profile.kind == archive_manifest.session.harness_kind;
924 let native_continuity =
925 native_continuity_preserved(profile.kind, archive_manifest.session.harness_kind);
926 let context_bytes = profile
927 .context_window_bytes
928 .unwrap_or(crate::compaction::DEFAULT_CONTEXT_BYTES);
929 let utility_config = (!native_continuity).then(|| self.config.clone());
933 let discard_queued_prompts = discard_queue || !same_harness;
934 let stored_frontier = crate::database::materialized_event_frontier(session_id)
938 .unwrap_or_else(|error| {
939 tracing::warn!(
940 session_id,
941 error = format!("{error:#}"),
942 "could not read the stored projection frontier; rebuilding it from the archive"
943 );
944 None
945 });
946 let rebuild_projection = projection_rebuild_required(
947 stored_frontier
948 .as_ref()
949 .map(|(ordinal, digest)| (*ordinal, digest.as_str())),
950 canonical_session.event_frontier,
951 &canonical_session.event_frontier_digest,
952 );
953 let projection_build = rebuild_projection.then(|| {
963 let canonical = Arc::clone(&canonical_session);
964 let session_id = session_id.to_owned();
965 tokio::task::spawn_blocking(move || {
966 materialized_session_from_canonical(session_id, &canonical)
967 })
968 });
969 let github_token = controller_github_token();
970
971 if let Some(conversion) = conversion
974 .as_ref()
975 .and_then(ResumeConversion::raw_to_workspace)
976 && let Some(bundle) = &conversion.new_bundle
977 {
978 let (config, ()) = Config::update(|config| {
979 if let Some(existing) = config.bundles.get(&conversion.bundle_id) {
980 ensure!(
981 existing == bundle,
982 "bundle {:?} was configured concurrently with a different definition; retry the resume",
983 conversion.bundle_id
984 );
985 } else {
986 config
987 .bundles
988 .insert(conversion.bundle_id.clone(), bundle.clone());
989 }
990 Ok(())
991 })
992 .context("save the bundle for a converted raw session")?;
993 self.config = config;
994 }
995
996 let record = self.state.sessions.get_mut(session_id).unwrap();
997 record.harness_kind = profile.kind;
998 record.last_profile = profile_id.to_string();
999 record.target_template_id = target_id.to_string();
1000 record.resource_allocation = resource_allocation;
1001 record.additional_mounts = additional_mounts;
1002 record.target = None;
1003 record.native_session_id =
1004 native_continuity.then(|| archive_manifest.session.native_session_id.clone());
1005 record.state = SessionState::Provisioning;
1006 record.updated_at = now();
1007 record.last_error = None;
1008 match &conversion {
1009 Some(ResumeConversion::RawToWorkspace(conversion)) => {
1010 apply_raw_to_workspace(record, conversion);
1011 }
1012 Some(ResumeConversion::WorkspaceToRaw(conversion)) => {
1013 apply_workspace_to_raw(record, conversion);
1014 }
1015 None => {}
1016 }
1017 let resumed_project_directory = record.project_directory.clone();
1018 if let Some(host) = history_host {
1019 self.state.remember_mount_sources(host, &history_mounts);
1020 crate::database::remember_mount_sources(host, &history_mounts)?;
1021 }
1022 if let Some(conversion) = conversion
1025 .as_ref()
1026 .and_then(ResumeConversion::raw_to_workspace)
1027 {
1028 crate::database::rebind_session_bundle(session_id, &conversion.bundle_id)?;
1029 }
1030 if let Some((host, size)) = selected_container_size.as_ref() {
1033 crate::database::save_session_with_container_size(
1034 &self.state.sessions[session_id],
1035 host,
1036 *size,
1037 )?;
1038 } else {
1039 crate::database::save_session(&self.state.sessions[session_id])?;
1040 }
1041 if let Some((host, size)) = selected_container_size.as_ref() {
1042 self.state.remember_container_size(host, *size);
1043 }
1044
1045 let mut recreated_managed_worktree = false;
1046 let mut conversion_checkpoint_written: Option<mj_core::state::CheckpointMetadata> = None;
1049 let result = async {
1050 if let Some(worktree) = previous.managed_worktree.as_ref() {
1051 recreated_managed_worktree = restore_managed_worktree(executor, worktree)?;
1052 if recreated_managed_worktree && plan == ResumePlan::RawToWorkspace {
1053 mj_checkpoint::checkpoint::restore_single_repository_onto_branch(
1054 &archive_path,
1055 &worktree.worktree_root,
1056 &worktree.branch,
1057 &SystemGit,
1058 )
1059 .context("restore the retired checkout before moving it into a target")?;
1060 }
1061 }
1062 if let Some(conversion) = conversion
1065 .as_ref()
1066 .and_then(ResumeConversion::workspace_to_raw)
1067 {
1068 if conversion.reuse_existing_branch {
1069 let recovery_ref =
1070 preserve_retained_managed_worktree_branch(executor, &conversion.worktree)?;
1071 restore_managed_worktree(executor, &conversion.worktree)?;
1072 resume_notices.push(format!(
1073 "Before restoring this session's retained branch, Mjolnir preserved its tip at {recovery_ref}."
1074 ));
1075 } else {
1076 create_managed_worktree(
1077 executor,
1078 &conversion.worktree,
1079 None,
1080 PrimaryCheckoutRequirement::Any,
1081 )?;
1082 }
1083 mj_checkpoint::checkpoint::restore_single_repository_onto_branch(
1084 &archive_path,
1085 &conversion.worktree.worktree_root,
1086 &conversion.worktree.branch,
1087 &SystemGit,
1088 )
1089 .context("restore this session's checkout")?;
1090 }
1091 if let Some(conversion) = conversion
1096 .as_ref()
1097 .and_then(ResumeConversion::raw_to_workspace)
1098 {
1099 let destination = PathBuf::from(
1100 previous
1101 .project_directory
1102 .as_deref()
1103 .context("a raw session has no project directory")?
1104 .file_name()
1105 .context("a raw project directory cannot be the filesystem root")?,
1106 );
1107 let snapshot = raw_checkout_snapshot(
1108 &conversion.checkout,
1109 &conversion.source,
1110 &destination,
1111 &SystemGit,
1112 )
1113 .context("snapshot the host checkout for its new target")?;
1114 resume_notices.push(conversion_notice(
1115 target_id,
1116 previous
1117 .project_directory
1118 .as_deref()
1119 .unwrap_or(&conversion.checkout),
1120 snapshot.metadata.branch.as_deref(),
1121 conversion.retire.as_ref(),
1122 ));
1123 let archives = mj_core::config::sessions_dir();
1124 std::fs::create_dir_all(&archives).with_context(|| {
1125 format!("create the checkpoint directory {}", archives.display())
1126 })?;
1127 let output = archives.join(format!(
1131 "{session_id}-converted-{}-{}.hel.zip",
1132 previous
1133 .checkpoint
1134 .as_ref()
1135 .map_or(0, |checkpoint| checkpoint.event_frontier),
1136 new_command_id("archive")?
1137 ));
1138 let written = conversion_checkpoint(&archive_path, snapshot, &output)?;
1139 conversion_checkpoint_written = Some(written.clone());
1140 let record = self.state.sessions.get_mut(session_id).unwrap();
1141 record.checkpoint = Some(written);
1142 record.updated_at = now();
1143 if let Some((host, size)) = selected_container_size.as_ref() {
1146 crate::database::save_session_with_container_size(
1147 &self.state.sessions[session_id],
1148 host,
1149 *size,
1150 )?;
1151 } else {
1152 crate::database::save_session(&self.state.sessions[session_id])?;
1153 }
1154 }
1155 let utility_handoff = {
1156 let _provisioning = ResumePhaseTimer::new(session_id, "provision destination");
1157 if let Some(config) = utility_config.as_ref() {
1158 Some(
1159 provision_with_cross_harness_handoff(
1160 self,
1161 session_id,
1162 executor,
1163 github_token.as_deref(),
1164 config,
1165 &canonical_session,
1166 context_bytes,
1167 )
1168 .context("prepare the cross-harness destination")?,
1169 )
1170 } else {
1171 self.provision_session_with_failure_disposition(
1172 session_id,
1173 executor,
1174 github_token.as_deref(),
1175 ProvisioningFailureDisposition::Preserve,
1176 )
1177 .await?;
1178 None
1179 }
1180 };
1181 let (backend, worker_root) = self.worker_placement(session_id)?;
1182 let harness_home = target_profile_home(&backend, session_id, &profile);
1183 let workspace_root = if let Some(project_directory) = &resumed_project_directory {
1184 project_directory
1185 .parent()
1186 .context("bare project directory has no parent")?
1187 .to_string_lossy()
1188 .into_owned()
1189 } else {
1190 match &backend {
1191 targets::TargetLocator::LocalPodman { .. }
1192 | targets::TargetLocator::LocalDocker { .. }
1193 | targets::TargetLocator::AppleContainer { .. }
1194 | targets::TargetLocator::SshPodman { .. }
1195 | targets::TargetLocator::SshDocker { .. } => "/workspace".to_string(),
1196 targets::TargetLocator::AwsEc2 { workspace, .. }
1197 | targets::TargetLocator::SshBare { workspace, .. } => workspace.clone(),
1198 targets::TargetLocator::LocalBare { worker_root } => worker_root.clone(),
1199 }
1200 };
1201 let target_path = |path: &str| match &backend {
1202 targets::TargetLocator::AwsEc2 { .. }
1203 | targets::TargetLocator::SshBare { .. }
1204 if !path.starts_with('/') =>
1205 {
1206 PathBuf::from(format!("~/{path}"))
1207 }
1208 _ => PathBuf::from(path),
1209 };
1210 let remote_archive = format!("{worker_root}/restore.hel.zip");
1211 let remote_spec = format!("{worker_root}/restore-spec.json");
1212 let restored_archive = conversion_checkpoint_written
1215 .as_ref()
1216 .map_or(archive_path.as_path(), |checkpoint| {
1217 checkpoint.archive_path.as_path()
1218 });
1219 let restore = CheckpointRestoreSpec {
1220 archive_path: restore_archive_path(
1221 &backend,
1222 restored_archive,
1223 &target_path(&remote_archive),
1224 ),
1225 workspace_root: target_path(&workspace_root),
1226 relay_root: target_path(&worker_root),
1227 harness_home: target_path(&harness_home),
1228 restore_repositories: (resumed_project_directory.is_none()
1234 && conversion.is_none())
1235 || plan == ResumePlan::RawToWorkspace
1236 || (recreated_managed_worktree && plan == ResumePlan::InPlace),
1237 restore_native: native_continuity,
1238 primary_repository_root: conversion
1245 .is_some()
1246 .then(|| resumed_project_directory.clone())
1247 .flatten()
1248 .map(|directory| target_path(&directory.to_string_lossy())),
1249 discard_queued_prompts,
1250 };
1251 {
1258 let syncing = &StagedExecutor::new(executor, ProvisionStage::Syncing);
1259 if let Some(command) = targets::clear_relay_state_plan(&backend, session_id)? {
1260 execute_checked(syncing, command)?;
1261 }
1262 execute_checked(
1264 syncing,
1265 targets::command_on_locator(
1266 &backend,
1267 session_id,
1268 vec!["mkdir".into(), "-p".into(), worker_root.clone()],
1269 "create the session worker root",
1270 )?,
1271 )?;
1272 }
1273 let staging = tempfile::tempdir().context("create restore staging")?;
1274 let local_spec = staging.path().join("restore-spec.json");
1275 std::fs::write(&local_spec, serde_json::to_vec_pretty(&restore)?)?;
1276 let controller = &*self;
1280 let backend_ref = &backend;
1281 let worker_root_ref = worker_root.as_str();
1282 let local_spec_ref = local_spec.as_path();
1283 execute_concurrent_lanes(
1284 || {
1285 let syncing = &StagedExecutor::new(executor, ProvisionStage::Syncing);
1286 controller.prepare_worker_files(
1287 session_id,
1288 backend_ref,
1289 worker_root_ref,
1290 syncing,
1291 )?;
1292 super::provisioning::install_inherited_git_settings(
1293 syncing,
1294 backend_ref,
1295 session_id,
1296 )?;
1297 Ok(())
1298 },
1299 || {
1300 let restoring = &StagedExecutor::new(executor, ProvisionStage::Restoring);
1301 if should_upload_restore_archive(&backend) {
1302 upload_checkpoint_spec(
1303 restoring,
1304 backend_ref,
1305 session_id,
1306 restored_archive,
1307 &remote_archive,
1308 )?;
1309 }
1310 upload_checkpoint_spec(
1311 restoring,
1312 backend_ref,
1313 session_id,
1314 local_spec_ref,
1315 &remote_spec,
1316 )
1317 },
1318 )?;
1319 {
1320 let restoring = &StagedExecutor::new(executor, ProvisionStage::Restoring);
1321 execute_checked(
1322 restoring,
1323 restore_command(&backend, session_id, &remote_spec)?,
1324 )?;
1325 }
1326 {
1327 let syncing = &StagedExecutor::new(executor, ProvisionStage::Syncing);
1328 install_attached_resources(
1329 &self.state,
1330 session_id,
1331 &backend,
1332 &worker_root,
1333 syncing,
1334 )?;
1335
1336 }
1337 match projection_build {
1338 Some(build) => {
1339 let mut restored_projection = build
1340 .await
1341 .context("rebuild the restored projection")?
1342 .context("rebuild the restored projection")?;
1343 if discard_queued_prompts {
1344 restored_projection.queued_prompts.clear();
1345 }
1346 crate::database::save_materialized_session(&restored_projection)?;
1347 }
1348 None if discard_queued_prompts => {
1351 crate::database::replace_materialized_queued_prompts(session_id, &[])?;
1352 }
1353 None => {}
1354 }
1355 let readiness_stage = bridge_readiness_stage(&profile);
1356 let spec = self.reconnect_command(session_id)?;
1357 let readiness = async {
1358 let mut relay = {
1359 let _starting = ProvisionStageGuard::new(executor, ProvisionStage::Starting);
1360 start_worker(executor, &backend, &worker_root)?;
1361 connect_started_worker(&spec, session_id, executor, &backend, &worker_root)
1362 .await?
1363 };
1364 let native_session_id =
1365 wait_for_native_session_in_stage(&mut relay, executor, readiness_stage).await?;
1366 Ok::<_, anyhow::Error>((relay, native_session_id))
1367 }
1368 .await;
1369 let (mut relay, native_session_id) = readiness
1370 .map_err(|error| worker_probe_diagnosis(executor, &backend, &worker_root, error))?;
1371 if native_continuity {
1372 if native_session_id != archive_manifest.session.native_session_id {
1373 bail!(
1374 "ACP loaded native session {native_session_id}, expected {}",
1375 archive_manifest.session.native_session_id
1376 );
1377 }
1378 } else {
1379 relay
1380 .install_prompt_context(
1381 utility_handoff
1382 .clone()
1383 .context("a resume into a fresh native session has no handoff")?,
1384 )
1385 .await?;
1386 if !discard_queue {
1387 for prompt in &canonical_session.queued_prompts {
1388 let command = match &prompt.kind {
1392 CanonicalQueuedCommandKind::Prompt => RelayCommand::Prompt {
1393 prompt: prompt
1394 .content
1395 .iter()
1396 .cloned()
1397 .map(serde_json::from_value)
1398 .collect::<serde_json::Result<Vec<ContentBlock>>>()?,
1399 },
1400 CanonicalQueuedCommandKind::SetConfig { key, value } => {
1401 RelayCommand::SetConfig {
1402 key: key.clone(),
1403 value: value.clone(),
1404 }
1405 }
1406 };
1407 relay.submit(prompt.command_id.clone(), command).await?;
1408 }
1409 }
1410 }
1411 if let Some(worktree) = conversion
1415 .as_ref()
1416 .and_then(ResumeConversion::raw_to_workspace)
1417 .and_then(|plan| plan.retire.as_ref())
1418 && let Err(error) = retire_managed_worktree(executor, worktree)
1419 {
1420 tracing::warn!(
1421 session_id,
1422 worktree = %worktree.worktree_root.display(),
1423 error = format!("{error:#}"),
1424 "could not retire the old managed worktree after resume"
1425 );
1426 resume_notices.push(worktree_cleanup_notice(&worktree.worktree_root, &error));
1427 }
1428 for notice in &resume_notices {
1429 let submitted = async {
1430 let command_id = new_command_id("resume-notice")?;
1431 relay
1432 .submit(
1433 command_id,
1434 RelayCommand::RecordNotice {
1435 text: notice.clone(),
1436 },
1437 )
1438 .await
1439 }
1440 .await;
1441 if let Err(error) = submitted {
1444 tracing::warn!(
1445 session_id,
1446 error = format!("{error:#}"),
1447 "could not record a resume notice in the conversation"
1448 );
1449 }
1450 }
1451 self.mark_worker_connected(session_id, Some(native_session_id))?;
1452 Ok::<_, anyhow::Error>(relay.sync().await?.materialized)
1453 }
1454 .await;
1455 match result {
1456 Ok(materialized) => {
1457 if let Some(written) = &conversion_checkpoint_written {
1460 super::checkpoint::prune_replaced_checkpoint(
1461 previous.checkpoint.as_ref(),
1462 written,
1463 );
1464 }
1465 Ok(materialized)
1466 }
1467 Err(error) => {
1468 if let Some(written) = &conversion_checkpoint_written
1471 && let Err(remove_error) = std::fs::remove_file(&written.archive_path)
1472 && remove_error.kind() != std::io::ErrorKind::NotFound
1473 {
1474 tracing::warn!(
1475 session_id,
1476 path = %written.archive_path.display(),
1477 "could not remove the conversion checkpoint after resume failed: {remove_error}"
1478 );
1479 }
1480 if rebuild_projection {
1485 match materialized_session_from_canonical(session_id, &canonical_session) {
1486 Ok(previous_projection) => {
1487 if let Err(restore_error) =
1488 crate::database::save_materialized_session(&previous_projection)
1489 {
1490 tracing::error!(
1491 session_id,
1492 error = format!("{restore_error:#}"),
1493 "could not restore the durable projection after resume failed"
1494 );
1495 }
1496 }
1497 Err(restore_error) => {
1498 tracing::error!(
1499 session_id,
1500 error = format!("{restore_error:#}"),
1501 "could not rebuild the durable projection after resume failed"
1502 );
1503 }
1504 }
1505 } else if discard_queued_prompts
1506 && let Err(restore_error) = crate::database::replace_materialized_queued_prompts(
1507 session_id,
1508 &mj_transcript::projection::materialized_queued_prompts_from_canonical(
1509 &canonical_session.queued_prompts,
1510 ),
1511 )
1512 {
1513 tracing::error!(
1514 session_id,
1515 error = format!("{restore_error:#}"),
1516 "could not restore queued prompts after resume failed"
1517 );
1518 }
1519 Err(self.rollback_failed_resume(
1520 session_id,
1521 &previous,
1522 recreated_managed_worktree,
1523 error,
1524 executor,
1525 )?)
1526 }
1527 }
1528 }
1529
1530 pub(super) fn rollback_failed_resume(
1531 &mut self,
1532 session_id: &str,
1533 previous: &SessionRecord,
1534 recreated_managed_worktree: bool,
1535 error: anyhow::Error,
1536 _executor: &impl CommandExecutor,
1537 ) -> Result<anyhow::Error> {
1538 let current = self
1539 .state
1540 .sessions
1541 .get(session_id)
1542 .with_context(|| format!("unknown session {session_id}"))?
1543 .clone();
1544 let cleanup = match current.target.as_ref() {
1545 Some(locator) => (|| -> Result<()> {
1546 let backend = backend_locator(locator, ¤t, &self.config)?;
1547 targets::close_plan(&backend, session_id)?
1548 .execute(&CancellableProcessExecutor::with_timeout(
1551 Duration::from_secs(15),
1552 ))
1553 .map(|_| ())
1554 })(),
1555 None => Ok(()),
1556 };
1557 let worktree_cleanup = if cleanup.is_err() {
1560 Ok(())
1561 } else {
1562 match (
1563 current.managed_worktree.as_ref(),
1564 previous.managed_worktree.as_ref(),
1565 ) {
1566 (_, Some(previous)) if recreated_managed_worktree => retire_managed_worktree(
1567 &CancellableProcessExecutor::with_timeout(Duration::from_secs(15)),
1568 previous,
1569 ),
1570 (Some(current), Some(previous)) if current == previous => Ok(()),
1571 (Some(worktree), _) => cleanup_managed_worktree(
1572 &CancellableProcessExecutor::with_timeout(Duration::from_secs(15)),
1573 worktree,
1574 ),
1575 (None, _) => Ok(()),
1576 }
1577 };
1578 let cleanup_error = [cleanup, worktree_cleanup]
1579 .into_iter()
1580 .filter_map(Result::err)
1581 .map(|cleanup_error| format!("{cleanup_error:#}"))
1582 .collect::<Vec<_>>()
1583 .join("; ");
1584 if !cleanup_error.is_empty() {
1585 tracing::warn!(
1586 session_id,
1587 error = %cleanup_error,
1588 "resume rollback cleanup reported failures"
1589 );
1590 }
1591 let original = format!("{error:#}");
1592 let record = self.state.sessions.get_mut(session_id).unwrap();
1593 let failure = apply_failed_resume_rollback(
1594 record,
1595 previous,
1596 &original,
1597 (!cleanup_error.is_empty()).then_some(cleanup_error),
1598 );
1599 if record.bundle_id != current.bundle_id {
1602 let bundle_id = record.bundle_id.clone();
1603 crate::database::rebind_session_bundle(session_id, &bundle_id)?;
1604 }
1605 crate::database::save_session(&self.state.sessions[session_id])?;
1608 Ok(failure)
1609 }
1610}
1611
1612fn worktree_cleanup_notice(worktree_root: &Path, error: &anyhow::Error) -> String {
1613 format!(
1614 "Mjolnir could not remove the worktree at {}: {error:#}. Remove it with `git worktree remove --force {}`.",
1615 worktree_root.display(),
1616 worktree_root.display()
1617 )
1618}
1619
1620pub(super) fn apply_failed_resume_rollback(
1621 current: &mut SessionRecord,
1622 previous: &SessionRecord,
1623 original_error: &str,
1624 cleanup_error: Option<String>,
1625) -> anyhow::Error {
1626 match cleanup_error {
1627 None => {
1628 *current = previous.clone();
1629 current.state = SessionState::Stopped;
1630 current.target = None;
1631 current.updated_at = now();
1632 current.last_error = Some(format!("resume failed: {original_error}"));
1633 anyhow::anyhow!(original_error.to_owned())
1634 }
1635 Some(cleanup_error) => {
1636 let failure = format!(
1637 "{original_error}; cleanup of the partial resume target failed: {cleanup_error}"
1638 );
1639 if current.managed_worktree.is_none() {
1644 current
1645 .project_directory
1646 .clone_from(&previous.project_directory);
1647 current
1648 .managed_worktree
1649 .clone_from(&previous.managed_worktree);
1650 current.bundle_id.clone_from(&previous.bundle_id);
1651 }
1652 current.state = SessionState::Error;
1653 current.updated_at = now();
1654 current.last_error = Some(format!("resume failed: {failure}"));
1655 anyhow::anyhow!(failure)
1656 }
1657 }
1658}
1659
1660fn conversion_notice(
1662 target_id: &str,
1663 checkout: &Path,
1664 branch: Option<&str>,
1665 retire: Option<&mj_core::state::ManagedWorktree>,
1666) -> String {
1667 let branch = branch.unwrap_or("a detached head");
1668 match retire {
1669 Some(worktree) => format!(
1670 "This session moved out of {} and into the {target_id} target, where its checkout is on {branch}. Its branch {} stays in {}.",
1671 checkout.display(),
1672 worktree.branch,
1673 worktree.source_repository.display()
1674 ),
1675 None => format!(
1676 "This session moved out of {} and into the {target_id} target, where its checkout is on {branch}. The checkout on this machine stays where it is.",
1677 checkout.display()
1678 ),
1679 }
1680}
1681
1682fn conversion_checkpoint(
1690 previous_archive: &Path,
1691 snapshot: mj_checkpoint::archive::RepositorySnapshot,
1692 output: &Path,
1693) -> Result<mj_core::state::CheckpointMetadata> {
1694 let previous = mj_checkpoint::archive::read_archive_verified(previous_archive)
1695 .with_context(|| format!("read checkpoint archive {}", previous_archive.display()))?;
1696 let native_artifacts = previous
1699 .manifest
1700 .payloads
1701 .iter()
1702 .filter_map(|descriptor| match &descriptor.role {
1703 mj_checkpoint::archive::PayloadRole::NativeArtifact { relative_path } => {
1704 Some((relative_path, descriptor))
1705 }
1706 _ => None,
1707 })
1708 .map(|(relative_path, descriptor)| {
1709 Ok(mj_checkpoint::archive::NativeArtifact {
1710 relative_path: relative_path.clone(),
1711 data: previous.payload(descriptor)?.to_vec(),
1712 mode: descriptor.mode,
1713 })
1714 })
1715 .collect::<Result<Vec<_>>>()?;
1716 let canonical_session = previous.canonical_session()?;
1717 let event_frontier = canonical_session.event_frontier;
1718 let written = mj_checkpoint::archive::write_archive_atomic(
1719 output,
1720 &mj_checkpoint::archive::ArchiveInput {
1721 session: previous.manifest.session.clone(),
1722 target: previous.manifest.target.clone(),
1725 bundle: mj_checkpoint::archive::BundleManifest {
1726 id: previous.manifest.bundle.id.clone(),
1727 primary_repository: snapshot.metadata.id.clone(),
1731 },
1732 canonical_session,
1733 native_artifacts,
1734 repositories: vec![snapshot],
1735 },
1736 )
1737 .with_context(|| format!("write the conversion archive {}", output.display()))?;
1738 Ok(mj_core::state::CheckpointMetadata {
1739 archive_path: output.to_path_buf(),
1740 sha256: written.archive_sha256,
1741 created_at: now(),
1742 event_frontier,
1743 })
1744}
1745
1746fn projection_rebuild_required(
1754 stored: Option<(u64, &str)>,
1755 archive_frontier: u64,
1756 archive_frontier_digest: &str,
1757) -> bool {
1758 stored != Some((archive_frontier, archive_frontier_digest))
1759}
1760
1761fn restore_archive_path(
1762 backend: &targets::TargetLocator,
1763 verified_archive: &Path,
1764 remote_archive: &Path,
1765) -> PathBuf {
1766 if matches!(backend, targets::TargetLocator::LocalBare { .. }) {
1767 verified_archive.to_path_buf()
1768 } else {
1769 remote_archive.to_path_buf()
1770 }
1771}
1772
1773fn should_upload_restore_archive(backend: &targets::TargetLocator) -> bool {
1774 !matches!(backend, targets::TargetLocator::LocalBare { .. })
1775}
1776
1777struct CrossHarnessProvisionExecutor<'a, E: CommandExecutor + ?Sized> {
1782 inner: &'a E,
1783 cancellation: CancellationToken,
1784}
1785
1786impl<E: CommandExecutor + ?Sized> CommandExecutor for CrossHarnessProvisionExecutor<'_, E> {
1787 fn execute(&self, command: &CommandSpec) -> Result<CommandOutput> {
1788 if self.cancellation.is_cancelled() {
1789 bail!("operation cancelled while provisioning destination");
1790 }
1791 self.inner.execute(command)
1792 }
1793
1794 fn cancellation_requested(&self) -> bool {
1795 self.cancellation.is_cancelled() || self.inner.cancellation_requested()
1796 }
1797
1798 fn stage_started(&self, stage: ProvisionStage) {
1799 self.inner.stage_started(stage);
1800 }
1801
1802 fn stage_finished(&self, stage: ProvisionStage) {
1803 self.inner.stage_finished(stage);
1804 }
1805
1806 fn notify_notice(&self, notice: &str) {
1807 self.inner.notify_notice(notice);
1808 }
1809
1810 fn execute_with_stdin(
1811 &self,
1812 command: &CommandSpec,
1813 input: &mut (dyn std::io::Read + Send),
1814 ) -> Result<CommandOutput> {
1815 if self.cancellation.is_cancelled() {
1816 bail!("operation cancelled while provisioning destination");
1817 }
1818 self.inner.execute_with_stdin(command, input)
1819 }
1820}
1821
1822fn provision_with_cross_harness_handoff(
1823 controller: &mut Controller,
1824 session_id: &str,
1825 executor: &(impl CommandExecutor + Sync),
1826 github_token: Option<&str>,
1827 config: &Config,
1828 snapshot: &CanonicalSessionSnapshot,
1829 context_bytes: usize,
1830) -> Result<String> {
1831 let (_provision, handoff) = execute_joined_cross_harness_work(
1832 "cross-harness provisioning",
1833 move |cancellation| {
1834 let provision_executor = CrossHarnessProvisionExecutor {
1835 inner: executor,
1836 cancellation,
1837 };
1838 futures::executor::block_on(controller.provision_session_with_failure_disposition(
1839 session_id,
1840 &provision_executor,
1841 github_token,
1842 ProvisioningFailureDisposition::Preserve,
1843 ))
1844 },
1845 "cross-harness handoff",
1846 move |cancellation| {
1847 let runtime = tokio::runtime::Builder::new_current_thread()
1848 .enable_all()
1849 .build()
1850 .context("create cross-harness handoff runtime")?;
1851 runtime.block_on(utility_handoff_while_cancellable(
1852 session_id,
1853 config,
1854 snapshot,
1855 context_bytes,
1856 executor,
1857 cancellation,
1858 ))
1859 },
1860 )?;
1861 ensure!(
1862 !executor.cancellation_requested(),
1863 "operation cancelled while provisioning destination"
1864 );
1865 Ok(handoff)
1866}
1867
1868fn execute_joined_cross_harness_work<A: Send, B: Send>(
1872 first_name: &'static str,
1873 first: impl FnOnce(CancellationToken) -> Result<A> + Send,
1874 second_name: &'static str,
1875 second: impl FnOnce(CancellationToken) -> Result<B> + Send,
1876) -> Result<(A, B)> {
1877 let cancellation = CancellationToken::new();
1878 std::thread::scope(|scope| {
1879 let first_cancel = cancellation.clone();
1880 let mut first_handle = Some(scope.spawn(move || first(first_cancel)));
1881 let second_cancel = cancellation.clone();
1882 let mut second_handle = Some(scope.spawn(move || second(second_cancel)));
1883 let mut first_result = None;
1884 let mut second_result = None;
1885
1886 while first_result.is_none() || second_result.is_none() {
1887 if first_result.is_none()
1888 && first_handle
1889 .as_ref()
1890 .is_some_and(|handle| handle.is_finished())
1891 {
1892 let handle = first_handle.take().expect("first lane handle present");
1893 first_result = Some(match handle.join() {
1894 Ok(result) => result,
1895 Err(panic) => {
1896 cancellation.cancel();
1897 Err(anyhow::anyhow!(
1898 "{first_name} thread panicked: {}",
1899 targets::command_thread_panic_message(panic.as_ref())
1900 ))
1901 }
1902 });
1903 if first_result.as_ref().is_some_and(Result::is_err) {
1904 cancellation.cancel();
1905 }
1906 }
1907 if second_result.is_none()
1908 && second_handle
1909 .as_ref()
1910 .is_some_and(|handle| handle.is_finished())
1911 {
1912 let handle = second_handle.take().expect("second lane handle present");
1913 second_result = Some(match handle.join() {
1914 Ok(result) => result,
1915 Err(panic) => {
1916 cancellation.cancel();
1917 Err(anyhow::anyhow!(
1918 "{second_name} thread panicked: {}",
1919 targets::command_thread_panic_message(panic.as_ref())
1920 ))
1921 }
1922 });
1923 if second_result.as_ref().is_some_and(Result::is_err) {
1924 cancellation.cancel();
1925 }
1926 }
1927 if first_result.is_none() || second_result.is_none() {
1928 std::thread::sleep(Duration::from_millis(10));
1929 }
1930 }
1931
1932 match (
1933 first_result.expect("first lane result received after joined handle"),
1934 second_result.expect("second lane result received after joined handle"),
1935 ) {
1936 (Err(first), Err(second)) => {
1937 Err(first.context(format!("{second_name} lane also failed: {second:#}")))
1938 }
1939 (Err(error), Ok(_)) => Err(error),
1940 (Ok(_), Err(error)) => Err(error),
1941 (Ok(first), Ok(second)) => Ok((first, second)),
1942 }
1943 })
1944}
1945
1946fn native_continuity_preserved(profile_kind: HarnessKind, archived_kind: HarnessKind) -> bool {
1955 profile_kind == archived_kind && profile_kind.captures_native_session()
1956}
1957
1958async fn utility_handoff_while_cancellable(
1962 session_id: &str,
1963 config: &Config,
1964 snapshot: &CanonicalSessionSnapshot,
1965 context_bytes: usize,
1966 executor: &impl CommandExecutor,
1967 cancellation: CancellationToken,
1968) -> Result<String> {
1969 let _phase = ResumePhaseTimer::new(session_id, "cross-harness handoff");
1970 if executor.cancellation_requested() {
1971 bail!("operation cancelled while compacting the cross-harness handoff");
1972 }
1973 let _compacting = ProvisionStageGuard::new(executor, ProvisionStage::Compacting);
1974 let cancel = cancellation.child_token();
1975 let operation = async {
1976 let candidates = match crate::utility_llm::UtilityLlmRuntime::shared()
1977 .resolve(config, &cancel)
1978 .await
1979 {
1980 Ok(candidates) => candidates,
1981 Err(error) if cancel.is_cancelled() => return Err(error),
1983 Err(error) => {
1987 tracing::warn!(
1988 session_id,
1989 error = format!("{error:#}"),
1990 "no utility model is available for the resume handoff; handing over the most recent transcript verbatim"
1991 );
1992 return Ok(crate::compaction::render_recent_snapshot(
1993 snapshot,
1994 context_bytes,
1995 ));
1996 }
1997 };
1998 let backend = crate::utility_llm::UtilityCompactionBackend::new(candidates, cancel.clone());
1999 let budget = crate::compaction::CompactionBudget {
2004 page_bytes: backend.page_bytes(),
2005 handoff_bytes: context_bytes,
2006 };
2007 crate::compaction::compact_snapshot(snapshot, budget, &backend).await
2008 };
2009 tokio::pin!(operation);
2010 loop {
2011 tokio::select! {
2012 context = &mut operation => return context,
2013 _ = cancellation.cancelled() => {
2014 cancel.cancel();
2015 bail!("operation cancelled while compacting the cross-harness handoff");
2016 }
2017 _ = tokio::time::sleep(super::readiness::CANCELLATION_POLL_INTERVAL) => {
2018 if executor.cancellation_requested() {
2019 cancel.cancel();
2020 bail!("operation cancelled while compacting the cross-harness handoff");
2021 }
2022 }
2023 }
2024 }
2025}
2026
2027#[cfg(test)]
2028mod tests {
2029 use std::cell::RefCell;
2030 use std::collections::BTreeMap;
2031 use std::path::{Path, PathBuf};
2032 use std::process::Command;
2033 use std::sync::{Barrier, Mutex};
2034
2035 use anyhow::Result;
2036
2037 use crate::controller::test_support::{
2038 FIXTURE_FETCH_URL, FixtureRemoteExecutor, checkout_with_network_remote,
2039 checkpoint_test_session, committed_repository, managed_worktree_session,
2040 network_remote_for, raw_session_on, resume_compatibility_config,
2041 write_checkpoint_archive_with_native_state, write_checkpoint_gate_archive,
2042 };
2043 use crate::controller::{Controller, SessionResumeOptions};
2044 use mj_checkpoint::archive::{GitCommandRunner, verify_archive_streaming};
2045 use mj_core::config::{
2046 Config, ContainerTemplate as ConfigContainer, HarnessProfile, ProjectBundle,
2047 ProjectRepository, TargetTemplate,
2048 };
2049 use mj_core::state::{SessionRecord, SessionState, State, TargetLocator};
2050 use mj_transcript::projection::materialized_session_from_canonical;
2051
2052 use crate::targets::{CommandExecutor, CommandOutput, CommandSpec, ProcessExecutor};
2053
2054 use super::*;
2055
2056 #[test]
2060 fn a_local_checkout_resuming_into_a_container_preflights_its_conversion() {
2061 let (checkout, _remote_parent, remote) = checkout_with_network_remote();
2062 std::fs::write(checkout.path().join("untracked.txt"), "u".repeat(2048)).unwrap();
2063 let mut session = raw_session_on("local-bare", &checkout.path().to_string_lossy());
2064 session.checkpoint = Some(mj_core::state::CheckpointMetadata {
2065 archive_path: checkout.path().join("unused.hel.zip"),
2066 sha256: "a".repeat(64),
2067 created_at: "2026-09-14T00:00:00Z".into(),
2068 event_frontier: 3,
2069 });
2070 let session_id = session.id.clone();
2071 let controller = Controller {
2072 config: resume_compatibility_config(),
2073 state: State {
2074 sessions: BTreeMap::from([(session_id.clone(), session)]),
2075 ..State::default()
2076 },
2077 };
2078 let executor = FixtureRemoteExecutor { remote };
2079
2080 let converting = controller
2081 .preflight_resume_repository_sources(&session_id, "podman", &executor)
2082 .unwrap();
2083 let ResumeRepositorySourcePreflight::ConvertingRawCheckout { receipt, preview } =
2084 converting
2085 else {
2086 panic!("a container destination converts the checkout, got {converting:?}");
2087 };
2088 assert_eq!(receipt.session_id, session_id);
2089 assert_eq!(preview.fetch_url, FIXTURE_FETCH_URL);
2090 assert_eq!(preview.untracked_files, 1);
2091 assert!(preview.host_checkout_retained);
2092
2093 assert!(
2094 matches!(
2095 controller
2096 .preflight_resume_repository_sources(&session_id, "local-bare", &executor)
2097 .unwrap(),
2098 ResumeRepositorySourcePreflight::Ready(_)
2099 ),
2100 "resuming in place asks nothing"
2101 );
2102 }
2103
2104 const RESUME_ROLLBACK_TEST_CHILD: &str = "MJ_RESUME_ROLLBACK_TEST_CHILD";
2105 const RETIRED_WORKTREE_RESUME_TEST_CHILD: &str = "MJ_RETIRED_WORKTREE_RESUME_TEST_CHILD";
2106 const WORKER_PREFLIGHT_TEST_CHILD: &str = "MJ_WORKER_PREFLIGHT_TEST_CHILD";
2107
2108 #[test]
2109 fn muse_resume_allows_workspace_relocation_before_provisioning() {
2110 let mut config = resume_compatibility_config();
2111 config
2112 .targets
2113 .insert("other-container".into(), config.targets["podman"].clone());
2114 let controller = Controller {
2115 config,
2116 state: State::default(),
2117 };
2118 let mut session = checkpoint_test_session("0123456789abcdef0123456789abcdef");
2119 session.harness_kind = HarnessKind::Muse;
2120 assert!(
2121 controller
2122 .validate_muse_resume_destination(&session, HarnessKind::Muse, "podman")
2123 .is_ok()
2124 );
2125 assert!(
2126 controller
2127 .validate_muse_resume_destination(&session, HarnessKind::Muse, "other-container")
2128 .is_ok()
2129 );
2130 controller
2131 .validate_muse_resume_destination(&session, HarnessKind::Muse, "ssh-bare")
2132 .unwrap();
2133 assert!(
2134 controller
2135 .validate_muse_resume_destination(&session, HarnessKind::Codex, "ssh-bare")
2136 .is_ok()
2137 );
2138 assert_eq!(session.state, SessionState::Running);
2139 }
2140
2141 #[test]
2145 fn a_resume_preflights_the_worker_binary_before_compacting() {
2146 if std::env::var_os(WORKER_PREFLIGHT_TEST_CHILD).is_none() {
2149 let directory = tempfile::tempdir().unwrap();
2150 let test_name = format!(
2151 "{}::a_resume_preflights_the_worker_binary_before_compacting",
2152 module_path!()
2153 .strip_prefix("mj_controller::")
2154 .unwrap_or(module_path!())
2155 );
2156 let output = Command::new(std::env::current_exe().unwrap())
2157 .args(["--exact", &test_name, "--nocapture"])
2158 .env(WORKER_PREFLIGHT_TEST_CHILD, "1")
2159 .env("MJ_DATA_DIR", directory.path().join("data"))
2160 .env("MJ_CONFIG_DIR", directory.path().join("config"))
2161 .env("MJ_WORKER_BINARY", directory.path().join("absent-worker"))
2164 .output()
2165 .unwrap();
2166 assert!(
2167 output.status.success(),
2168 "isolated worker preflight test failed\nstdout:\n{}\nstderr:\n{}",
2169 String::from_utf8_lossy(&output.stdout),
2170 String::from_utf8_lossy(&output.stderr)
2171 );
2172 return;
2173 }
2174 let _writer = crate::database::install_isolated_test_writer();
2176
2177 let data_directory = PathBuf::from(std::env::var_os("MJ_DATA_DIR").unwrap());
2178 let archive_directory = data_directory.join("archives");
2179 std::fs::create_dir_all(&archive_directory).unwrap();
2180 let session_id = "0123456789abcdef0123456789abcdef";
2181 let checkpoint = write_checkpoint_gate_archive(&archive_directory, session_id, 7);
2182 let repository = committed_repository();
2183 let mut session = managed_worktree_session(repository.path(), session_id);
2184 session.checkpoint = Some(checkpoint);
2185
2186 let profile_home = data_directory.join("profile");
2187 std::fs::create_dir_all(&profile_home).unwrap();
2188 let mut config = resume_compatibility_config();
2189 config.profiles.insert(
2192 "claude".into(),
2193 HarnessProfile {
2194 enabled: true,
2195 kind: mj_core::config::HarnessKind::Claude,
2196 home: profile_home,
2197 environment: BTreeMap::new(),
2198 context_window_bytes: None,
2199 },
2200 );
2201 let mut controller = Controller {
2202 config,
2203 state: State {
2204 sessions: BTreeMap::from([(session_id.into(), session)]),
2205 ..State::default()
2206 },
2207 };
2208 crate::database::save_state(&controller.state).unwrap();
2209
2210 let error = tokio::runtime::Builder::new_current_thread()
2211 .enable_all()
2212 .build()
2213 .unwrap()
2214 .block_on(controller.resume_session_controlled(
2215 session_id,
2216 "claude",
2217 "local-bare",
2218 SessionResumeOptions {
2219 additional_mounts: None,
2220 resource_allocation: None,
2221 discard_queue: false,
2222 },
2223 &ProcessExecutor,
2224 ))
2225 .unwrap_err();
2226
2227 let detail = format!("{error:#}");
2228 assert!(
2229 detail.contains("preflight the worker binary before resuming"),
2230 "{detail}"
2231 );
2232 assert!(detail.contains("absent-worker"), "{detail}");
2233 assert!(
2234 !detail.contains("compact the cross-harness handoff transcript"),
2235 "compaction must not run for a resume that cannot install a worker: {detail}"
2236 );
2237 assert_eq!(
2238 controller.state.sessions[session_id].state,
2239 SessionState::Stopped
2240 );
2241 }
2242
2243 #[test]
2244 fn network_resume_ignores_host_history_but_an_explicit_raw_move_checks_it() {
2245 let directory = tempfile::tempdir().unwrap();
2246 let repository = committed_repository();
2247 let session_id = "0123456789abcdef0123456789abcdef";
2248 let mut session = checkpoint_test_session(session_id);
2249 session.state = SessionState::Stopped;
2250 session.checkpoint = Some(
2251 super::super::test_support::write_network_checkpoint_archive(
2252 directory.path(),
2253 session_id,
2254 0,
2255 ),
2256 );
2257 let mut config = resume_compatibility_config();
2258 config.bundles.insert(
2259 "project".into(),
2260 super::super::test_support::local_bundle(repository.path()),
2261 );
2262 let controller = Controller {
2263 config,
2264 state: State {
2265 sessions: BTreeMap::from([(session_id.into(), session)]),
2266 ..State::default()
2267 },
2268 };
2269 assert!(matches!(
2270 controller
2271 .preflight_resume_repository_sources(session_id, "podman", &ProcessExecutor,)
2272 .unwrap(),
2273 ResumeRepositorySourcePreflight::Ready(_)
2274 ));
2275 let result = controller
2276 .preflight_resume_repository_sources(session_id, "local-bare", &ProcessExecutor)
2277 .unwrap();
2278 let ResumeRepositorySourcePreflight::RepositoryMoved(mismatch) = result else {
2279 panic!("moving into a host checkout must detect its missing archive base");
2280 };
2281 assert_eq!(mismatch.missing_commit, "a".repeat(40));
2282 assert!(!repository.path().join(".mj/worktrees").exists());
2283 }
2284
2285 #[test]
2286 fn raw_in_place_preflight_does_not_require_its_synthetic_bundle() {
2287 struct UnusedExecutor;
2288
2289 impl CommandExecutor for UnusedExecutor {
2290 fn execute(&self, command: &CommandSpec) -> Result<CommandOutput> {
2291 panic!("raw in-place preflight ran {}", command.purpose);
2292 }
2293 }
2294
2295 let directory = tempfile::tempdir().unwrap();
2296 let session_id = "0123456789abcdef0123456789abcdef";
2297 let mut session = checkpoint_test_session(session_id);
2298 session.checkpoint = Some(write_checkpoint_gate_archive(
2299 directory.path(),
2300 session_id,
2301 3,
2302 ));
2303 session.bundle_id = "remote-project-a66373eef659f856".into();
2304 session.target_template_id = "localhost".into();
2305 session.project_directory = Some("/mnt/optane/bifrost-fird".into());
2306 let controller = Controller {
2307 config: Config {
2308 targets: BTreeMap::from([("localhost".into(), TargetTemplate::LocalBare)]),
2309 bundles: BTreeMap::new(),
2312 ..Config::default()
2313 },
2314 state: State {
2315 sessions: BTreeMap::from([(session_id.into(), session)]),
2316 ..State::default()
2317 },
2318 };
2319
2320 let preflight = controller
2321 .preflight_resume_repository_sources(session_id, "localhost", &UnusedExecutor)
2322 .unwrap();
2323 let ResumeRepositorySourcePreflight::Ready(receipt) = preflight else {
2324 panic!("raw in-place resume unexpectedly needs a repository replacement");
2325 };
2326 assert!(controller.repository_source_receipt_is_current(session_id, &receipt));
2327 }
2328
2329 #[test]
2330 fn repository_preflight_distinguishes_the_original_source_from_a_reused_name() {
2331 fn git(repository: &Path, arguments: &[&str]) {
2332 let output = SystemGit
2333 .run(
2334 repository,
2335 &mj_checkpoint::archive::GitCommand {
2336 arguments: arguments.iter().map(std::ffi::OsString::from).collect(),
2337 stdin: Vec::new(),
2338 env: Vec::new(),
2339 },
2340 )
2341 .unwrap();
2342 assert_eq!(
2343 output.status,
2344 0,
2345 "git {arguments:?}: {}",
2346 String::from_utf8_lossy(&output.stderr)
2347 );
2348 }
2349
2350 let directory = tempfile::tempdir().unwrap();
2351 let origin = directory.path().join("original");
2352 std::fs::create_dir(&origin).unwrap();
2353 git(&origin, &["init", "-q", "-b", "main"]);
2354 git(&origin, &["config", "user.name", "Hel Test"]);
2355 git(&origin, &["config", "user.email", "hel@example.test"]);
2356 git(&origin, &["commit", "--allow-empty", "-qm", "base"]);
2357 let source = directory.path().join("source");
2358 git(
2359 directory.path(),
2360 &["clone", "-q", origin.to_str().unwrap(), "source"],
2361 );
2362 git(&source, &["config", "user.name", "Hel Test"]);
2363 git(&source, &["config", "user.email", "hel@example.test"]);
2364 git(&source, &["commit", "--allow-empty", "-qm", "session"]);
2365 let snapshot = mj_checkpoint::archive::collect_git_snapshot(
2366 &SystemGit,
2367 &source,
2368 &mj_checkpoint::archive::GitCollectionSpec {
2369 id: "project".into(),
2370 relative_destination: "project".into(),
2371 history: mj_checkpoint::archive::GitHistoryMode::SessionDelta,
2372 origin_override: None,
2373 },
2374 )
2375 .unwrap();
2376 let configured = ProjectRepository {
2377 id: "project".into(),
2378 github: None,
2379 local: Some(origin.clone()),
2380 destination: "project".into(),
2381 git_ref: None,
2382 };
2383 assert_eq!(
2384 checkpoint_source_missing_commit(
2385 &configured,
2386 &CheckpointRepositoryBundle {
2387 metadata: snapshot.metadata.clone(),
2388 committed_bundle: snapshot.committed_bundle.clone(),
2389 },
2390 &ProcessExecutor,
2391 None,
2392 )
2393 .unwrap(),
2394 None
2395 );
2396
2397 let replacement = directory.path().join("replacement");
2398 std::fs::create_dir(&replacement).unwrap();
2399 git(&replacement, &["init", "-q", "-b", "main"]);
2400 git(&replacement, &["config", "user.name", "Hel Test"]);
2401 git(&replacement, &["config", "user.email", "hel@example.test"]);
2402 git(
2403 &replacement,
2404 &["commit", "--allow-empty", "-qm", "different history"],
2405 );
2406 let configured = ProjectRepository {
2407 local: Some(replacement),
2408 ..configured
2409 };
2410 assert!(
2411 checkpoint_source_missing_commit(
2412 &configured,
2413 &CheckpointRepositoryBundle {
2414 metadata: snapshot.metadata,
2415 committed_bundle: snapshot.committed_bundle,
2416 },
2417 &ProcessExecutor,
2418 None,
2419 )
2420 .unwrap()
2421 .is_some()
2422 );
2423 }
2424
2425 #[test]
2426 fn repository_preflight_checks_independent_sources_concurrently_and_receipts_are_scoped() {
2427 struct ConcurrentSourceExecutor {
2428 source_checks: Barrier,
2429 }
2430
2431 impl CommandExecutor for ConcurrentSourceExecutor {
2432 fn execute(&self, command: &CommandSpec) -> Result<CommandOutput> {
2433 if command.purpose == "check checkpoint base commit" {
2434 self.source_checks.wait();
2435 }
2436 Ok(CommandOutput {
2437 status: 0,
2438 stdout: Vec::new(),
2439 stderr: Vec::new(),
2440 })
2441 }
2442 }
2443
2444 let directory = tempfile::tempdir().unwrap();
2445 let session_id = "0123456789abcdef0123456789abcdef";
2446 let checkpoint = write_checkpoint_gate_archive(directory.path(), session_id, 3);
2447 let repositories = ["one", "two"]
2448 .map(|id| ProjectRepository {
2449 id: id.into(),
2450 github: None,
2451 local: Some(PathBuf::from(format!("/origin/{id}"))),
2452 destination: id.into(),
2453 git_ref: None,
2454 })
2455 .to_vec();
2456 let mut session = checkpoint_test_session(session_id);
2457 session.checkpoint = Some(checkpoint.clone());
2458 let mut controller = Controller {
2459 config: Config {
2460 bundles: BTreeMap::from([(
2461 session.bundle_id.clone(),
2462 ProjectBundle {
2463 primary_repo: "one".into(),
2464 repositories: repositories.clone(),
2465 },
2466 )]),
2467 ..Config::default()
2468 },
2469 state: State {
2470 sessions: BTreeMap::from([(session_id.into(), session)]),
2471 ..State::default()
2472 },
2473 };
2474 let verified = ResumeRepositoryBundles {
2475 checkpoint_sha256: checkpoint.sha256,
2476 repositories: repositories
2477 .iter()
2478 .map(|repository| CheckpointRepositoryBundle {
2479 metadata: mj_checkpoint::archive::RepositoryMetadata {
2480 push_urls: Vec::new(),
2481 remote_workspace: false,
2482 id: repository.id.clone(),
2483 relative_destination: repository.destination.clone(),
2484 origin: repository.source_label(),
2485 base_commit: String::new(),
2486 head_commit: if repository.id == "one" {
2487 "a".repeat(40)
2488 } else {
2489 "b".repeat(40)
2490 },
2491 branch: Some("main".into()),
2492 },
2493 committed_bundle: Vec::new(),
2494 })
2495 .collect(),
2496 };
2497 let executor = ConcurrentSourceExecutor {
2498 source_checks: Barrier::new(2),
2499 };
2500 let pool = rayon::ThreadPoolBuilder::new()
2501 .num_threads(2)
2502 .build()
2503 .unwrap();
2504 let preflight = pool
2505 .install(|| {
2506 controller.preflight_verified_repository_sources(
2507 session_id, verified, None, false, &executor,
2508 )
2509 })
2510 .unwrap();
2511 let ResumeRepositorySourcePreflight::Ready(receipt) = preflight else {
2512 panic!("expected repository source receipt");
2513 };
2514 assert!(controller.repository_source_receipt_is_current(session_id, &receipt));
2515
2516 controller
2517 .config
2518 .bundles
2519 .values_mut()
2520 .next()
2521 .unwrap()
2522 .repositories[0]
2523 .local = Some(PathBuf::from("/different-origin"));
2524 assert!(!controller.repository_source_receipt_is_current(session_id, &receipt));
2525 }
2526
2527 #[test]
2528 fn repository_preflight_checks_declared_boundary_without_importing_delta_bundle() {
2529 struct RecordingExecutor {
2530 commands: Mutex<Vec<CommandSpec>>,
2531 }
2532
2533 impl CommandExecutor for RecordingExecutor {
2534 fn execute(&self, command: &CommandSpec) -> Result<CommandOutput> {
2535 self.commands.lock().unwrap().push(command.clone());
2536 Ok(CommandOutput {
2537 status: 0,
2538 stdout: Vec::new(),
2539 stderr: Vec::new(),
2540 })
2541 }
2542 }
2543
2544 let prerequisite = "a".repeat(40);
2545 let head = "b".repeat(40);
2546 let archived = CheckpointRepositoryBundle {
2547 metadata: mj_checkpoint::archive::RepositoryMetadata {
2548 push_urls: Vec::new(),
2549 remote_workspace: false,
2550 id: "project".into(),
2551 relative_destination: "project".into(),
2552 origin: "https://github.com/archived/should-not-be-contacted.git".into(),
2553 base_commit: prerequisite.clone(),
2554 head_commit: head.clone(),
2555 branch: Some("main".into()),
2556 },
2557 committed_bundle: format!(
2558 "# v2 git bundle\n-{prerequisite} base\n{head} HEAD\n\nPACKnot-read"
2559 )
2560 .into_bytes(),
2561 };
2562 let configured = ProjectRepository {
2563 id: "project".into(),
2564 github: Some("configured/project".into()),
2565 local: None,
2566 destination: "project".into(),
2567 git_ref: None,
2568 };
2569 let executor = RecordingExecutor {
2570 commands: Mutex::new(Vec::new()),
2571 };
2572
2573 assert_eq!(
2574 checkpoint_source_missing_commit(
2575 &configured,
2576 &archived,
2577 &executor,
2578 Some("secret-token")
2579 )
2580 .unwrap(),
2581 None
2582 );
2583
2584 let commands = executor.commands.into_inner().unwrap();
2585 assert_eq!(commands.len(), 2, "commands: {commands:?}");
2586 assert_eq!(
2587 commands
2588 .iter()
2589 .map(|command| command.purpose.as_str())
2590 .collect::<Vec<_>>(),
2591 [
2592 "initialize repository source preflight",
2593 "check checkpoint base commit"
2594 ]
2595 );
2596 let source_check = &commands[1];
2597 assert!(
2598 source_check
2599 .args
2600 .iter()
2601 .any(|argument| argument == "credential.helper=")
2602 );
2603 assert_eq!(
2604 source_check
2605 .env
2606 .get("GIT_NO_LAZY_FETCH")
2607 .map(String::as_str),
2608 Some("1")
2609 );
2610 assert_eq!(
2611 source_check
2612 .env
2613 .get("GIT_TERMINAL_PROMPT")
2614 .map(String::as_str),
2615 Some("0")
2616 );
2617 assert_eq!(
2618 source_check.args.last().map(String::as_str),
2619 Some(prerequisite.as_str())
2620 );
2621 assert!(
2622 !source_check
2623 .args
2624 .iter()
2625 .any(|argument| argument.contains("archived"))
2626 );
2627 }
2628
2629 #[test]
2630 fn self_contained_bundle_validation_cannot_lazy_fetch_or_prompt() {
2631 let command = checkpoint_bundle_import_command(
2632 Path::new("/tmp/repository.git"),
2633 Path::new("/tmp/checkpoint.bundle"),
2634 );
2635 assert_eq!(
2636 command.env.get("GIT_NO_LAZY_FETCH").map(String::as_str),
2637 Some("1")
2638 );
2639 assert_eq!(
2640 command.env.get("GIT_TERMINAL_PROMPT").map(String::as_str),
2641 Some("0")
2642 );
2643 }
2644
2645 #[test]
2646 fn lost_bundle_sessions_reach_resume_compatibility_before_the_record_changes() {
2647 struct UnusedExecutor;
2648
2649 impl CommandExecutor for UnusedExecutor {
2650 fn execute(&self, command: &CommandSpec) -> Result<CommandOutput> {
2651 panic!("resume ran {} before rejecting the target", command.program);
2652 }
2653 }
2654
2655 let directory = tempfile::tempdir().unwrap();
2656 let session_id = "0123456789abcdef0123456789abcdef";
2657 let checkpoint = write_checkpoint_gate_archive(directory.path(), session_id, 3);
2658 let mut session = checkpoint_test_session(session_id);
2659 session.state = SessionState::Lost;
2660 session.checkpoint = Some(checkpoint);
2661 let previous = session.clone();
2662 let profile_home = directory.path().join("profile");
2663 std::fs::create_dir_all(&profile_home).unwrap();
2664 let mut config = Config::default();
2665 config.profiles.insert(
2666 "codex".into(),
2667 HarnessProfile {
2668 enabled: true,
2669 kind: mj_core::config::HarnessKind::Codex,
2670 home: profile_home,
2671 environment: BTreeMap::new(),
2672 context_window_bytes: None,
2673 },
2674 );
2675 config
2676 .targets
2677 .insert("localhost".into(), TargetTemplate::LocalBare);
2678 let mut controller = Controller {
2679 config,
2680 state: State {
2681 sessions: BTreeMap::from([(session_id.into(), session)]),
2682 ..State::default()
2683 },
2684 };
2685
2686 let error = tokio::runtime::Builder::new_current_thread()
2687 .enable_all()
2688 .build()
2689 .unwrap()
2690 .block_on(controller.resume_session_controlled(
2691 session_id,
2692 "codex",
2693 "localhost",
2694 SessionResumeOptions {
2695 additional_mounts: None,
2696 resource_allocation: None,
2697 discard_queue: false,
2698 },
2699 &UnusedExecutor,
2700 ))
2701 .unwrap_err();
2702
2703 let detail = format!("{error:#}");
2704 assert!(detail.contains("created from a project bundle"), "{detail}");
2705 assert!(
2706 detail.contains("resume it on a container, SSH, or EC2 target"),
2707 "{detail}"
2708 );
2709 assert_eq!(controller.state.sessions[session_id], previous);
2710 }
2711 struct BarrierExecutor {
2715 seen: Mutex<Vec<String>>,
2716 barrier: Barrier,
2717 }
2718
2719 impl CommandExecutor for BarrierExecutor {
2720 fn execute(&self, command: &CommandSpec) -> Result<CommandOutput> {
2721 self.seen.lock().unwrap().push(command.purpose.clone());
2722 self.barrier.wait();
2723 Ok(CommandOutput {
2724 status: 0,
2725 stdout: Vec::new(),
2726 stderr: Vec::new(),
2727 })
2728 }
2729 }
2730
2731 fn lane_command(purpose: &str) -> CommandSpec {
2732 CommandSpec::new("hel", ["worker"]).purpose(purpose)
2733 }
2734
2735 #[test]
2740 fn start_begins_at_the_worker_launch_not_at_the_transfers_before_it() {
2741 struct RecordingExecutor {
2742 commands: RefCell<Vec<CommandSpec>>,
2743 }
2744 impl CommandExecutor for RecordingExecutor {
2745 fn execute(&self, command: &CommandSpec) -> Result<CommandOutput> {
2746 self.commands.borrow_mut().push(command.clone());
2747 Ok(CommandOutput {
2748 status: 0,
2749 stdout: Vec::new(),
2750 stderr: Vec::new(),
2751 })
2752 }
2753 }
2754
2755 let session_id = "0123456789abcdef0123456789abcdef";
2756 let worker_root = format!("/var/lib/hel/workers/{session_id}");
2757 let executor = RecordingExecutor {
2758 commands: RefCell::new(Vec::new()),
2759 };
2760 let syncing = StagedExecutor::new(&executor, ProvisionStage::Syncing);
2761 let backend = targets::TargetLocator::LocalPodman {
2762 container_id: "abcdef0123456789".into(),
2763 workspace_storage: Default::default(),
2764 };
2765
2766 upload_checkpoint_spec(
2767 &syncing,
2768 &backend,
2769 session_id,
2770 Path::new("/archives/session.hel.zip"),
2771 &format!("{worker_root}/restore.hel.zip"),
2772 )
2773 .unwrap();
2774 execute_checked(
2775 &syncing,
2776 restore_command(
2777 &backend,
2778 session_id,
2779 &format!("{worker_root}/restore-spec.json"),
2780 )
2781 .unwrap(),
2782 )
2783 .unwrap();
2784 start_worker(&syncing, &backend, &worker_root).unwrap();
2787
2788 let stages = executor
2789 .commands
2790 .borrow()
2791 .iter()
2792 .map(|command| (command.purpose.clone(), command.stage))
2793 .collect::<Vec<_>>();
2794 assert_eq!(
2795 stages,
2796 vec![
2797 (
2798 "upload checkpoint specification".to_owned(),
2799 Some(ProvisionStage::Syncing)
2800 ),
2801 (
2802 "restore target checkpoint".to_owned(),
2803 Some(ProvisionStage::Syncing)
2804 ),
2805 (
2806 "start detached Mjolnir worker".to_owned(),
2807 Some(ProvisionStage::Starting)
2808 ),
2809 ]
2810 );
2811 }
2812 #[test]
2813 fn independent_target_lanes_run_at_the_same_time() {
2814 let executor = BarrierExecutor {
2815 seen: Mutex::new(Vec::new()),
2816 barrier: Barrier::new(2),
2817 };
2818
2819 execute_concurrent_lanes(
2820 || execute_checked(&executor, lane_command("install the worker")).map(|_| ()),
2821 || execute_checked(&executor, lane_command("upload the checkpoint")).map(|_| ()),
2822 )
2823 .unwrap();
2824
2825 let mut seen = executor.seen.into_inner().unwrap();
2826 seen.sort();
2827 assert_eq!(seen, ["install the worker", "upload the checkpoint"]);
2828 }
2829 #[test]
2830 fn a_lane_failure_is_reported_in_lane_order_and_never_abandons_the_other_lane() {
2831 let reached = Mutex::new(Vec::new());
2832
2833 let error = execute_concurrent_lanes(
2836 || -> Result<()> {
2837 std::thread::sleep(Duration::from_millis(50));
2838 bail!("worker install failed")
2839 },
2840 || -> Result<()> {
2841 reached.lock().unwrap().push("second");
2842 bail!("checkpoint upload failed")
2843 },
2844 )
2845 .unwrap_err();
2846
2847 assert_eq!(error.to_string(), "worker install failed");
2848 assert_eq!(
2849 *reached.lock().unwrap(),
2850 ["second"],
2851 "a failing first lane must not cut the second one short"
2852 );
2853
2854 let error = execute_concurrent_lanes(
2855 || Ok(()),
2856 || -> Result<()> { bail!("checkpoint upload failed") },
2857 )
2858 .unwrap_err();
2859 assert_eq!(error.to_string(), "checkpoint upload failed");
2860 }
2861
2862 #[test]
2863 fn cross_harness_lanes_prove_overlap_with_handshake_channels() {
2864 let (provision_started_tx, provision_started_rx) = std::sync::mpsc::sync_channel::<()>(1);
2865 let (handoff_started_tx, handoff_started_rx) = std::sync::mpsc::sync_channel::<()>(1);
2866 let (provision_seen_handoff_tx, provision_seen_handoff_rx) =
2867 std::sync::mpsc::sync_channel::<()>(1);
2868 let (handoff_seen_provision_tx, handoff_seen_provision_rx) =
2869 std::sync::mpsc::sync_channel::<()>(1);
2870
2871 execute_joined_cross_harness_work(
2872 "provision",
2873 move |_cancellation| -> Result<()> {
2874 provision_started_tx
2875 .send(())
2876 .map_err(|error| anyhow::anyhow!("signal provisioning start: {error}"))?;
2877 handoff_started_rx
2878 .recv_timeout(Duration::from_secs(2))
2879 .map_err(|error| anyhow::anyhow!("wait for handoff start: {error}"))?;
2880 provision_seen_handoff_tx
2881 .send(())
2882 .map_err(|error| anyhow::anyhow!("signal provisioning overlap: {error}"))?;
2883 Ok(())
2884 },
2885 "handoff",
2886 move |_cancellation| -> Result<()> {
2887 handoff_started_tx
2888 .send(())
2889 .map_err(|error| anyhow::anyhow!("signal handoff start: {error}"))?;
2890 provision_started_rx
2891 .recv_timeout(Duration::from_secs(2))
2892 .map_err(|error| anyhow::anyhow!("wait for provisioning start: {error}"))?;
2893 handoff_seen_provision_tx
2894 .send(())
2895 .map_err(|error| anyhow::anyhow!("signal handoff overlap: {error}"))?;
2896 Ok(())
2897 },
2898 )
2899 .unwrap();
2900
2901 assert!(provision_seen_handoff_rx.recv().is_ok());
2902 assert!(handoff_seen_provision_rx.recv().is_ok());
2903 }
2904
2905 #[test]
2906 fn cross_harness_lane_failure_cancels_and_joins_the_peer() {
2907 let (handoff_started_tx, handoff_started_rx) = std::sync::mpsc::sync_channel::<()>(1);
2908 let (handoff_joined_tx, handoff_joined_rx) = std::sync::mpsc::sync_channel::<()>(1);
2909
2910 let error = execute_joined_cross_harness_work(
2911 "provision",
2912 move |_cancellation| -> Result<()> {
2913 handoff_started_rx
2914 .recv_timeout(Duration::from_secs(2))
2915 .map_err(|error| anyhow::anyhow!("wait for handoff start: {error}"))?;
2916 bail!("provisioning failed after handoff started");
2917 },
2918 "handoff",
2919 move |cancellation| -> Result<()> {
2920 handoff_started_tx
2921 .send(())
2922 .map_err(|error| anyhow::anyhow!("signal handoff start: {error}"))?;
2923 let runtime = tokio::runtime::Builder::new_current_thread()
2924 .enable_all()
2925 .build()
2926 .map_err(|error| {
2927 anyhow::anyhow!("create cancellation test runtime: {error}")
2928 })?;
2929 runtime
2930 .block_on(async {
2931 tokio::time::timeout(Duration::from_secs(2), cancellation.cancelled()).await
2932 })
2933 .map_err(|error| anyhow::anyhow!("peer was not cancelled: {error}"))?;
2934 handoff_joined_tx
2935 .send(())
2936 .map_err(|error| anyhow::anyhow!("signal handoff join: {error}"))?;
2937 Ok(())
2938 },
2939 )
2940 .unwrap_err();
2941
2942 assert_eq!(
2943 error.to_string(),
2944 "provisioning failed after handoff started"
2945 );
2946 assert!(handoff_joined_rx.recv().is_ok());
2947 }
2948
2949 #[test]
2950 fn a_projection_standing_at_the_archived_frontier_is_reused() {
2951 let digest = "a".repeat(64);
2952 let other = "b".repeat(64);
2953
2954 assert!(!projection_rebuild_required(
2955 Some((82_000, &digest)),
2956 82_000,
2957 &digest
2958 ));
2959
2960 for stored in [
2961 Some((82_000, other.as_str())),
2963 Some((81_999, digest.as_str())),
2965 Some((82_001, digest.as_str())),
2966 None,
2968 ] {
2969 assert!(
2970 projection_rebuild_required(stored, 82_000, &digest),
2971 "{stored:?} must not be mistaken for the archived projection"
2972 );
2973 }
2974 }
2975
2976 #[test]
2977 fn local_bare_restore_reuses_verified_absolute_archive_without_upload() {
2978 let archive = Path::new("/var/lib/hel/archives/session.hel.zip");
2979 let remote = Path::new("/var/lib/hel/workers/session/restore.hel.zip");
2980 let local = targets::TargetLocator::LocalBare {
2981 worker_root: "/var/lib/hel/workers/session".into(),
2982 };
2983 let container = targets::TargetLocator::LocalPodman {
2984 container_id: "container".into(),
2985 workspace_storage: Default::default(),
2986 };
2987
2988 assert_eq!(restore_archive_path(&local, archive, remote), archive);
2989 assert!(!should_upload_restore_archive(&local));
2990 assert_eq!(restore_archive_path(&container, archive, remote), remote);
2991 assert!(should_upload_restore_archive(&container));
2992 }
2993
2994 #[test]
2995 fn cross_harness_provision_cancellation_stops_the_next_command() {
2996 struct RecordingExecutor {
2997 commands: Mutex<Vec<String>>,
2998 }
2999
3000 impl CommandExecutor for RecordingExecutor {
3001 fn execute(&self, command: &CommandSpec) -> Result<CommandOutput> {
3002 self.commands.lock().unwrap().push(command.purpose.clone());
3003 Ok(CommandOutput {
3004 status: 0,
3005 stdout: Vec::new(),
3006 stderr: Vec::new(),
3007 })
3008 }
3009 }
3010
3011 let inner = RecordingExecutor {
3012 commands: Mutex::new(Vec::new()),
3013 };
3014 let cancellation = CancellationToken::new();
3015 let provision = CrossHarnessProvisionExecutor {
3016 inner: &inner,
3017 cancellation: cancellation.clone(),
3018 };
3019 let command = CommandSpec::new("hel", ["worker"]).purpose("provision target");
3020 provision.execute(&command).unwrap();
3021 cancellation.cancel();
3022
3023 let error = provision.execute(&command).unwrap_err();
3024 assert!(error.to_string().contains("cancelled while provisioning"));
3025 assert_eq!(
3026 inner.commands.lock().unwrap().as_slice(),
3027 ["provision target"]
3028 );
3029 }
3030
3031 #[test]
3032 fn failed_resume_rolls_back_only_after_target_cleanup() {
3033 let previous = SessionRecord {
3034 mjolnir_subagents: None,
3035 create_managed_worktree: None,
3036 workspace_id: mj_core::workspace::DEFAULT_WORKSPACE_ID.to_owned(),
3037 archived: false,
3038 container_cpus: None,
3039 container_memory: None,
3040 id: "0123456789abcdef0123456789abcdef".into(),
3041 title: "imported session".into(),
3042 harness_kind: mj_core::config::HarnessKind::Codex,
3043 last_profile: "codex-old".into(),
3044 bundle_id: "project".into(),
3045 project_directory: None,
3046 managed_worktree: None,
3047 target_template_id: "podman-old".into(),
3048 resource_allocation: None,
3049 additional_mounts: Vec::new(),
3050 state: SessionState::Stopped,
3051 target: None,
3052 native_session_id: Some("native-session".into()),
3053 acp_session_title: None,
3054 session_title_override: None,
3055 created_at: "2026-08-12T00:00:00Z".into(),
3056 updated_at: "2026-08-12T00:00:00Z".into(),
3057 viewed_through_event_ordinal: 0,
3058 draft_input: String::new(),
3059 last_error: None,
3060 last_checkpoint_error: None,
3061 checkpoint: None,
3062 };
3063 let partial_target = TargetLocator::LocalPodman {
3064 container_id: "partial-container".into(),
3065 workspace_storage: Default::default(),
3066 };
3067 let mut cleaned = previous.clone();
3068 cleaned.state = SessionState::Error;
3069 cleaned.last_profile = "codex-new".into();
3070 cleaned.target = Some(partial_target.clone());
3071
3072 let failure =
3073 apply_failed_resume_rollback(&mut cleaned, &previous, "worker upload failed", None);
3074
3075 assert_eq!(cleaned.state, SessionState::Stopped);
3076 assert_eq!(cleaned.last_profile, "codex-old");
3077 assert_eq!(cleaned.target, None);
3078 assert_eq!(failure.to_string(), "worker upload failed");
3079 assert_eq!(
3080 cleaned.last_error.as_deref(),
3081 Some("resume failed: worker upload failed")
3082 );
3083
3084 let mut cleanup_failed = previous.clone();
3085 cleanup_failed.state = SessionState::Error;
3086 cleanup_failed.last_profile = "codex-new".into();
3087 cleanup_failed.target = Some(partial_target.clone());
3088 let partial_checkout = crate::controller::test_support::managed_raw_session(
3089 mj_core::state::ManagedWorktreeTarget::Local,
3090 );
3091 cleanup_failed.project_directory = partial_checkout.project_directory.clone();
3092 cleanup_failed.managed_worktree = partial_checkout.managed_worktree.clone();
3093
3094 let failure = apply_failed_resume_rollback(
3095 &mut cleanup_failed,
3096 &previous,
3097 "worker upload failed",
3098 Some("podman rm failed".into()),
3099 );
3100
3101 assert_eq!(cleanup_failed.state, SessionState::Error);
3102 assert_eq!(cleanup_failed.last_profile, "codex-new");
3103 assert_eq!(cleanup_failed.target, Some(partial_target));
3104 assert_eq!(
3105 cleanup_failed.project_directory,
3106 partial_checkout.project_directory
3107 );
3108 assert_eq!(
3109 cleanup_failed.managed_worktree,
3110 partial_checkout.managed_worktree
3111 );
3112 assert!(failure.to_string().contains("cleanup"));
3113 }
3114 #[test]
3115 fn failed_worktree_cleanup_notice_names_mjolnir_and_the_recovery_command() {
3116 let notice = worktree_cleanup_notice(
3117 Path::new("/workspace/project"),
3118 &anyhow::anyhow!("permission denied"),
3119 );
3120
3121 assert!(
3122 notice.starts_with(
3123 "Mjolnir could not remove the worktree at /workspace/project: permission denied."
3124 ),
3125 "{notice}"
3126 );
3127 assert!(
3128 notice.contains("`git worktree remove --force /workspace/project`"),
3129 "{notice}"
3130 );
3131 assert!(!notice.contains("Hel"), "{notice}");
3132 }
3133 #[test]
3134 fn failed_resume_provisioning_preserves_checkpoint_and_projection_lineage() {
3135 if std::env::var_os(RESUME_ROLLBACK_TEST_CHILD).is_none() {
3138 let directory = tempfile::tempdir().unwrap();
3139 let test_name = format!(
3140 "{}::failed_resume_provisioning_preserves_checkpoint_and_projection_lineage",
3141 module_path!()
3142 .strip_prefix("mj_controller::")
3143 .unwrap_or(module_path!())
3144 );
3145 let output = Command::new(std::env::current_exe().unwrap())
3146 .args(["--exact", &test_name, "--nocapture"])
3147 .env(RESUME_ROLLBACK_TEST_CHILD, "1")
3148 .env("MJ_WORKER_BINARY", std::env::current_exe().unwrap())
3151 .env("MJ_DATA_DIR", directory.path())
3152 .env("GH_TOKEN", "test-token")
3153 .output()
3154 .unwrap();
3155 assert!(
3156 output.status.success(),
3157 "isolated resume rollback test failed\nstdout:\n{}\nstderr:\n{}",
3158 String::from_utf8_lossy(&output.stdout),
3159 String::from_utf8_lossy(&output.stderr)
3160 );
3161 return;
3162 }
3163 let _writer = crate::database::install_isolated_test_writer();
3165
3166 #[derive(Default)]
3169 struct FailingPreflightExecutor {
3170 mounts_during_provisioning: Mutex<Option<Vec<AdditionalMount>>>,
3171 }
3172
3173 impl CommandExecutor for FailingPreflightExecutor {
3174 fn execute(&self, command: &CommandSpec) -> Result<CommandOutput> {
3175 if command.program == "stat" {
3178 return Ok(CommandOutput {
3179 status: 0,
3180 stdout: b"ext4\n".to_vec(),
3181 stderr: Vec::new(),
3182 });
3183 }
3184 assert_eq!(command.program, "podman");
3185 let mut observed = self.mounts_during_provisioning.lock().unwrap();
3186 if observed.is_none() {
3187 let durable = crate::database::load_state().unwrap();
3188 *observed = Some(
3189 durable.sessions["0123456789abcdef0123456789abcdef"]
3190 .additional_mounts
3191 .clone(),
3192 );
3193 }
3194 Ok(CommandOutput {
3195 status: 1,
3196 stdout: Vec::new(),
3197 stderr: b"podman is temporarily unavailable".to_vec(),
3198 })
3199 }
3200 }
3201
3202 let data_directory = PathBuf::from(std::env::var_os("MJ_DATA_DIR").unwrap());
3203 let archive_directory = data_directory.join("archives");
3204 std::fs::create_dir_all(&archive_directory).unwrap();
3205 let session_id = "0123456789abcdef0123456789abcdef";
3206 let checkpoint = super::super::test_support::write_network_checkpoint_archive(
3207 &archive_directory,
3208 session_id,
3209 7,
3210 );
3211 let archive = verify_archive_streaming(&checkpoint.archive_path).unwrap();
3212 let expected_projection =
3213 materialized_session_from_canonical(session_id, &archive.canonical_session).unwrap();
3214
3215 let mut session = checkpoint_test_session(session_id);
3216 session.state = SessionState::Stopped;
3217 session.checkpoint = Some(checkpoint.clone());
3218 session.additional_mounts = vec![AdditionalMount {
3219 source: PathBuf::from("/host/old"),
3220 destination: PathBuf::from("/mnt/old"),
3221 read_only: false,
3222 }];
3223 let previous = session.clone();
3224 let resumed_mounts = vec![AdditionalMount {
3225 source: PathBuf::from("/host/new"),
3226 destination: PathBuf::from("/mnt/new"),
3227 read_only: false,
3228 }];
3229 let profile_home = data_directory.join("profile");
3230 std::fs::create_dir_all(&profile_home).unwrap();
3231 let mut config = Config::default();
3232 config.profiles.insert(
3233 "codex".into(),
3234 HarnessProfile {
3235 enabled: true,
3236 kind: mj_core::config::HarnessKind::Codex,
3237 home: profile_home,
3238 environment: BTreeMap::new(),
3239 context_window_bytes: None,
3240 },
3241 );
3242 config.bundles.insert(
3243 "project".into(),
3244 ProjectBundle {
3245 primary_repo: "project".into(),
3246 repositories: vec![ProjectRepository {
3247 id: "project".into(),
3248 github: None,
3249 local: Some(data_directory.join("host-clone-that-no-longer-exists")),
3250 destination: "project".into(),
3251 git_ref: None,
3252 }],
3253 },
3254 );
3255 config.targets.insert(
3256 "podman".into(),
3257 TargetTemplate::LocalPodman {
3258 container: ConfigContainer {
3259 image: "example.invalid/hel-test:latest".into(),
3260 pull_policy: Default::default(),
3261 platform: None,
3262 cpus: None,
3263 memory: None,
3264 environment: BTreeMap::new(),
3265 workspace_storage: Default::default(),
3266 },
3267 },
3268 );
3269 let mut controller = Controller {
3270 config,
3271 state: State {
3272 sessions: BTreeMap::from([(session_id.into(), session)]),
3273 ..State::default()
3274 },
3275 };
3276 crate::database::save_state(&controller.state).unwrap();
3277 crate::database::save_materialized_session(&expected_projection).unwrap();
3278
3279 let runtime = tokio::runtime::Builder::new_current_thread()
3280 .enable_all()
3281 .build()
3282 .unwrap();
3283 let executor = FailingPreflightExecutor::default();
3284 let error = runtime
3285 .block_on(controller.resume_session_controlled(
3286 session_id,
3287 "codex",
3288 "podman",
3289 SessionResumeOptions {
3290 additional_mounts: Some(resumed_mounts.clone()),
3291 resource_allocation: None,
3292 discard_queue: false,
3293 },
3294 &executor,
3295 ))
3296 .unwrap_err();
3297 let detail = format!("{error:#}");
3298 assert!(
3299 detail.contains("podman is temporarily unavailable"),
3300 "{detail}"
3301 );
3302 assert!(!detail.contains("returned to stopped"), "{detail}");
3303 assert!(!detail.contains("unknown session"), "{detail}");
3304 assert_eq!(
3305 executor.mounts_during_provisioning.into_inner().unwrap(),
3306 Some(resumed_mounts)
3307 );
3308
3309 let retained = controller.state.sessions.get(session_id).unwrap();
3310 assert_eq!(retained.state, SessionState::Stopped);
3311 assert_eq!(retained.checkpoint, previous.checkpoint);
3312 assert_eq!(retained.managed_worktree, previous.managed_worktree);
3313 assert!(checkpoint.archive_path.is_file());
3314
3315 let durable = crate::database::load_state().unwrap();
3316 let durable_session = durable.sessions.get(session_id).unwrap();
3317 assert_eq!(durable_session.state, SessionState::Stopped);
3318 assert_eq!(durable_session.checkpoint, previous.checkpoint);
3319 assert_eq!(
3320 durable_session.additional_mounts,
3321 previous.additional_mounts
3322 );
3323 assert_eq!(
3324 crate::database::load_materialized_session(session_id).unwrap(),
3325 Some(expected_projection)
3326 );
3327 }
3328 #[test]
3329 fn failed_resume_retires_a_checkout_it_recreated() {
3330 if std::env::var_os(RETIRED_WORKTREE_RESUME_TEST_CHILD).is_none() {
3331 let directory = tempfile::tempdir().unwrap();
3332 let test_name = format!(
3333 "{}::failed_resume_retires_a_checkout_it_recreated",
3334 module_path!()
3335 .strip_prefix("mj_controller::")
3336 .unwrap_or(module_path!())
3337 );
3338 let output = Command::new(std::env::current_exe().unwrap())
3339 .args(["--exact", &test_name, "--nocapture"])
3340 .env(RETIRED_WORKTREE_RESUME_TEST_CHILD, "1")
3341 .env("MJ_WORKER_BINARY", std::env::current_exe().unwrap())
3344 .env("MJ_DATA_DIR", directory.path().join("data"))
3345 .env("MJ_CONFIG_DIR", directory.path().join("config"))
3346 .output()
3347 .unwrap();
3348 assert!(
3349 output.status.success(),
3350 "isolated retired-worktree resume test failed\nstdout:\n{}\nstderr:\n{}",
3351 String::from_utf8_lossy(&output.stdout),
3352 String::from_utf8_lossy(&output.stderr)
3353 );
3354 return;
3355 }
3356 let _writer = crate::database::install_isolated_test_writer();
3358
3359 struct FailAfterWorktreeRestore;
3360
3361 impl CommandExecutor for FailAfterWorktreeRestore {
3362 fn execute(&self, command: &CommandSpec) -> Result<CommandOutput> {
3363 if matches!(command.program.as_str(), "git" | "mkdir") {
3364 return ProcessExecutor.execute(command);
3365 }
3366 Ok(CommandOutput {
3367 status: 1,
3368 stdout: Vec::new(),
3369 stderr: b"stop after recreating the checkout".to_vec(),
3370 })
3371 }
3372 }
3373
3374 let data_directory = PathBuf::from(std::env::var_os("MJ_DATA_DIR").unwrap());
3375 let archive_directory = data_directory.join("archives");
3376 std::fs::create_dir_all(&archive_directory).unwrap();
3377 let session_id = "0123456789abcdef0123456789abcdef";
3378 let checkpoint = write_checkpoint_gate_archive(&archive_directory, session_id, 7);
3379 let repository = committed_repository();
3380 let mut session = managed_worktree_session(repository.path(), session_id);
3381 session.checkpoint = Some(checkpoint);
3382 let worktree = session.managed_worktree.clone().unwrap();
3383 retire_managed_worktree(&ProcessExecutor, &worktree).unwrap();
3384 assert!(!worktree.worktree_root.exists());
3385
3386 let profile_home = data_directory.join("profile");
3387 std::fs::create_dir_all(&profile_home).unwrap();
3388 let mut config = resume_compatibility_config();
3389 config.profiles.insert(
3390 "codex".into(),
3391 HarnessProfile {
3392 enabled: true,
3393 kind: mj_core::config::HarnessKind::Codex,
3394 home: profile_home,
3395 environment: BTreeMap::new(),
3396 context_window_bytes: None,
3397 },
3398 );
3399 let mut controller = Controller {
3400 config,
3401 state: State {
3402 sessions: BTreeMap::from([(session_id.into(), session)]),
3403 ..State::default()
3404 },
3405 };
3406 crate::database::save_state(&controller.state).unwrap();
3407
3408 let error = tokio::runtime::Builder::new_current_thread()
3409 .enable_all()
3410 .build()
3411 .unwrap()
3412 .block_on(controller.resume_session_controlled(
3413 session_id,
3414 "codex",
3415 "local-bare",
3416 SessionResumeOptions {
3417 additional_mounts: None,
3418 resource_allocation: None,
3419 discard_queue: false,
3420 },
3421 &FailAfterWorktreeRestore,
3422 ))
3423 .unwrap_err();
3424
3425 assert!(
3426 format!("{error:#}").contains("stop after recreating the checkout"),
3427 "{error:#}"
3428 );
3429 assert!(!worktree.worktree_root.exists());
3430 assert_eq!(
3431 controller.state.sessions[session_id].state,
3432 SessionState::Stopped
3433 );
3434 let branch = Command::new("git")
3435 .arg("-C")
3436 .arg(repository.path())
3437 .args([
3438 "show-ref",
3439 "--verify",
3440 &format!("refs/heads/{}", worktree.branch),
3441 ])
3442 .status()
3443 .unwrap();
3444 assert!(branch.success(), "resume rollback must retain the branch");
3445 }
3446 #[test]
3447 fn a_conversion_archive_carries_the_checkouts_remote_and_the_conversation() {
3448 let directory = tempfile::tempdir().unwrap();
3449 let session_id = "0123456789abcdef0123456789abcdef";
3450 let previous = write_checkpoint_archive_with_native_state(directory.path(), session_id, 7);
3451 let (checkout, _remote_parent, _remote) = checkout_with_network_remote();
3452 let source =
3453 mj_core::remote_git::resolve_local_repository(checkout.path(), &ProcessExecutor)
3454 .unwrap();
3455 let dirname = PathBuf::from(checkout.path().file_name().unwrap());
3456 let snapshot =
3457 raw_checkout_snapshot(checkout.path(), &source, &dirname, &SystemGit).unwrap();
3458
3459 let output = directory.path().join("converted.hel.zip");
3460 let converted = conversion_checkpoint(&previous.archive_path, snapshot, &output).unwrap();
3461
3462 let verified = mj_checkpoint::archive::read_archive_verified(&output).unwrap();
3465 assert_eq!(converted.archive_path, output);
3466 assert_eq!(converted.sha256, verified.archive_sha256);
3467 assert_eq!(converted.event_frontier, previous.event_frontier);
3468 let bundle =
3469 crate::controller::network_git::bundle_from_manifest(&verified.manifest).unwrap();
3470 assert_eq!(bundle.primary, dirname.to_string_lossy());
3471 assert_eq!(bundle.repositories.len(), 1);
3472 assert_eq!(
3473 bundle.repositories[0].url.as_deref(),
3474 Some(FIXTURE_FETCH_URL)
3475 );
3476 assert_eq!(bundle.repositories[0].push_urls, [FIXTURE_FETCH_URL]);
3477 assert_eq!(
3478 bundle.repositories[0].destination,
3479 dirname.to_string_lossy()
3480 );
3481
3482 let original =
3484 mj_checkpoint::archive::read_archive_verified(&previous.archive_path).unwrap();
3485 assert_eq!(
3486 verified.canonical_session().unwrap(),
3487 original.canonical_session().unwrap()
3488 );
3489 assert_eq!(verified.manifest.session, original.manifest.session);
3490 assert_eq!(native_state(&original), native_state(&verified));
3491 assert!(
3492 !native_state(&verified).is_empty(),
3493 "the fixture has native state"
3494 );
3495 }
3496
3497 fn native_state(
3499 archive: &mj_checkpoint::archive::VerifiedArchive,
3500 ) -> Vec<(PathBuf, u32, Vec<u8>)> {
3501 archive
3502 .manifest
3503 .payloads
3504 .iter()
3505 .filter_map(|descriptor| match &descriptor.role {
3506 mj_checkpoint::archive::PayloadRole::NativeArtifact { relative_path } => Some((
3507 relative_path.clone(),
3508 descriptor.mode,
3509 archive.payload(descriptor).unwrap().to_vec(),
3510 )),
3511 _ => None,
3512 })
3513 .collect()
3514 }
3515 const RAW_CONVERSION_TEST_CHILD: &str = "MJ_RAW_CONVERSION_TEST_CHILD";
3516 #[test]
3517 fn a_failed_raw_conversion_keeps_the_checkout_and_its_previous_checkpoint() {
3518 if std::env::var_os(RAW_CONVERSION_TEST_CHILD).is_none() {
3521 let directory = tempfile::tempdir().unwrap();
3522 let test_name = format!(
3523 "{}::a_failed_raw_conversion_keeps_the_checkout_and_its_previous_checkpoint",
3524 module_path!()
3525 .strip_prefix("mj_controller::")
3526 .unwrap_or(module_path!())
3527 );
3528 let output = Command::new(std::env::current_exe().unwrap())
3529 .args(["--exact", &test_name, "--nocapture"])
3530 .env(RAW_CONVERSION_TEST_CHILD, "1")
3531 .env("MJ_WORKER_BINARY", std::env::current_exe().unwrap())
3534 .env("MJ_DATA_DIR", directory.path().join("data"))
3535 .env("MJ_CONFIG_DIR", directory.path().join("config"))
3536 .env("GH_TOKEN", "test-token")
3537 .output()
3538 .unwrap();
3539 assert!(
3540 output.status.success(),
3541 "isolated raw conversion test failed\nstdout:\n{}\nstderr:\n{}",
3542 String::from_utf8_lossy(&output.stdout),
3543 String::from_utf8_lossy(&output.stderr)
3544 );
3545 return;
3546 }
3547 let _writer = crate::database::install_isolated_test_writer();
3549
3550 struct GitWithoutPodmanExecutor;
3553
3554 impl CommandExecutor for GitWithoutPodmanExecutor {
3555 fn execute(&self, command: &CommandSpec) -> Result<CommandOutput> {
3556 if command.program == "git" {
3557 return ProcessExecutor.execute(command);
3558 }
3559 Ok(CommandOutput {
3560 status: 1,
3561 stdout: Vec::new(),
3562 stderr: b"podman is temporarily unavailable".to_vec(),
3563 })
3564 }
3565 }
3566
3567 let data_directory = PathBuf::from(std::env::var_os("MJ_DATA_DIR").unwrap());
3568 let archive_directory = data_directory.join("archives");
3569 std::fs::create_dir_all(&archive_directory).unwrap();
3570 std::fs::create_dir_all(mj_core::config::config_dir()).unwrap();
3571 let session_id = "0123456789abcdef0123456789abcdef";
3572 let checkpoint = write_checkpoint_gate_archive(&archive_directory, session_id, 7);
3573
3574 let repository = committed_repository();
3575 let (_remote_parent, _remote) = network_remote_for(repository.path());
3578 let mut session = managed_worktree_session(repository.path(), session_id);
3579 session.checkpoint = Some(checkpoint.clone());
3580 let worktree = session.managed_worktree.clone().unwrap();
3581 let previous = session.clone();
3582
3583 let profile_home = data_directory.join("profile");
3584 std::fs::create_dir_all(&profile_home).unwrap();
3585 let mut config = resume_compatibility_config();
3586 config.profiles.insert(
3587 "codex".into(),
3588 HarnessProfile {
3589 enabled: true,
3590 kind: mj_core::config::HarnessKind::Codex,
3591 home: profile_home,
3592 environment: BTreeMap::new(),
3593 context_window_bytes: None,
3594 },
3595 );
3596 config.save().unwrap();
3599 let original_config = config.clone();
3600 let mut controller = Controller {
3601 config,
3602 state: State {
3603 sessions: BTreeMap::from([(session_id.into(), session)]),
3604 ..State::default()
3605 },
3606 };
3607 crate::database::save_state(&controller.state).unwrap();
3608
3609 let error = tokio::runtime::Builder::new_current_thread()
3610 .enable_all()
3611 .build()
3612 .unwrap()
3613 .block_on(controller.resume_session_controlled(
3614 session_id,
3615 "codex",
3616 "podman",
3617 SessionResumeOptions {
3618 additional_mounts: None,
3619 resource_allocation: None,
3620 discard_queue: false,
3621 },
3622 &GitWithoutPodmanExecutor,
3623 ))
3624 .unwrap_err();
3625 assert!(
3628 format!("{error:#}").contains("podman is temporarily unavailable"),
3629 "{error:#}"
3630 );
3631 assert!(!format!("{error:#}").contains("returned to stopped"));
3632
3633 let mut expected_config = original_config.clone();
3636 let (bundle_id, bundle) = mj_core::config::Config::load()
3637 .unwrap()
3638 .bundles
3639 .into_iter()
3640 .next()
3641 .expect("the conversion installed a bundle for the checkout");
3642 expected_config.bundles.insert(bundle_id, bundle);
3643 assert_eq!(controller.config, expected_config);
3644 assert_eq!(mj_core::config::Config::load().unwrap(), expected_config);
3645
3646 let retained = controller.state.sessions.get(session_id).unwrap();
3647 assert_eq!(retained.state, SessionState::Stopped);
3648 assert_eq!(retained.checkpoint, Some(checkpoint.clone()));
3649 assert_eq!(retained.project_directory, previous.project_directory);
3650 assert_eq!(retained.managed_worktree, previous.managed_worktree);
3651 assert_eq!(retained.bundle_id, previous.bundle_id);
3652 assert!(worktree.worktree_root.is_dir(), "the checkout stays put");
3653 assert!(
3654 checkpoint.archive_path.is_file(),
3655 "the previous archive is what the rolled-back record names"
3656 );
3657 let durable = crate::database::load_state().unwrap();
3658 assert_eq!(durable.sessions[session_id].checkpoint, Some(checkpoint));
3659 let sessions = mj_core::config::sessions_dir();
3662 assert!(sessions.is_dir(), "the conversion wrote an archive");
3663 let leftover: Vec<_> = std::fs::read_dir(&sessions)
3664 .map(|entries| {
3665 entries
3666 .map(|entry| entry.unwrap().file_name())
3667 .filter(|name| name.to_string_lossy().ends_with(".hel.zip"))
3668 .collect()
3669 })
3670 .unwrap_or_default();
3671 assert!(
3672 leftover.is_empty(),
3673 "{leftover:?} in {}",
3674 sessions.display()
3675 );
3676 }
3677
3678 #[test]
3682 fn only_a_harness_that_captures_its_native_session_keeps_continuity_on_resume() {
3683 use mj_core::config::HarnessKind;
3684
3685 assert!(!super::native_continuity_preserved(
3686 HarnessKind::Zcode,
3687 HarnessKind::Zcode
3688 ));
3689 assert!(super::native_continuity_preserved(
3690 HarnessKind::Codex,
3691 HarnessKind::Codex
3692 ));
3693 assert!(super::native_continuity_preserved(
3694 HarnessKind::Claude,
3695 HarnessKind::Claude
3696 ));
3697 assert!(!super::native_continuity_preserved(
3700 HarnessKind::Claude,
3701 HarnessKind::Codex
3702 ));
3703 assert!(!super::native_continuity_preserved(
3704 HarnessKind::Zcode,
3705 HarnessKind::Codex
3706 ));
3707 }
3708}