1use std::path::{Path, PathBuf};
34use std::sync::atomic::Ordering;
35use std::sync::Arc;
36use std::time::Duration;
37
38use serde::{Deserialize, Serialize};
39
40use super::registry::{self, DaemonLock, DaemonPaths};
41use super::DaemonState;
42use crate::session::wire::{ClientFrame, CHECKPOINT_QUERY_ID, PROTOCOL_VERSION};
43use crate::session::{
44 CheckpointReason, SessionCommand, SessionConfig, SessionEventWire, SessionHandle,
45 SessionLifecycle, SessionReloadRecord, SessionSetting,
46};
47
48pub const RELOAD_STATE_ENV: &str = "SYNAPS_DAEMON_RELOAD_STATE";
50pub const LOCK_FD_ENV: &str = "SYNAPS_DAEMON_LOCK_FD";
52pub const SUBAGENT_ID_FLOOR_ENV: &str = "SYNAPS_DAEMON_SUBAGENT_ID_FLOOR";
56pub const HANDOFF_EVENT_ENV: &str = "SYNAPS_DAEMON_HANDOFF_EVENT";
59pub const HANDOFF_PID_ENV: &str = "SYNAPS_DAEMON_HANDOFF_PID";
62#[cfg(windows)]
65const HANDOFF_ARM_TIMEOUT: Duration = Duration::from_secs(15);
66#[cfg(windows)]
68const HANDOFF_EXIT_TIMEOUT: Duration = Duration::from_secs(30);
69pub const DRAIN_SECS_ENV: &str = "SYNAPS_DAEMON_RELOAD_DRAIN_SECS";
71const DEFAULT_DRAIN: Duration = Duration::from_secs(30);
72const VERSION_PROBE_TIMEOUT: Duration = Duration::from_secs(5);
73const CHECKPOINT_BUDGET: Duration =
77 Duration::from_secs(crate::session::budgets::RELOAD_CHECKPOINT_BUDGET_SECS);
78pub const RETRY_AFTER_MS: u64 = 500;
80
81#[derive(Debug, Clone, Default)]
82pub struct ReloadRequest {
83 pub now: bool,
84 pub drain_secs: Option<u64>,
85 pub exe: Option<PathBuf>,
86}
87
88impl ReloadRequest {
89 pub fn from_frame(frame: &ClientFrame) -> Option<Self> {
90 match frame {
91 ClientFrame::Reload { now, drain_secs, exe } => {
92 Some(Self { now: *now, drain_secs: *drain_secs, exe: exe.clone() })
93 }
94 _ => None,
95 }
96 }
97}
98
99#[derive(Debug)]
100pub enum ReloadError {
101 Refused(String),
103 ExecFailed(String),
105}
106
107impl std::fmt::Display for ReloadError {
108 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
109 match self {
110 Self::Refused(w) => write!(f, "reload refused: {w}"),
111 Self::ExecFailed(e) => write!(f, "reload exec failed: {e}"),
112 }
113 }
114}
115
116#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
118pub struct PrintVersion {
119 pub binary_version: String,
120 pub protocol_version: u32,
121}
122
123impl PrintVersion {
124 pub fn current() -> Self {
125 Self {
126 binary_version: crate::session::wire::binary_version(),
127 protocol_version: PROTOCOL_VERSION,
128 }
129 }
130}
131
132#[derive(Debug, Clone, Serialize, Deserialize)]
134pub struct ReloadSession {
135 pub id: String,
136 pub journal_id: String,
137 pub config: SessionConfig,
138 pub keep_warm: bool,
139 pub lifecycle: SessionLifecycle,
140 #[serde(default)]
142 pub settings_replay: Vec<SessionSetting>,
143 #[serde(default)]
145 pub cache_keepwarm: Option<String>,
146 pub input_owner_kind: Option<crate::session::ClientKind>,
147 #[serde(default)]
150 pub workers: Vec<crate::runtime::subagent::CarriedWorker>,
151 #[serde(default)]
155 pub tool_activations: Vec<crate::tools::activation::CarriedActivation>,
156}
157
158#[derive(Debug, Clone, Serialize, Deserialize)]
160pub struct ReloadState {
161 pub generation: u64,
162 pub written_at: chrono::DateTime<chrono::Utc>,
163 pub sessions: Vec<ReloadSession>,
164 pub expected_clients: usize,
165 #[serde(default)]
170 pub next_subagent_id: u64,
171}
172
173impl ReloadState {
174 pub fn subagent_id_floor(&self) -> u64 {
177 let carried = self
178 .sessions
179 .iter()
180 .flat_map(|s| &s.workers)
181 .filter_map(|w| crate::runtime::subagent::handle_number(&w.id))
182 .max()
183 .map_or(0, |n| n + 1);
184 self.next_subagent_id.max(carried)
185 }
186}
187
188pub fn reload_state_path(paths: &DaemonPaths) -> PathBuf {
189 let stem = paths
190 .json
191 .file_stem()
192 .map(|s| s.to_string_lossy().into_owned())
193 .unwrap_or_else(|| "daemon".into());
194 paths.dir.join(format!("{stem}.reload.json"))
195}
196
197pub fn resolve_exe() -> PathBuf {
201 if let Some(a0) = std::env::args_os().next() {
202 let p = PathBuf::from(&a0);
203 if p.components().count() > 1 {
204 if let Ok(c) = p.canonicalize() {
205 return c;
206 }
207 }
208 }
209 std::env::current_exe().unwrap_or_else(|_| PathBuf::from("synaps"))
210}
211
212fn semver_tuple(s: &str) -> Option<(u64, u64, u64)> {
213 let core = s.split(['-', '+']).next()?;
214 let mut it = core.split('.').map(|p| p.parse::<u64>().ok());
215 Some((it.next()??, it.next()??, it.next()??))
216}
217
218pub fn version_gate(current: &PrintVersion, new: &PrintVersion) -> Result<(), String> {
222 if new.protocol_version < current.protocol_version {
223 return Err(format!(
224 "new binary speaks protocol {} < {}: reconnecting clients would be refused",
225 new.protocol_version, current.protocol_version
226 ));
227 }
228 match (semver_tuple(¤t.binary_version), semver_tuple(&new.binary_version)) {
229 (Some(c), Some(n)) if n < c => Err(format!(
230 "new binary {} is older than the running {} (reload is newer-or-equal only)",
231 new.binary_version, current.binary_version
232 )),
233 (Some(_), Some(_)) => Ok(()),
234 _ => Err(format!(
235 "cannot compare versions {:?} vs {:?}",
236 current.binary_version, new.binary_version
237 )),
238 }
239}
240
241#[cfg(unix)]
247pub fn validate_exe(exe: &Path) -> Result<PathBuf, String> {
248 use std::os::unix::fs::{MetadataExt, PermissionsExt};
249 let canon = exe
250 .canonicalize()
251 .map_err(|e| format!("--exe {}: {e}", exe.display()))?;
252 let md = std::fs::metadata(&canon).map_err(|e| format!("--exe {}: {e}", canon.display()))?;
253 if !md.is_file() {
254 return Err(format!("--exe {}: not a regular file", canon.display()));
255 }
256 let mode = md.permissions().mode();
257 if mode & 0o111 == 0 {
258 return Err(format!("--exe {}: not executable", canon.display()));
259 }
260 let uid = unsafe { libc::getuid() };
262 if md.uid() != uid && md.uid() != 0 {
263 return Err(format!(
264 "--exe {}: owned by uid {} (expected {} or root)",
265 canon.display(),
266 md.uid(),
267 uid
268 ));
269 }
270 if mode & 0o022 != 0 {
271 return Err(format!(
272 "--exe {}: group/world-writable (mode {:o})",
273 canon.display(),
274 mode & 0o777
275 ));
276 }
277 Ok(canon)
278}
279
280#[cfg(not(unix))]
281pub fn validate_exe(exe: &Path) -> Result<PathBuf, String> {
282 exe.canonicalize().map_err(|e| format!("--exe {}: {e}", exe.display()))
283}
284
285pub async fn probe_version(exe: &Path) -> Result<PrintVersion, String> {
287 let out = tokio::time::timeout(
288 VERSION_PROBE_TIMEOUT,
289 tokio::process::Command::new(exe)
290 .args(["daemon", "--print-version"])
291 .env("SYNAPS_DAEMON", "1")
292 .stdin(std::process::Stdio::null())
293 .stderr(std::process::Stdio::piped())
294 .stdout(std::process::Stdio::piped())
295 .output(),
296 )
297 .await
298 .map_err(|_| format!("{} --print-version timed out", exe.display()))?
299 .map_err(|e| format!("cannot run {}: {e}", exe.display()))?;
300 if !out.status.success() {
301 return Err(format!(
302 "{} --print-version exited {:?}: {}",
303 exe.display(),
304 out.status.code(),
305 String::from_utf8_lossy(&out.stderr).trim()
306 ));
307 }
308 serde_json::from_slice(&out.stdout)
309 .map_err(|e| format!("{} --print-version: unparsable output ({e})", exe.display()))
310}
311
312async fn checkpoint_one(handle: SessionHandle) -> Option<SessionReloadRecord> {
315 let mut rx = handle.subscribe();
316 if handle
317 .send(SessionCommand::Checkpoint { reason: CheckpointReason::Reload })
318 .await
319 .is_err()
320 {
321 return None;
322 }
323 let wait = async {
324 loop {
325 match rx.recv().await {
326 Ok(env) => {
327 if let SessionEventWire::QueryResult { id, value } = env.event {
328 if id == CHECKPOINT_QUERY_ID {
329 return serde_json::from_value(value["record"].clone()).ok();
330 }
331 }
332 }
333 Err(tokio::sync::broadcast::error::RecvError::Lagged(_)) => continue,
334 Err(tokio::sync::broadcast::error::RecvError::Closed) => return None,
335 }
336 }
337 };
338 tokio::time::timeout(CHECKPOINT_BUDGET, wait).await.unwrap_or(None)
339}
340
341fn record(handle: &SessionHandle, rec: Option<SessionReloadRecord>) -> ReloadSession {
345 let meta = handle.meta();
346 let journal_id = handle.journal_id();
347 if rec.is_none() {
348 tracing::warn!(
349 session = %handle.id,
350 "reload: no checkpoint reply in time — this session's workers are not carried: \
351 no archive is guaranteed and their parent will not be woken"
352 );
353 }
354 let (config, keep_warm, lifecycle, settings_replay, cache_keepwarm, model, workers, tool_activations) = match rec {
355 Some(r) => (
356 r.config,
357 r.keep_warm,
358 r.lifecycle,
359 r.settings_replay,
360 r.cache_keepwarm,
361 r.model,
362 r.workers,
363 r.tool_activations,
364 ),
365 None => (
366 SessionConfig {
367 cwd: meta.cwd.clone(),
368 memory_mode: handle.memory_mode(),
371 env: None,
372 env_stripped: Vec::new(),
373 ..SessionConfig::default()
374 },
375 handle.pinned(),
378 handle.lifecycle(),
379 Vec::new(),
380 handle.cache_keepwarm_override(),
383 handle.view().model.clone(),
384 Vec::new(),
387 Vec::new(),
389 ),
390 };
391 ReloadSession {
392 id: meta.id.as_str().to_string(),
393 journal_id: journal_id.clone(),
394 config: SessionConfig {
395 continue_session: Some(Some(journal_id)),
396 model_override: Some(model),
397 keep_warm,
398 name: None,
401 ..config
402 },
403 keep_warm,
404 lifecycle,
405 settings_replay,
406 cache_keepwarm,
407 input_owner_kind: None,
408 workers,
409 tool_activations,
410 }
411}
412
413pub struct Prepared {
418 pub exe: PathBuf,
419 pub rs_path: PathBuf,
420 pub generation: u64,
421}
422
423pub async fn prepare(
424 state: &Arc<DaemonState>,
425 paths: &DaemonPaths,
426 req: ReloadRequest,
427) -> Result<Prepared, ReloadError> {
428 let exe = req.exe.clone().unwrap_or_else(|| state.daemon_info().exe.unwrap_or_else(resolve_exe));
432 let exe = validate_exe(&exe).map_err(ReloadError::Refused)?;
433 let new = probe_version(&exe).await.map_err(ReloadError::Refused)?;
434 version_gate(&PrintVersion::current(), &new).map_err(ReloadError::Refused)?;
435 if state.reloading.swap(true, Ordering::SeqCst) {
436 return Err(ReloadError::Refused("a reload is already in progress".into()));
437 }
438 let generation = state.generation + 1;
439 tracing::info!(exe = %exe.display(), new = %new.binary_version, generation, "daemon: reload accepted");
440
441 let drain = if req.now {
443 Duration::ZERO
444 } else {
445 req.drain_secs
446 .map(Duration::from_secs)
447 .or_else(|| std::env::var(DRAIN_SECS_ENV).ok()?.parse().ok().map(Duration::from_secs))
448 .unwrap_or(DEFAULT_DRAIN)
449 };
450 let t0 = std::time::Instant::now();
451 while t0.elapsed() < drain {
452 let mut all_idle = true;
453 for h in state.live_sessions() {
454 if !super::session_is_idle(&h).await {
455 all_idle = false;
456 break;
457 }
458 }
459 if all_idle {
460 break;
461 }
462 tokio::time::sleep(Duration::from_millis(200)).await;
463 }
464 let handles = state.live_sessions();
465 let results = futures::future::join_all(handles.iter().cloned().map(checkpoint_one)).await;
466 let mut sessions = Vec::with_capacity(handles.len());
467 for (h, rec) in handles.iter().zip(results) {
468 if rec.is_none() {
469 tracing::warn!(session = %h.id, "reload: checkpoint did not confirm; recording from the handle");
470 }
471 sessions.push(record(h, rec));
472 }
473
474 let rs = ReloadState {
476 generation,
477 written_at: chrono::Utc::now(),
478 sessions,
479 expected_clients: state.connections.load(Ordering::SeqCst),
480 next_subagent_id: crate::tools::peek_next_subagent_id(),
482 };
483 let rs_path = reload_state_path(paths);
484 let written = serde_json::to_vec_pretty(&rs)
489 .map_err(|e| e.to_string())
490 .and_then(|body| registry::write_private_atomic(&rs_path, &body).map_err(|e| e.to_string()));
491 if let Err(e) = written {
492 abort_checkpoints(state).await;
495 state.reloading.store(false, Ordering::SeqCst);
496 return Err(ReloadError::ExecFailed(e));
497 }
498
499 state.reload_generation.store(generation, Ordering::SeqCst);
502 state.fire_announce();
503 let t1 = std::time::Instant::now();
504 while state.connections.load(Ordering::SeqCst) > 1 && t1.elapsed() < Duration::from_secs(crate::session::budgets::RELOAD_ANNOUNCE_WAIT_SECS) {
505 tokio::time::sleep(Duration::from_millis(20)).await;
506 }
507 Ok(Prepared { exe, rs_path, generation })
508}
509
510pub async fn exec(state: &Arc<DaemonState>, p: Prepared) -> ReloadError {
519 let previous = state.daemon_info();
524 let prev_generation = previous.generation;
525 let info = registry::DaemonInfo {
526 generation: p.generation,
527 started_at: chrono::Utc::now(),
528 exe: Some(p.exe.clone()),
529 ..previous.clone()
530 };
531 if let Err(e) = state.meta.publish(info) {
532 tracing::warn!(path = %state.paths.json.display(), error = %e, "daemon: daemon.json write failed at reload");
533 }
534 let _ = tokio::time::timeout(
535 Duration::from_secs(crate::session::budgets::RELOAD_EXT_SHUTDOWN_SECS),
536 async { state.host.ext_manager().write().await.shutdown_all().await },
537 )
538 .await;
539
540 #[cfg(unix)]
541 let err = exec_self(&p.exe, &p.rs_path, state);
542 #[cfg(windows)]
543 let err = exec_successor(&p.exe, &p.rs_path, state).await;
544 tracing::error!(error = %err, "daemon: reload exec failed; continuing on the old image");
545 if let Err(e) = state.meta.publish(previous) {
548 tracing::warn!(
549 path = %state.paths.json.display(),
550 error = %e,
551 "daemon: daemon.json write failed after the failed reload exec"
552 );
553 }
554 let _ = std::fs::remove_file(&p.rs_path);
555 state.reload_generation.store(prev_generation, Ordering::SeqCst);
556 state.reset_announce();
557 let (tx, mut rx) = tokio::sync::mpsc::unbounded_channel();
560 let _loader = crate::extensions::loader::spawn_discover_and_load(
561 Arc::clone(state.host.ext_manager()),
562 tx,
563 None,
564 );
565 let wait = async {
566 while let Some(ev) = rx.recv().await {
567 if let crate::extensions::loader::ExtensionLoaderEvent::Finished { loaded, failed } = ev {
568 tracing::info!(loaded = loaded.len(), failed = failed.len(), "daemon: extensions re-discovered after failed reload");
569 break;
570 }
571 }
572 };
573 if tokio::time::timeout(
574 Duration::from_secs(crate::session::budgets::RELOAD_EXT_REDISCOVERY_SECS),
575 wait,
576 )
577 .await
578 .is_err()
579 {
580 tracing::warn!("daemon: extension re-discovery still running; resuming the sessions");
581 }
582 abort_checkpoints(state).await;
586 state.reloading.store(false, Ordering::SeqCst);
587 ReloadError::ExecFailed(err)
588}
589
590#[cfg(unix)]
591fn exec_self(exe: &Path, rs_path: &Path, state: &DaemonState) -> String {
592 use std::os::unix::process::CommandExt;
593 let Some(lock_fd) = state.lock_fd.lock().unwrap_or_else(|e| e.into_inner()).as_ref().map(|l| l.raw_fd()) else {
594 return "no lock fd to inherit".into();
595 };
596 unsafe {
601 let flags = libc::fcntl(lock_fd, libc::F_GETFD);
602 if flags >= 0 {
603 libc::fcntl(lock_fd, libc::F_SETFD, flags & !libc::FD_CLOEXEC);
604 }
605 }
606 let mut args: Vec<std::ffi::OsString> = std::env::args_os().skip(1).collect();
607 if args.is_empty() || args.iter().all(|a| a != "daemon") {
608 args = vec!["daemon".into(), "--foreground".into()];
609 }
610 let mut cmd = std::process::Command::new(exe);
611 cmd.args(&args)
612 .env("SYNAPS_DAEMON", "1")
613 .env(LOCK_FD_ENV, lock_fd.to_string())
614 .env(RELOAD_STATE_ENV, rs_path)
615 .env(SUBAGENT_ID_FLOOR_ENV, crate::tools::peek_next_subagent_id().to_string());
616 let e = cmd.exec();
617 unsafe {
620 let flags = libc::fcntl(lock_fd, libc::F_GETFD);
621 if flags >= 0 {
622 libc::fcntl(lock_fd, libc::F_SETFD, flags | libc::FD_CLOEXEC);
623 }
624 }
625 e.to_string()
626}
627
628#[cfg(windows)]
635async fn exec_successor(exe: &Path, rs_path: &Path, state: &DaemonState) -> String {
636 use super::win;
637 let armed = match win::Event::create_unique("handoff") {
638 Ok(e) => e,
639 Err(e) => return format!("hand-off event: {e}"),
640 };
641 let mut args: Vec<std::ffi::OsString> = std::env::args_os().skip(1).collect();
642 if args.is_empty() || args.iter().all(|a| a != "daemon") {
643 args = vec!["daemon".into(), "--foreground".into()];
644 }
645 let mut env = crate::process_scope::inherited_env();
646 env.retain(|(k, _)| k != super::READY_EVENT_ENV);
647 crate::process_scope::set_env(&mut env, "SYNAPS_DAEMON", "1");
648 crate::process_scope::set_env(&mut env, RELOAD_STATE_ENV, rs_path.as_os_str());
649 crate::process_scope::set_env(&mut env, HANDOFF_EVENT_ENV, armed.name());
650 crate::process_scope::set_env(&mut env, HANDOFF_PID_ENV, std::process::id().to_string());
651 crate::process_scope::set_env(&mut env, SUBAGENT_ID_FLOOR_ENV, crate::tools::peek_next_subagent_id().to_string());
652 let child = match win::spawn_detached(exe, &args, &env) {
653 Ok(c) => c,
654 Err(e) => return format!("cannot start {}: {e}", exe.display()),
655 };
656 let waited = tokio::task::spawn_blocking(move || {
657 let outcome = win::wait_event_or_exit(&armed, &child, HANDOFF_ARM_TIMEOUT);
658 (child, outcome)
659 })
660 .await;
661 let (child, outcome) = match waited {
662 Ok(r) => r,
663 Err(e) => return format!("hand-off wait: {e}"),
664 };
665 match outcome {
666 win::Wait::Signalled => {
667 tracing::info!(pid = child.id(), "daemon: reload hand-off armed; this image exits");
668 state.host.flush_logs();
669 std::process::exit(0);
670 }
671 win::Wait::Exited => {
672 let mut child = child;
673 let code = child.wait(Duration::ZERO);
674 let tail = win::stderr_tail(child.take_stderr(), 64 * 1024, Duration::from_millis(500));
675 let why = tail
676 .lines()
677 .map(str::trim)
678 .rfind(|l| !l.is_empty())
679 .map(|l| l.trim_start_matches("synaps daemon: ").to_string())
680 .unwrap_or_else(|| "no output".into());
681 format!(
682 "new image exited (code {}) before taking over: {why}",
683 code.map_or_else(|| "?".to_string(), |c| c.to_string())
684 )
685 }
686 win::Wait::TimedOut => {
687 child.kill();
688 let _ = child.wait(Duration::from_secs(5));
689 format!("new image did not take over within {HANDOFF_ARM_TIMEOUT:?}")
690 }
691 }
692}
693
694type Adopted = (DaemonLock, ReloadState, PathBuf);
696
697#[cfg(windows)]
700static TAKEN_OVER: std::sync::Mutex<Option<Adopted>> = std::sync::Mutex::new(None);
701
702#[cfg(windows)]
706pub fn take_over_from_env(paths: &DaemonPaths) -> anyhow::Result<()> {
707 if let Some(adopted) = handoff_from_env(paths)? {
708 *TAKEN_OVER.lock().unwrap_or_else(|e| e.into_inner()) = Some(adopted);
709 }
710 Ok(())
711}
712
713#[cfg(windows)]
716pub fn adopt_from_env(paths: &DaemonPaths) -> anyhow::Result<Option<Adopted>> {
717 if let Some(adopted) = TAKEN_OVER.lock().unwrap_or_else(|e| e.into_inner()).take() {
718 return Ok(Some(adopted));
719 }
720 handoff_from_env(paths)
721}
722
723#[cfg(windows)]
730fn handoff_from_env(paths: &DaemonPaths) -> anyhow::Result<Option<Adopted>> {
731 let Some(rs_path) = std::env::var_os(RELOAD_STATE_ENV) else {
732 return Ok(None);
733 };
734 let rs_path = PathBuf::from(rs_path);
735 std::env::remove_var(RELOAD_STATE_ENV);
736 let event = std::env::var(HANDOFF_EVENT_ENV).ok();
737 std::env::remove_var(HANDOFF_EVENT_ENV);
738 let old_pid = std::env::var(HANDOFF_PID_ENV).ok().and_then(|s| s.parse::<u32>().ok());
739 std::env::remove_var(HANDOFF_PID_ENV);
740 let id_floor = std::env::var(SUBAGENT_ID_FLOOR_ENV).ok().and_then(|s| s.parse::<u64>().ok());
741 std::env::remove_var(SUBAGENT_ID_FLOOR_ENV);
742 let event = event.ok_or_else(|| {
743 anyhow::anyhow!("{RELOAD_STATE_ENV} set without {HANDOFF_EVENT_ENV}: refusing to start (the lock is the liveness oracle)")
744 })?;
745 let body = std::fs::read_to_string(&rs_path)?;
746 let mut rs: ReloadState = serde_json::from_str(&body)?;
747 if let Some(floor) = id_floor {
748 rs.next_subagent_id = rs.next_subagent_id.max(floor);
749 }
750 let old = old_pid.and_then(super::win::Process::open);
752 let mut signal_err = None;
753 let lock = DaemonLock::acquire_handoff(paths, HANDOFF_EXIT_TIMEOUT, || {
754 signal_err = super::win::signal_event(&event).err();
755 })
756 .map_err(|e| match &signal_err {
757 Some(s) => anyhow::anyhow!("taking daemon.lock over: {e} (the previous image was never told: {s})"),
758 None => anyhow::anyhow!("taking daemon.lock over from the previous image: {e}"),
759 })?;
760 if let Some(old) = old {
761 if !old.wait(Duration::from_secs(10)) {
762 tracing::warn!(pid = old_pid, "daemon: the previous image still runs after releasing daemon.lock");
763 }
764 }
765 tracing::info!(generation = rs.generation, "daemon: took daemon.lock over from the previous image");
766 super::win::detach_stdio();
768 Ok(Some((lock, rs, rs_path)))
769}
770
771#[cfg(unix)]
775pub fn adopt_from_env(_paths: &DaemonPaths) -> anyhow::Result<Option<Adopted>> {
776 let Some(rs_path) = std::env::var_os(RELOAD_STATE_ENV) else {
777 return Ok(None);
778 };
779 let rs_path = PathBuf::from(rs_path);
780 std::env::remove_var(RELOAD_STATE_ENV);
781 let fd: i32 = std::env::var(LOCK_FD_ENV)
782 .ok()
783 .and_then(|s| s.parse().ok())
784 .ok_or_else(|| anyhow::anyhow!("{RELOAD_STATE_ENV} set without {LOCK_FD_ENV}: refusing to start (the flock is the liveness oracle)"))?;
785 std::env::remove_var(LOCK_FD_ENV);
786 let id_floor = std::env::var(SUBAGENT_ID_FLOOR_ENV).ok().and_then(|s| s.parse::<u64>().ok());
787 std::env::remove_var(SUBAGENT_ID_FLOOR_ENV);
788 let lock = DaemonLock::adopt(fd)?;
789 let body = std::fs::read_to_string(&rs_path)?;
790 let mut rs: ReloadState = serde_json::from_str(&body)?;
791 if let Some(floor) = id_floor {
792 rs.next_subagent_id = rs.next_subagent_id.max(floor);
793 }
794 Ok(Some((lock, rs, rs_path)))
795}
796
797pub(crate) fn is_lock_held_error(e: &str) -> bool {
809 e.contains("is live in another process")
810}
811
812pub(crate) fn parse_holder_from_error(e: &str) -> String {
814 if let Some(start) = e.find("(pid ") {
816 let after = &e[start + 1..]; if let Some(end) = after.find(')') {
818 return after[..end].to_string();
819 }
820 }
821 "unknown holder".to_string()
822}
823
824async fn abort_checkpoints(state: &Arc<DaemonState>) {
830 let budget = Duration::from_secs(crate::session::budgets::RELOAD_ABORT_SEND_SECS);
831 let sends = state
832 .live_sessions()
833 .into_iter()
834 .map(|h| async move { send_reload_aborted(&h, budget).await });
835 futures::future::join_all(sends).await;
836}
837
838pub(crate) async fn send_reload_aborted(h: &SessionHandle, budget: Duration) -> bool {
842 use crate::session::TransportError;
843 let deadline = tokio::time::Instant::now() + budget;
844 loop {
845 match h.send(SessionCommand::ReloadAborted).await {
846 Ok(()) => return true,
847 Err(TransportError::Backpressure) if tokio::time::Instant::now() < deadline => {
848 tokio::time::sleep(Duration::from_millis(20)).await;
849 }
850 Err(e) => {
851 tracing::warn!(
852 session = %h.id,
853 error = %e,
854 "reload: could not tell the session the exec failed; its reload fence lifts at the backstop"
855 );
856 return false;
857 }
858 }
859 }
860}
861
862fn restore_config(saved: &SessionConfig, lifecycle: SessionLifecycle) -> SessionConfig {
869 SessionConfig {
870 await_extensions: false,
871 restore_parked: matches!(lifecycle, SessionLifecycle::Parked | SessionLifecycle::Parking),
872 ..saved.clone()
873 }
874}
875
876pub async fn rehydrate(state: &Arc<DaemonState>, rs: &ReloadState) {
877 crate::tools::seed_next_subagent_id(rs.subagent_id_floor());
880 for s in &rs.sessions {
881 let config = restore_config(&s.config, s.lifecycle);
882 let created = match state.create(config.clone()).await {
883 Ok(h) => Ok(h),
884 Err(e) if is_lock_held_error(&e) => {
887 let holder = parse_holder_from_error(&e);
888 tracing::warn!(
889 session = %s.id, holder = %holder,
890 "daemon: journal locked by another process; registering Parked placeholder"
891 );
892 let sid = crate::session::SessionId::from(s.id.as_str());
893 let (handle, _task) = crate::session::handle::locked_placeholder::spawn(
894 sid,
895 s.config.clone(),
896 holder,
897 );
898 state.insert(handle);
899 continue;
900 }
901 Err(e) if s.config.continue_session.is_some() => {
902 tracing::warn!(session = %s.id, error = %e, "daemon: journal not continuable; recreating fresh");
903 state
904 .create(SessionConfig {
905 continue_session: None,
906 ..config.clone()
907 })
908 .await
909 }
910 Err(e) => Err(e),
911 };
912 match created {
913 Ok(h) => {
914 let new_id = h.id.as_str().to_string();
915 let continued = h.meta().continued;
921 if continued && !s.tool_activations.is_empty() {
922 let _ = h
923 .send(SessionCommand::RestoreToolActivations {
924 activations: s.tool_activations.clone(),
925 })
926 .await;
927 }
928 if new_id != s.id {
929 state
930 .reload_aliases
931 .lock()
932 .unwrap_or_else(|e| e.into_inner())
933 .insert(s.id.clone(), new_id.clone());
934 }
935 for (i, setting) in s.settings_replay.iter().enumerate() {
937 let _ = h
938 .send(SessionCommand::Set {
939 id: u64::MAX - i as u64,
940 setting: setting.clone(),
941 })
942 .await;
943 }
944 if s.keep_warm {
945 let _ = h.send(SessionCommand::KeepWarm { on: true }).await;
946 }
947 if let Some(mode) = &s.cache_keepwarm {
948 let _ = h
949 .send(SessionCommand::EngineCommand {
950 id: u64::MAX,
951 name: "keepwarm".into(),
952 arg: mode.clone(),
953 })
954 .await;
955 }
956 if !s.workers.is_empty() {
959 let _ = h
960 .send(SessionCommand::AdoptWorkers { workers: s.workers.clone() })
961 .await;
962 }
963 if matches!(s.lifecycle, SessionLifecycle::Parked | SessionLifecycle::Parking) {
964 let _ = h.send(SessionCommand::Park).await;
965 } else {
966 let _ = h.send(SessionCommand::ArmPark).await;
970 }
971 tracing::info!(old = %s.id, new = %new_id, lifecycle = ?s.lifecycle, "daemon: session rehydrated after reload");
972 }
973 Err(e) => tracing::warn!(session = %s.id, error = %e, "daemon: session could not be rehydrated"),
974 }
975 }
976}
977
978#[cfg(test)]
979mod tests {
980 use super::*;
981
982 #[test]
985 fn reload_session_carries_tool_activations() {
986 use crate::tools::activation::{CarriedActivation, CarriedProvenance};
987 let carried = vec![CarriedActivation {
988 tool_id: "builtin:subagent_start".into(),
989 schema_digest: "cd".repeat(32),
990 provenance: CarriedProvenance::BuiltinRuntime,
991 }];
992 let s = ReloadSession {
993 id: "s".into(),
994 journal_id: "j".into(),
995 config: SessionConfig::default(),
996 keep_warm: false,
997 lifecycle: SessionLifecycle::Live,
998 settings_replay: Vec::new(),
999 cache_keepwarm: None,
1000 input_owner_kind: None,
1001 workers: Vec::new(),
1002 tool_activations: carried.clone(),
1003 };
1004 let v = serde_json::to_value(&s).unwrap();
1005 let back: ReloadSession = serde_json::from_value(v.clone()).unwrap();
1006 assert_eq!(back.tool_activations, carried);
1007 assert!(v["config"].get("restore_tool_activations").is_none());
1009
1010 let mut older = v;
1011 older.as_object_mut().unwrap().remove("tool_activations");
1012 let back: ReloadSession = serde_json::from_value(older).unwrap();
1013 assert!(back.tool_activations.is_empty());
1014 }
1015
1016 #[test]
1019 fn reload_state_from_an_older_image_parses() {
1020 let rs: ReloadState = serde_json::from_value(serde_json::json!({
1021 "generation": 9,
1022 "written_at": "2026-10-14T18:17:03Z",
1023 "sessions": [],
1024 "expected_clients": 1
1025 }))
1026 .unwrap();
1027 assert_eq!(rs.next_subagent_id, 0);
1028 assert_eq!(rs.subagent_id_floor(), 0);
1029 }
1030
1031 #[test]
1034 fn subagent_id_floor_covers_the_counter_and_every_carried_id() {
1035 let worker = |id: &str| crate::runtime::subagent::CarriedWorker {
1036 id: id.into(),
1037 subagent_id: 0,
1038 agent: String::new(),
1039 status: "cancelled".into(),
1040 archive_path: None,
1041 stopped_by_reload: true,
1042 wake_lost: false,
1043 wake: true,
1044 elapsed_secs: 0,
1045 preview: String::new(),
1046 };
1047 let session = |workers| ReloadSession {
1048 id: "s".into(),
1049 journal_id: "j".into(),
1050 config: SessionConfig::default(),
1051 keep_warm: false,
1052 lifecycle: SessionLifecycle::Live,
1053 settings_replay: Vec::new(),
1054 cache_keepwarm: None,
1055 input_owner_kind: None,
1056 workers,
1057 tool_activations: Vec::new(),
1058 };
1059 let mut rs = ReloadState {
1060 generation: 10,
1061 written_at: chrono::Utc::now(),
1062 sessions: vec![session(vec![worker("sa_288"), worker("sa_286")]), session(vec![])],
1063 expected_clients: 0,
1064 next_subagent_id: 0,
1065 };
1066 assert_eq!(rs.subagent_id_floor(), 289);
1067 rs.next_subagent_id = 300;
1068 assert_eq!(rs.subagent_id_floor(), 300);
1069 let back: ReloadState = serde_json::from_slice(&serde_json::to_vec(&rs).unwrap()).unwrap();
1071 assert_eq!(back.subagent_id_floor(), 300);
1072 assert_eq!(back.sessions[0].workers.len(), 2);
1073 }
1074
1075 fn pv(b: &str, p: u32) -> PrintVersion {
1076 PrintVersion { binary_version: b.into(), protocol_version: p }
1077 }
1078
1079 #[test]
1080 fn gate_is_directional_newer_or_equal() {
1081 let cur = pv("0.9.0", 2);
1082 assert!(version_gate(&cur, &pv("0.9.0", 2)).is_ok(), "equal allowed");
1083 assert!(version_gate(&cur, &pv("0.9.1", 2)).is_ok());
1084 assert!(version_gate(&cur, &pv("1.0.0-rc1", 3)).is_ok());
1085 assert!(version_gate(&cur, &pv("0.8.9", 2)).is_err(), "older refused");
1086 assert!(version_gate(&cur, &pv("0.9.0", 1)).is_err(), "older protocol refused");
1087 assert!(version_gate(&cur, &pv("garbage", 2)).is_err());
1088 assert!(
1091 version_gate(&pv("0.9.1", 3), &pv("0.10.0", 3)).is_ok(),
1092 "0.10.0 is newer than 0.9.1"
1093 );
1094 assert!(
1095 version_gate(&pv("0.10.0", 3), &pv("0.9.1", 3)).is_err(),
1096 "0.9.1 is older than 0.10.0"
1097 );
1098 }
1099
1100 #[cfg(unix)]
1101 #[test]
1102 fn validate_exe_refuses_missing_dirs_non_exec_and_writable() {
1103 use std::os::unix::fs::PermissionsExt;
1104 let d = tempfile::tempdir().unwrap();
1105 assert!(validate_exe(&d.path().join("nope")).unwrap_err().contains("nope"));
1106 assert!(validate_exe(d.path()).unwrap_err().contains("not a regular file"));
1107 let f = d.path().join("bin");
1108 std::fs::write(&f, "#!/bin/sh\nexit 0\n").unwrap();
1109 std::fs::set_permissions(&f, std::fs::Permissions::from_mode(0o644)).unwrap();
1110 assert!(validate_exe(&f).unwrap_err().contains("not executable"));
1111 std::fs::set_permissions(&f, std::fs::Permissions::from_mode(0o777)).unwrap();
1112 assert!(validate_exe(&f).unwrap_err().contains("writable"));
1113 std::fs::set_permissions(&f, std::fs::Permissions::from_mode(0o755)).unwrap();
1114 assert_eq!(validate_exe(&f).unwrap(), f.canonicalize().unwrap());
1115 let l = d.path().join("link");
1117 std::os::unix::fs::symlink(&f, &l).unwrap();
1118 assert_eq!(validate_exe(&l).unwrap(), f.canonicalize().unwrap());
1119 }
1120
1121 #[test]
1125 fn a_restore_does_not_wait_on_the_session_start_hook() {
1126 let saved = SessionConfig {
1127 model_override: Some("claude-sonnet-4-5".into()),
1128 keep_warm: true,
1129 await_extensions: true,
1130 ..Default::default()
1131 };
1132 let restored = restore_config(&saved, SessionLifecycle::Live);
1133 assert!(!restored.await_extensions);
1134 assert!(!restored.restore_parked);
1135 let as_saved = SessionConfig {
1136 await_extensions: true,
1137 ..restored
1138 };
1139 assert_eq!(
1140 serde_json::to_value(&as_saved).unwrap(),
1141 serde_json::to_value(&saved).unwrap()
1142 );
1143 for parked in [SessionLifecycle::Parked, SessionLifecycle::Parking] {
1144 let restored = restore_config(&saved, parked);
1145 assert!(restored.restore_parked, "{parked:?}");
1146 assert!(!restored.await_extensions);
1147 let wire = serde_json::to_value(&restored).unwrap();
1148 assert!(wire.get("restore_parked").is_none(), "never on the wire");
1149 }
1150 }
1151
1152 #[test]
1153 fn reload_state_path_follows_profile() {
1154 let d = tempfile::tempdir().unwrap();
1155 let p = registry::daemon_paths_in(d.path(), Some("work"));
1156 assert!(reload_state_path(&p).ends_with("daemon-work.reload.json"), "{:?}", reload_state_path(&p));
1157 let p = registry::daemon_paths_in(d.path(), None);
1158 assert!(reload_state_path(&p).ends_with("daemon.reload.json"));
1159 }
1160
1161 #[test]
1165 fn keepwarm_override_survives_an_unconfirmed_checkpoint() {
1166 use crate::core::config::CacheKeepWarm;
1167 use crate::session::handle::echo::{meta_for, view_for};
1168 let id = crate::session::SessionId("kw-reload".into());
1169 let (handle, ep) = SessionHandle::new(meta_for(&id), view_for());
1170 assert_eq!(record(&handle, None).cache_keepwarm, None, "no override yet");
1171 *ep.cache_keepwarm.lock().unwrap() = Some(CacheKeepWarm::Off);
1173 assert_eq!(record(&handle, None).cache_keepwarm.as_deref(), Some("off"));
1174 *ep.cache_keepwarm.lock().unwrap() = Some(CacheKeepWarm::Hours(6));
1175 assert_eq!(record(&handle, None).cache_keepwarm.as_deref(), Some("6h"));
1176 }
1177
1178 #[test]
1181 fn the_pin_survives_an_unconfirmed_checkpoint() {
1182 use crate::session::handle::echo::{meta_for, view_for};
1183 let id = crate::session::SessionId("pin-reload".into());
1184 let (handle, ep) = SessionHandle::new(meta_for(&id), view_for());
1185 assert!(!record(&handle, None).keep_warm);
1186 ep.keep_warm.store(true, std::sync::atomic::Ordering::Release);
1187 let r = record(&handle, None);
1188 assert!(r.keep_warm && r.config.keep_warm);
1189 }
1190
1191 #[test]
1196 fn an_unknown_memory_mode_reads_as_off() {
1197 use crate::session::SessionMemoryMode;
1198 let cfg = |v: serde_json::Value| {
1199 serde_json::from_value::<SessionConfig>(serde_json::json!({ "memory_mode": v }))
1200 };
1201 assert_eq!(cfg(serde_json::json!("off")).unwrap().memory_mode, Some(SessionMemoryMode::Off));
1202 assert_eq!(
1203 cfg(serde_json::json!("capture_and_recall")).unwrap().memory_mode,
1204 Some(SessionMemoryMode::CaptureAndRecall)
1205 );
1206 assert_eq!(
1207 cfg(serde_json::json!("recall_forever")).unwrap().memory_mode,
1208 Some(SessionMemoryMode::Off)
1209 );
1210 assert_eq!(cfg(serde_json::json!(7)).unwrap().memory_mode, Some(SessionMemoryMode::Off));
1211 assert_eq!(cfg(serde_json::Value::Null).unwrap().memory_mode, None);
1212 let missing = serde_json::from_value::<SessionConfig>(serde_json::json!({})).unwrap();
1213 assert_eq!(missing.memory_mode, None);
1214 let rs: ReloadState = serde_json::from_value(serde_json::json!({
1215 "generation": 2,
1216 "written_at": "2026-10-14T18:17:03Z",
1217 "sessions": [{
1218 "id": "s1", "journal_id": "s1", "keep_warm": false, "lifecycle": "live",
1219 "config": { "memory_mode": "recall_forever" }
1220 }],
1221 "expected_clients": 0
1222 }))
1223 .expect("the reload state still parses");
1224 assert_eq!(rs.sessions[0].config.memory_mode, Some(SessionMemoryMode::Off));
1225 }
1226
1227 #[test]
1230 fn a_restore_keeps_the_sessions_memory_choice() {
1231 use crate::session::SessionMemoryMode;
1232 for mode in [Some(SessionMemoryMode::Off), Some(SessionMemoryMode::CaptureOnly), None] {
1233 let saved = SessionConfig {
1234 memory_mode: mode,
1235 await_extensions: true,
1236 ..Default::default()
1237 };
1238 for lifecycle in [SessionLifecycle::Live, SessionLifecycle::Parked, SessionLifecycle::Parking] {
1239 let restored = restore_config(&saved, lifecycle);
1240 assert_eq!(restored.memory_mode, mode, "{lifecycle:?}");
1241 assert!(!restored.await_extensions, "{lifecycle:?}");
1242 }
1243 }
1244 }
1245
1246 #[test]
1249 fn memory_choice_survives_an_unconfirmed_checkpoint() {
1250 use crate::session::handle::echo::{meta_for, view_for};
1251 use crate::session::SessionMemoryMode;
1252 let id = crate::session::SessionId("mem-reload".into());
1253 let (handle, ep) = SessionHandle::new(meta_for(&id), view_for());
1254 assert_eq!(record(&handle, None).config.memory_mode, None);
1255 *ep.memory_mode.lock().unwrap() = Some(SessionMemoryMode::Off);
1256 assert_eq!(record(&handle, None).config.memory_mode, Some(SessionMemoryMode::Off));
1257 }
1258
1259 #[tokio::test]
1263 async fn a_locked_placeholder_keeps_its_memory_choice() {
1264 use crate::session::SessionMemoryMode;
1265 let sid = crate::session::SessionId("locked-mem".into());
1266 let config = SessionConfig {
1267 memory_mode: Some(SessionMemoryMode::Off),
1268 ..SessionConfig::default()
1269 };
1270 let (handle, task) =
1271 crate::session::handle::locked_placeholder::spawn(sid, config, "pid 1 (tui)".into());
1272 assert_eq!(record(&handle, None).config.memory_mode, Some(SessionMemoryMode::Off));
1273 task.abort();
1274 }
1275
1276 #[tokio::test]
1280 async fn reload_aborted_waits_for_room_in_a_full_queue() {
1281 use crate::session::handle::echo::{meta_for, view_for};
1282 let id = crate::session::SessionId("abort-full".into());
1283 let (handle, mut ep) = SessionHandle::new(meta_for(&id), view_for());
1284 while handle.send(SessionCommand::Save).await.is_ok() {}
1285 let drain = tokio::spawn(async move {
1286 tokio::time::sleep(Duration::from_millis(100)).await;
1287 let mut got_abort = false;
1288 while let Some(a) = ep.cmd_rx.recv().await {
1289 if matches!(a.cmd, SessionCommand::ReloadAborted) {
1290 got_abort = true;
1291 break;
1292 }
1293 }
1294 got_abort
1295 });
1296 assert!(send_reload_aborted(&handle, Duration::from_secs(5)).await);
1297 assert!(drain.await.unwrap(), "delivered");
1298 let id = crate::session::SessionId("abort-stuck".into());
1300 let (stuck, _ep) = SessionHandle::new(meta_for(&id), view_for());
1301 while stuck.send(SessionCommand::Save).await.is_ok() {}
1302 assert!(!send_reload_aborted(&stuck, Duration::from_millis(100)).await);
1303 }
1304}