1use std::path::{Path, PathBuf};
22use std::sync::atomic::Ordering;
23use std::sync::Arc;
24use std::time::Duration;
25
26use serde::{Deserialize, Serialize};
27
28use super::registry::{self, DaemonLock, DaemonPaths};
29use super::DaemonState;
30use crate::session::wire::{ClientFrame, CHECKPOINT_QUERY_ID, PROTOCOL_VERSION};
31use crate::session::{
32 CheckpointReason, SessionCommand, SessionConfig, SessionEventWire, SessionHandle,
33 SessionLifecycle, SessionReloadRecord, SessionSetting,
34};
35
36pub const RELOAD_STATE_ENV: &str = "SYNAPS_DAEMON_RELOAD_STATE";
38pub const LOCK_FD_ENV: &str = "SYNAPS_DAEMON_LOCK_FD";
40pub const DRAIN_SECS_ENV: &str = "SYNAPS_DAEMON_RELOAD_DRAIN_SECS";
42const DEFAULT_DRAIN: Duration = Duration::from_secs(30);
43const VERSION_PROBE_TIMEOUT: Duration = Duration::from_secs(5);
44const CHECKPOINT_BUDGET: Duration = Duration::from_secs(
47 crate::session::budgets::CANCEL_TURN_TIMEOUT_SECS
48 + crate::session::budgets::SAVE_TIMEOUT_SECS
49 + 1,
50);
51pub const RETRY_AFTER_MS: u64 = 500;
53
54#[derive(Debug, Clone, Default)]
55pub struct ReloadRequest {
56 pub now: bool,
57 pub drain_secs: Option<u64>,
58 pub exe: Option<PathBuf>,
59}
60
61impl ReloadRequest {
62 pub fn from_frame(frame: &ClientFrame) -> Option<Self> {
63 match frame {
64 ClientFrame::Reload { now, drain_secs, exe } => {
65 Some(Self { now: *now, drain_secs: *drain_secs, exe: exe.clone() })
66 }
67 _ => None,
68 }
69 }
70}
71
72#[derive(Debug)]
73pub enum ReloadError {
74 Refused(String),
76 ExecFailed(String),
78}
79
80impl std::fmt::Display for ReloadError {
81 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
82 match self {
83 Self::Refused(w) => write!(f, "reload refused: {w}"),
84 Self::ExecFailed(e) => write!(f, "reload exec failed: {e}"),
85 }
86 }
87}
88
89#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
91pub struct PrintVersion {
92 pub binary_version: String,
93 pub protocol_version: u32,
94}
95
96impl PrintVersion {
97 pub fn current() -> Self {
98 Self {
99 binary_version: crate::session::wire::binary_version(),
100 protocol_version: PROTOCOL_VERSION,
101 }
102 }
103}
104
105#[derive(Debug, Clone, Serialize, Deserialize)]
107pub struct ReloadSession {
108 pub id: String,
109 pub journal_id: String,
110 pub config: SessionConfig,
111 pub keep_warm: bool,
112 pub lifecycle: SessionLifecycle,
113 #[serde(default)]
115 pub settings_replay: Vec<SessionSetting>,
116 pub input_owner_kind: Option<crate::session::ClientKind>,
117}
118
119#[derive(Debug, Clone, Serialize, Deserialize)]
121pub struct ReloadState {
122 pub generation: u64,
123 pub written_at: chrono::DateTime<chrono::Utc>,
124 pub sessions: Vec<ReloadSession>,
125 pub expected_clients: usize,
126}
127
128pub fn reload_state_path(paths: &DaemonPaths) -> PathBuf {
129 let stem = paths
130 .json
131 .file_stem()
132 .map(|s| s.to_string_lossy().into_owned())
133 .unwrap_or_else(|| "daemon".into());
134 paths.dir.join(format!("{stem}.reload.json"))
135}
136
137pub fn resolve_exe() -> PathBuf {
141 if let Some(a0) = std::env::args_os().next() {
142 let p = PathBuf::from(&a0);
143 if p.components().count() > 1 {
144 if let Ok(c) = p.canonicalize() {
145 return c;
146 }
147 }
148 }
149 std::env::current_exe().unwrap_or_else(|_| PathBuf::from("synaps"))
150}
151
152fn semver_tuple(s: &str) -> Option<(u64, u64, u64)> {
153 let core = s.split(['-', '+']).next()?;
154 let mut it = core.split('.').map(|p| p.parse::<u64>().ok());
155 Some((it.next()??, it.next()??, it.next()??))
156}
157
158pub fn version_gate(current: &PrintVersion, new: &PrintVersion) -> Result<(), String> {
162 if new.protocol_version < current.protocol_version {
163 return Err(format!(
164 "new binary speaks protocol {} < {}: reconnecting clients would be refused",
165 new.protocol_version, current.protocol_version
166 ));
167 }
168 match (semver_tuple(¤t.binary_version), semver_tuple(&new.binary_version)) {
169 (Some(c), Some(n)) if n < c => Err(format!(
170 "new binary {} is older than the running {} (reload is newer-or-equal only)",
171 new.binary_version, current.binary_version
172 )),
173 (Some(_), Some(_)) => Ok(()),
174 _ => Err(format!(
175 "cannot compare versions {:?} vs {:?}",
176 current.binary_version, new.binary_version
177 )),
178 }
179}
180
181#[cfg(unix)]
187pub fn validate_exe(exe: &Path) -> Result<PathBuf, String> {
188 use std::os::unix::fs::{MetadataExt, PermissionsExt};
189 let canon = exe
190 .canonicalize()
191 .map_err(|e| format!("--exe {}: {e}", exe.display()))?;
192 let md = std::fs::metadata(&canon).map_err(|e| format!("--exe {}: {e}", canon.display()))?;
193 if !md.is_file() {
194 return Err(format!("--exe {}: not a regular file", canon.display()));
195 }
196 let mode = md.permissions().mode();
197 if mode & 0o111 == 0 {
198 return Err(format!("--exe {}: not executable", canon.display()));
199 }
200 let uid = unsafe { libc::getuid() };
202 if md.uid() != uid && md.uid() != 0 {
203 return Err(format!(
204 "--exe {}: owned by uid {} (expected {} or root)",
205 canon.display(),
206 md.uid(),
207 uid
208 ));
209 }
210 if mode & 0o022 != 0 {
211 return Err(format!(
212 "--exe {}: group/world-writable (mode {:o})",
213 canon.display(),
214 mode & 0o777
215 ));
216 }
217 Ok(canon)
218}
219
220#[cfg(not(unix))]
221pub fn validate_exe(exe: &Path) -> Result<PathBuf, String> {
222 exe.canonicalize().map_err(|e| format!("--exe {}: {e}", exe.display()))
223}
224
225pub async fn probe_version(exe: &Path) -> Result<PrintVersion, String> {
227 let out = tokio::time::timeout(
228 VERSION_PROBE_TIMEOUT,
229 tokio::process::Command::new(exe)
230 .args(["daemon", "--print-version"])
231 .env("SYNAPS_DAEMON", "1")
232 .stdin(std::process::Stdio::null())
233 .stderr(std::process::Stdio::piped())
234 .stdout(std::process::Stdio::piped())
235 .output(),
236 )
237 .await
238 .map_err(|_| format!("{} --print-version timed out", exe.display()))?
239 .map_err(|e| format!("cannot run {}: {e}", exe.display()))?;
240 if !out.status.success() {
241 return Err(format!(
242 "{} --print-version exited {:?}: {}",
243 exe.display(),
244 out.status.code(),
245 String::from_utf8_lossy(&out.stderr).trim()
246 ));
247 }
248 serde_json::from_slice(&out.stdout)
249 .map_err(|e| format!("{} --print-version: unparsable output ({e})", exe.display()))
250}
251
252async fn checkpoint_one(handle: SessionHandle) -> Option<SessionReloadRecord> {
255 let mut rx = handle.subscribe();
256 if handle
257 .send(SessionCommand::Checkpoint { reason: CheckpointReason::Reload })
258 .await
259 .is_err()
260 {
261 return None;
262 }
263 let wait = async {
264 loop {
265 match rx.recv().await {
266 Ok(env) => {
267 if let SessionEventWire::QueryResult { id, value } = env.event {
268 if id == CHECKPOINT_QUERY_ID {
269 return serde_json::from_value(value["record"].clone()).ok();
270 }
271 }
272 }
273 Err(tokio::sync::broadcast::error::RecvError::Lagged(_)) => continue,
274 Err(tokio::sync::broadcast::error::RecvError::Closed) => return None,
275 }
276 }
277 };
278 tokio::time::timeout(CHECKPOINT_BUDGET, wait).await.unwrap_or(None)
279}
280
281fn record(handle: &SessionHandle, rec: Option<SessionReloadRecord>) -> ReloadSession {
285 let meta = handle.meta();
286 let journal_id = handle.journal_id();
287 let (config, keep_warm, lifecycle, settings_replay, model) = match rec {
288 Some(r) => (r.config, r.keep_warm, r.lifecycle, r.settings_replay, r.model),
289 None => (
290 SessionConfig {
291 cwd: meta.cwd.clone(),
292 env: None,
293 env_stripped: Vec::new(),
294 ..SessionConfig::default()
295 },
296 false,
297 handle.lifecycle(),
298 Vec::new(),
299 handle.view().model.clone(),
300 ),
301 };
302 ReloadSession {
303 id: meta.id.as_str().to_string(),
304 journal_id: journal_id.clone(),
305 config: SessionConfig {
306 continue_session: Some(Some(journal_id)),
307 model_override: Some(model),
308 keep_warm,
309 name: None,
312 ..config
313 },
314 keep_warm,
315 lifecycle,
316 settings_replay,
317 input_owner_kind: None,
318 }
319}
320
321pub struct Prepared {
326 pub exe: PathBuf,
327 pub rs_path: PathBuf,
328 pub generation: u64,
329}
330
331pub async fn prepare(
332 state: &Arc<DaemonState>,
333 paths: &DaemonPaths,
334 req: ReloadRequest,
335) -> Result<Prepared, ReloadError> {
336 let exe = req.exe.clone().unwrap_or_else(|| {
338 registry::read_daemon_json(paths)
339 .and_then(|i| i.exe)
340 .unwrap_or_else(resolve_exe)
341 });
342 let exe = validate_exe(&exe).map_err(ReloadError::Refused)?;
343 let new = probe_version(&exe).await.map_err(ReloadError::Refused)?;
344 version_gate(&PrintVersion::current(), &new).map_err(ReloadError::Refused)?;
345 if state.reloading.swap(true, Ordering::SeqCst) {
346 return Err(ReloadError::Refused("a reload is already in progress".into()));
347 }
348 let generation = state.generation + 1;
349 tracing::info!(exe = %exe.display(), new = %new.binary_version, generation, "daemon: reload accepted");
350
351 let drain = if req.now {
353 Duration::ZERO
354 } else {
355 req.drain_secs
356 .map(Duration::from_secs)
357 .or_else(|| std::env::var(DRAIN_SECS_ENV).ok()?.parse().ok().map(Duration::from_secs))
358 .unwrap_or(DEFAULT_DRAIN)
359 };
360 let t0 = std::time::Instant::now();
361 while t0.elapsed() < drain {
362 let mut all_idle = true;
363 for h in state.live_sessions() {
364 if !super::session_is_idle(&h).await {
365 all_idle = false;
366 break;
367 }
368 }
369 if all_idle {
370 break;
371 }
372 tokio::time::sleep(Duration::from_millis(200)).await;
373 }
374 let handles = state.live_sessions();
375 let results = futures::future::join_all(handles.iter().cloned().map(checkpoint_one)).await;
376 let mut sessions = Vec::with_capacity(handles.len());
377 for (h, rec) in handles.iter().zip(results) {
378 if rec.is_none() {
379 tracing::warn!(session = %h.id, "reload: checkpoint did not confirm; recording from the handle");
380 }
381 sessions.push(record(h, rec));
382 }
383
384 let rs = ReloadState {
386 generation,
387 written_at: chrono::Utc::now(),
388 sessions,
389 expected_clients: state.connections.load(Ordering::SeqCst),
390 };
391 let rs_path = reload_state_path(paths);
392 let body = serde_json::to_vec_pretty(&rs).map_err(|e| {
397 state.reloading.store(false, Ordering::SeqCst);
398 ReloadError::ExecFailed(e.to_string())
399 })?;
400 registry::write_private_atomic(&rs_path, &body).map_err(|e| {
401 state.reloading.store(false, Ordering::SeqCst);
402 ReloadError::ExecFailed(e.to_string())
403 })?;
404
405 state.reload_generation.store(generation, Ordering::SeqCst);
408 state.fire_announce();
409 let t1 = std::time::Instant::now();
410 while state.connections.load(Ordering::SeqCst) > 1 && t1.elapsed() < Duration::from_secs(1) {
411 tokio::time::sleep(Duration::from_millis(20)).await;
412 }
413 Ok(Prepared { exe, rs_path, generation })
414}
415
416pub async fn exec(state: &Arc<DaemonState>, paths: &DaemonPaths, p: Prepared) -> ReloadError {
425 let mut info = registry::read_daemon_json(paths).unwrap_or_else(|| registry::DaemonInfo {
426 pid: std::process::id(),
427 protocol_version: PROTOCOL_VERSION,
428 daemon_version: crate::session::wire::binary_version(),
429 profile: state.profile.clone(),
430 started_at: chrono::Utc::now(),
431 socket: paths.sock.to_string_lossy().into_owned(),
432 exe: None,
433 generation: 1,
434 });
435 let prev_generation = info.generation;
436 info.generation = p.generation;
437 info.started_at = chrono::Utc::now();
438 info.exe = Some(p.exe.clone());
439 let _ = registry::write_daemon_json(paths, &info);
440 let _ = tokio::time::timeout(
441 Duration::from_secs(5),
442 async { state.host.ext_manager().write().await.shutdown_all().await },
443 )
444 .await;
445
446 let err = exec_self(&p.exe, &p.rs_path, state);
447 tracing::error!(error = %err, "daemon: reload exec failed; continuing on the old image");
448 info.generation = prev_generation;
449 let _ = registry::write_daemon_json(paths, &info);
450 let _ = std::fs::remove_file(&p.rs_path);
451 state.reload_generation.store(prev_generation, Ordering::SeqCst);
452 state.reset_announce();
453 let (tx, mut rx) = tokio::sync::mpsc::unbounded_channel();
456 let _loader = crate::extensions::loader::spawn_discover_and_load(
457 Arc::clone(state.host.ext_manager()),
458 tx,
459 None,
460 );
461 let wait = async {
462 while let Some(ev) = rx.recv().await {
463 if let crate::extensions::loader::ExtensionLoaderEvent::Finished { loaded, failed } = ev {
464 tracing::info!(loaded = loaded.len(), failed = failed.len(), "daemon: extensions re-discovered after failed reload");
465 break;
466 }
467 }
468 };
469 if tokio::time::timeout(Duration::from_secs(10), wait).await.is_err() {
470 tracing::warn!("daemon: extension re-discovery still running after 10 s");
471 }
472 state.reloading.store(false, Ordering::SeqCst);
473 ReloadError::ExecFailed(err)
474}
475
476#[cfg(unix)]
477fn exec_self(exe: &Path, rs_path: &Path, state: &DaemonState) -> String {
478 use std::os::unix::process::CommandExt;
479 let Some(lock_fd) = state.lock_fd.lock().unwrap_or_else(|e| e.into_inner()).as_ref().map(|l| l.raw_fd()) else {
480 return "no lock fd to inherit".into();
481 };
482 unsafe {
487 let flags = libc::fcntl(lock_fd, libc::F_GETFD);
488 if flags >= 0 {
489 libc::fcntl(lock_fd, libc::F_SETFD, flags & !libc::FD_CLOEXEC);
490 }
491 }
492 let mut args: Vec<std::ffi::OsString> = std::env::args_os().skip(1).collect();
493 if args.is_empty() || args.iter().all(|a| a != "daemon") {
494 args = vec!["daemon".into(), "--foreground".into()];
495 }
496 let mut cmd = std::process::Command::new(exe);
497 cmd.args(&args)
498 .env("SYNAPS_DAEMON", "1")
499 .env(LOCK_FD_ENV, lock_fd.to_string())
500 .env(RELOAD_STATE_ENV, rs_path);
501 let e = cmd.exec();
502 unsafe {
505 let flags = libc::fcntl(lock_fd, libc::F_GETFD);
506 if flags >= 0 {
507 libc::fcntl(lock_fd, libc::F_SETFD, flags | libc::FD_CLOEXEC);
508 }
509 }
510 e.to_string()
511}
512
513#[cfg(not(unix))]
514fn exec_self(_exe: &Path, _rs_path: &Path, _state: &DaemonState) -> String {
515 "reload is unix-only".into()
516}
517
518pub fn adopt_from_env() -> anyhow::Result<Option<(DaemonLock, ReloadState, PathBuf)>> {
522 let Some(rs_path) = std::env::var_os(RELOAD_STATE_ENV) else {
523 return Ok(None);
524 };
525 let rs_path = PathBuf::from(rs_path);
526 std::env::remove_var(RELOAD_STATE_ENV);
527 let fd: i32 = std::env::var(LOCK_FD_ENV)
528 .ok()
529 .and_then(|s| s.parse().ok())
530 .ok_or_else(|| anyhow::anyhow!("{RELOAD_STATE_ENV} set without {LOCK_FD_ENV}: refusing to start (the flock is the liveness oracle)"))?;
531 std::env::remove_var(LOCK_FD_ENV);
532 let lock = DaemonLock::adopt(fd)?;
533 let body = std::fs::read_to_string(&rs_path)?;
534 let rs: ReloadState = serde_json::from_str(&body)?;
535 Ok(Some((lock, rs, rs_path)))
536}
537
538fn is_lock_held_error(e: &str) -> bool {
550 e.contains("is live in another process")
551}
552
553fn parse_holder_from_error(e: &str) -> String {
555 if let Some(start) = e.find("(pid ") {
557 let after = &e[start + 1..]; if let Some(end) = after.find(')') {
559 return after[..end].to_string();
560 }
561 }
562 "unknown holder".to_string()
563}
564
565pub async fn rehydrate(state: &Arc<DaemonState>, rs: &ReloadState) {
566 for s in &rs.sessions {
567 let created = match state.create(s.config.clone()).await {
568 Ok(h) => Ok(h),
569 Err(e) if is_lock_held_error(&e) => {
572 let holder = parse_holder_from_error(&e);
573 tracing::warn!(
574 session = %s.id, holder = %holder,
575 "daemon: journal locked by another process; registering Parked placeholder"
576 );
577 let sid = crate::session::SessionId::from(s.id.as_str());
578 let (handle, _task) = crate::session::handle::locked_placeholder::spawn(
579 sid,
580 s.config.clone(),
581 holder,
582 );
583 state.insert(handle);
584 continue;
585 }
586 Err(e) if s.config.continue_session.is_some() => {
587 tracing::warn!(session = %s.id, error = %e, "daemon: journal not continuable; recreating fresh");
588 state
589 .create(SessionConfig {
590 continue_session: None,
591 ..s.config.clone()
592 })
593 .await
594 }
595 Err(e) => Err(e),
596 };
597 match created {
598 Ok(h) => {
599 let new_id = h.id.as_str().to_string();
600 if new_id != s.id {
601 state
602 .reload_aliases
603 .lock()
604 .unwrap_or_else(|e| e.into_inner())
605 .insert(s.id.clone(), new_id.clone());
606 }
607 for (i, setting) in s.settings_replay.iter().enumerate() {
609 let _ = h
610 .send(SessionCommand::Set {
611 id: u64::MAX - i as u64,
612 setting: setting.clone(),
613 })
614 .await;
615 }
616 if s.keep_warm {
617 let _ = h.send(SessionCommand::KeepWarm { on: true }).await;
618 }
619 if matches!(s.lifecycle, SessionLifecycle::Parked | SessionLifecycle::Parking) {
620 let _ = h.send(SessionCommand::Park).await;
621 }
622 tracing::info!(old = %s.id, new = %new_id, lifecycle = ?s.lifecycle, "daemon: session rehydrated after reload");
623 }
624 Err(e) => tracing::warn!(session = %s.id, error = %e, "daemon: session could not be rehydrated"),
625 }
626 }
627}
628
629#[cfg(test)]
630mod tests {
631 use super::*;
632
633 fn pv(b: &str, p: u32) -> PrintVersion {
634 PrintVersion { binary_version: b.into(), protocol_version: p }
635 }
636
637 #[test]
638 fn gate_is_directional_newer_or_equal() {
639 let cur = pv("0.9.0", 2);
640 assert!(version_gate(&cur, &pv("0.9.0", 2)).is_ok(), "equal allowed");
641 assert!(version_gate(&cur, &pv("0.9.1", 2)).is_ok());
642 assert!(version_gate(&cur, &pv("1.0.0-rc1", 3)).is_ok());
643 assert!(version_gate(&cur, &pv("0.8.9", 2)).is_err(), "older refused");
644 assert!(version_gate(&cur, &pv("0.9.0", 1)).is_err(), "older protocol refused");
645 assert!(version_gate(&cur, &pv("garbage", 2)).is_err());
646 assert!(
649 version_gate(&pv("0.9.1", 3), &pv("0.10.0", 3)).is_ok(),
650 "0.10.0 is newer than 0.9.1"
651 );
652 assert!(
653 version_gate(&pv("0.10.0", 3), &pv("0.9.1", 3)).is_err(),
654 "0.9.1 is older than 0.10.0"
655 );
656 }
657
658 #[cfg(unix)]
659 #[test]
660 fn validate_exe_refuses_missing_dirs_non_exec_and_writable() {
661 use std::os::unix::fs::PermissionsExt;
662 let d = tempfile::tempdir().unwrap();
663 assert!(validate_exe(&d.path().join("nope")).unwrap_err().contains("nope"));
664 assert!(validate_exe(d.path()).unwrap_err().contains("not a regular file"));
665 let f = d.path().join("bin");
666 std::fs::write(&f, "#!/bin/sh\nexit 0\n").unwrap();
667 std::fs::set_permissions(&f, std::fs::Permissions::from_mode(0o644)).unwrap();
668 assert!(validate_exe(&f).unwrap_err().contains("not executable"));
669 std::fs::set_permissions(&f, std::fs::Permissions::from_mode(0o777)).unwrap();
670 assert!(validate_exe(&f).unwrap_err().contains("writable"));
671 std::fs::set_permissions(&f, std::fs::Permissions::from_mode(0o755)).unwrap();
672 assert_eq!(validate_exe(&f).unwrap(), f.canonicalize().unwrap());
673 let l = d.path().join("link");
675 std::os::unix::fs::symlink(&f, &l).unwrap();
676 assert_eq!(validate_exe(&l).unwrap(), f.canonicalize().unwrap());
677 }
678
679 #[test]
680 fn reload_state_path_follows_profile() {
681 let d = tempfile::tempdir().unwrap();
682 let p = registry::daemon_paths_in(d.path(), Some("work"));
683 assert!(reload_state_path(&p).ends_with("daemon-work.reload.json"), "{:?}", reload_state_path(&p));
684 let p = registry::daemon_paths_in(d.path(), None);
685 assert!(reload_state_path(&p).ends_with("daemon.reload.json"));
686 }
687}