1use std::path::{Path, PathBuf};
22use std::sync::atomic::Ordering;
23use std::sync::Arc;
24use std::time::Duration;
25
26use serde::{Deserialize, Serialize};
27
28use super::registry::{self, DaemonLock, DaemonPaths};
29use super::DaemonState;
30use crate::session::wire::{ClientFrame, CHECKPOINT_QUERY_ID, PROTOCOL_VERSION};
31use crate::session::{
32 CheckpointReason, SessionCommand, SessionConfig, SessionEventWire, SessionHandle,
33 SessionLifecycle, SessionReloadRecord, SessionSetting,
34};
35
36pub const RELOAD_STATE_ENV: &str = "SYNAPS_DAEMON_RELOAD_STATE";
38pub const LOCK_FD_ENV: &str = "SYNAPS_DAEMON_LOCK_FD";
40pub const DRAIN_SECS_ENV: &str = "SYNAPS_DAEMON_RELOAD_DRAIN_SECS";
42const DEFAULT_DRAIN: Duration = Duration::from_secs(30);
43const VERSION_PROBE_TIMEOUT: Duration = Duration::from_secs(5);
44const CHECKPOINT_BUDGET: Duration =
45 Duration::from_secs(crate::session::budgets::SAVE_TIMEOUT_SECS + 1);
46pub const RETRY_AFTER_MS: u64 = 500;
48
49#[derive(Debug, Clone, Default)]
50pub struct ReloadRequest {
51 pub now: bool,
52 pub drain_secs: Option<u64>,
53 pub exe: Option<PathBuf>,
54}
55
56impl ReloadRequest {
57 pub fn from_frame(frame: &ClientFrame) -> Option<Self> {
58 match frame {
59 ClientFrame::Reload { now, drain_secs, exe } => {
60 Some(Self { now: *now, drain_secs: *drain_secs, exe: exe.clone() })
61 }
62 _ => None,
63 }
64 }
65}
66
67#[derive(Debug)]
68pub enum ReloadError {
69 Refused(String),
71 ExecFailed(String),
73}
74
75impl std::fmt::Display for ReloadError {
76 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
77 match self {
78 Self::Refused(w) => write!(f, "reload refused: {w}"),
79 Self::ExecFailed(e) => write!(f, "reload exec failed: {e}"),
80 }
81 }
82}
83
84#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
86pub struct PrintVersion {
87 pub binary_version: String,
88 pub protocol_version: u32,
89}
90
91impl PrintVersion {
92 pub fn current() -> Self {
93 Self {
94 binary_version: crate::session::wire::binary_version(),
95 protocol_version: PROTOCOL_VERSION,
96 }
97 }
98}
99
100#[derive(Debug, Clone, Serialize, Deserialize)]
102pub struct ReloadSession {
103 pub id: String,
104 pub journal_id: String,
105 pub config: SessionConfig,
106 pub keep_warm: bool,
107 pub lifecycle: SessionLifecycle,
108 #[serde(default)]
110 pub settings_replay: Vec<SessionSetting>,
111 pub input_owner_kind: Option<crate::session::ClientKind>,
112}
113
114#[derive(Debug, Clone, Serialize, Deserialize)]
116pub struct ReloadState {
117 pub generation: u64,
118 pub written_at: chrono::DateTime<chrono::Utc>,
119 pub sessions: Vec<ReloadSession>,
120 pub expected_clients: usize,
121}
122
123pub fn reload_state_path(paths: &DaemonPaths) -> PathBuf {
124 let stem = paths
125 .json
126 .file_stem()
127 .map(|s| s.to_string_lossy().into_owned())
128 .unwrap_or_else(|| "daemon".into());
129 paths.dir.join(format!("{stem}.reload.json"))
130}
131
132pub fn resolve_exe() -> PathBuf {
136 if let Some(a0) = std::env::args_os().next() {
137 let p = PathBuf::from(&a0);
138 if p.components().count() > 1 {
139 if let Ok(c) = p.canonicalize() {
140 return c;
141 }
142 }
143 }
144 std::env::current_exe().unwrap_or_else(|_| PathBuf::from("synaps"))
145}
146
147fn semver_tuple(s: &str) -> Option<(u64, u64, u64)> {
148 let core = s.split(['-', '+']).next()?;
149 let mut it = core.split('.').map(|p| p.parse::<u64>().ok());
150 Some((it.next()??, it.next()??, it.next()??))
151}
152
153pub fn version_gate(current: &PrintVersion, new: &PrintVersion) -> Result<(), String> {
157 if new.protocol_version < current.protocol_version {
158 return Err(format!(
159 "new binary speaks protocol {} < {}: reconnecting clients would be refused",
160 new.protocol_version, current.protocol_version
161 ));
162 }
163 match (semver_tuple(¤t.binary_version), semver_tuple(&new.binary_version)) {
164 (Some(c), Some(n)) if n < c => Err(format!(
165 "new binary {} is older than the running {} (reload is newer-or-equal only)",
166 new.binary_version, current.binary_version
167 )),
168 (Some(_), Some(_)) => Ok(()),
169 _ => Err(format!(
170 "cannot compare versions {:?} vs {:?}",
171 current.binary_version, new.binary_version
172 )),
173 }
174}
175
176#[cfg(unix)]
182pub fn validate_exe(exe: &Path) -> Result<PathBuf, String> {
183 use std::os::unix::fs::{MetadataExt, PermissionsExt};
184 let canon = exe
185 .canonicalize()
186 .map_err(|e| format!("--exe {}: {e}", exe.display()))?;
187 let md = std::fs::metadata(&canon).map_err(|e| format!("--exe {}: {e}", canon.display()))?;
188 if !md.is_file() {
189 return Err(format!("--exe {}: not a regular file", canon.display()));
190 }
191 let mode = md.permissions().mode();
192 if mode & 0o111 == 0 {
193 return Err(format!("--exe {}: not executable", canon.display()));
194 }
195 let uid = unsafe { libc::getuid() };
197 if md.uid() != uid && md.uid() != 0 {
198 return Err(format!(
199 "--exe {}: owned by uid {} (expected {} or root)",
200 canon.display(),
201 md.uid(),
202 uid
203 ));
204 }
205 if mode & 0o022 != 0 {
206 return Err(format!(
207 "--exe {}: group/world-writable (mode {:o})",
208 canon.display(),
209 mode & 0o777
210 ));
211 }
212 Ok(canon)
213}
214
215#[cfg(not(unix))]
216pub fn validate_exe(exe: &Path) -> Result<PathBuf, String> {
217 exe.canonicalize().map_err(|e| format!("--exe {}: {e}", exe.display()))
218}
219
220pub async fn probe_version(exe: &Path) -> Result<PrintVersion, String> {
222 let out = tokio::time::timeout(
223 VERSION_PROBE_TIMEOUT,
224 tokio::process::Command::new(exe)
225 .args(["daemon", "--print-version"])
226 .env("SYNAPS_DAEMON", "1")
227 .stdin(std::process::Stdio::null())
228 .stderr(std::process::Stdio::piped())
229 .stdout(std::process::Stdio::piped())
230 .output(),
231 )
232 .await
233 .map_err(|_| format!("{} --print-version timed out", exe.display()))?
234 .map_err(|e| format!("cannot run {}: {e}", exe.display()))?;
235 if !out.status.success() {
236 return Err(format!(
237 "{} --print-version exited {:?}: {}",
238 exe.display(),
239 out.status.code(),
240 String::from_utf8_lossy(&out.stderr).trim()
241 ));
242 }
243 serde_json::from_slice(&out.stdout)
244 .map_err(|e| format!("{} --print-version: unparsable output ({e})", exe.display()))
245}
246
247async fn checkpoint_one(handle: SessionHandle) -> Option<SessionReloadRecord> {
250 let mut rx = handle.subscribe();
251 if handle
252 .send(SessionCommand::Checkpoint { reason: CheckpointReason::Reload })
253 .await
254 .is_err()
255 {
256 return None;
257 }
258 let wait = async {
259 loop {
260 match rx.recv().await {
261 Ok(env) => {
262 if let SessionEventWire::QueryResult { id, value } = env.event {
263 if id == CHECKPOINT_QUERY_ID {
264 return serde_json::from_value(value["record"].clone()).ok();
265 }
266 }
267 }
268 Err(tokio::sync::broadcast::error::RecvError::Lagged(_)) => continue,
269 Err(tokio::sync::broadcast::error::RecvError::Closed) => return None,
270 }
271 }
272 };
273 tokio::time::timeout(CHECKPOINT_BUDGET, wait).await.unwrap_or(None)
274}
275
276fn record(handle: &SessionHandle, rec: Option<SessionReloadRecord>) -> ReloadSession {
280 let meta = handle.meta();
281 let journal_id = handle.journal_id();
282 let (config, keep_warm, lifecycle, settings_replay, model) = match rec {
283 Some(r) => (r.config, r.keep_warm, r.lifecycle, r.settings_replay, r.model),
284 None => (
285 SessionConfig {
286 cwd: meta.cwd.clone(),
287 env: None,
288 env_stripped: Vec::new(),
289 ..SessionConfig::default()
290 },
291 false,
292 handle.lifecycle(),
293 Vec::new(),
294 handle.view().model.clone(),
295 ),
296 };
297 ReloadSession {
298 id: meta.id.as_str().to_string(),
299 journal_id: journal_id.clone(),
300 config: SessionConfig {
301 continue_session: Some(Some(journal_id)),
302 model_override: Some(model),
303 keep_warm,
304 name: None,
307 ..config
308 },
309 keep_warm,
310 lifecycle,
311 settings_replay,
312 input_owner_kind: None,
313 }
314}
315
316pub struct Prepared {
321 pub exe: PathBuf,
322 pub rs_path: PathBuf,
323 pub generation: u64,
324}
325
326pub async fn prepare(
327 state: &Arc<DaemonState>,
328 paths: &DaemonPaths,
329 req: ReloadRequest,
330) -> Result<Prepared, ReloadError> {
331 let exe = req.exe.clone().unwrap_or_else(|| {
333 registry::read_daemon_json(paths)
334 .and_then(|i| i.exe)
335 .unwrap_or_else(resolve_exe)
336 });
337 let exe = validate_exe(&exe).map_err(ReloadError::Refused)?;
338 let new = probe_version(&exe).await.map_err(ReloadError::Refused)?;
339 version_gate(&PrintVersion::current(), &new).map_err(ReloadError::Refused)?;
340 if state.reloading.swap(true, Ordering::SeqCst) {
341 return Err(ReloadError::Refused("a reload is already in progress".into()));
342 }
343 let generation = state.generation + 1;
344 tracing::info!(exe = %exe.display(), new = %new.binary_version, generation, "daemon: reload accepted");
345
346 let drain = if req.now {
348 Duration::ZERO
349 } else {
350 req.drain_secs
351 .map(Duration::from_secs)
352 .or_else(|| std::env::var(DRAIN_SECS_ENV).ok()?.parse().ok().map(Duration::from_secs))
353 .unwrap_or(DEFAULT_DRAIN)
354 };
355 let t0 = std::time::Instant::now();
356 while t0.elapsed() < drain {
357 let mut all_idle = true;
358 for h in state.live_sessions() {
359 if !super::session_is_idle(&h).await {
360 all_idle = false;
361 break;
362 }
363 }
364 if all_idle {
365 break;
366 }
367 tokio::time::sleep(Duration::from_millis(200)).await;
368 }
369 let handles = state.live_sessions();
370 let results = futures::future::join_all(handles.iter().cloned().map(checkpoint_one)).await;
371 let mut sessions = Vec::with_capacity(handles.len());
372 for (h, rec) in handles.iter().zip(results) {
373 if rec.is_none() {
374 tracing::warn!(session = %h.id, "reload: checkpoint did not confirm; recording from the handle");
375 }
376 sessions.push(record(h, rec));
377 }
378
379 let rs = ReloadState {
381 generation,
382 written_at: chrono::Utc::now(),
383 sessions,
384 expected_clients: state.connections.load(Ordering::SeqCst),
385 };
386 let rs_path = reload_state_path(paths);
387 let body = serde_json::to_vec_pretty(&rs).map_err(|e| {
392 state.reloading.store(false, Ordering::SeqCst);
393 ReloadError::ExecFailed(e.to_string())
394 })?;
395 registry::write_private_atomic(&rs_path, &body).map_err(|e| {
396 state.reloading.store(false, Ordering::SeqCst);
397 ReloadError::ExecFailed(e.to_string())
398 })?;
399
400 state.reload_generation.store(generation, Ordering::SeqCst);
403 state.fire_announce();
404 let t1 = std::time::Instant::now();
405 while state.connections.load(Ordering::SeqCst) > 1 && t1.elapsed() < Duration::from_secs(1) {
406 tokio::time::sleep(Duration::from_millis(20)).await;
407 }
408 Ok(Prepared { exe, rs_path, generation })
409}
410
411pub async fn exec(state: &Arc<DaemonState>, paths: &DaemonPaths, p: Prepared) -> ReloadError {
420 let mut info = registry::read_daemon_json(paths).unwrap_or_else(|| registry::DaemonInfo {
421 pid: std::process::id(),
422 protocol_version: PROTOCOL_VERSION,
423 daemon_version: crate::session::wire::binary_version(),
424 profile: state.profile.clone(),
425 started_at: chrono::Utc::now(),
426 socket: paths.sock.to_string_lossy().into_owned(),
427 exe: None,
428 generation: 1,
429 });
430 let prev_generation = info.generation;
431 info.generation = p.generation;
432 info.started_at = chrono::Utc::now();
433 info.exe = Some(p.exe.clone());
434 let _ = registry::write_daemon_json(paths, &info);
435 let _ = tokio::time::timeout(
436 Duration::from_secs(5),
437 async { state.host.ext_manager().write().await.shutdown_all().await },
438 )
439 .await;
440
441 let err = exec_self(&p.exe, &p.rs_path, state);
442 tracing::error!(error = %err, "daemon: reload exec failed; continuing on the old image");
443 info.generation = prev_generation;
444 let _ = registry::write_daemon_json(paths, &info);
445 let _ = std::fs::remove_file(&p.rs_path);
446 state.reload_generation.store(prev_generation, Ordering::SeqCst);
447 state.reset_announce();
448 let (tx, mut rx) = tokio::sync::mpsc::unbounded_channel();
451 let _loader = crate::extensions::loader::spawn_discover_and_load(
452 Arc::clone(state.host.ext_manager()),
453 tx,
454 None,
455 );
456 let wait = async {
457 while let Some(ev) = rx.recv().await {
458 if let crate::extensions::loader::ExtensionLoaderEvent::Finished { loaded, failed } = ev {
459 tracing::info!(loaded = loaded.len(), failed = failed.len(), "daemon: extensions re-discovered after failed reload");
460 break;
461 }
462 }
463 };
464 if tokio::time::timeout(Duration::from_secs(10), wait).await.is_err() {
465 tracing::warn!("daemon: extension re-discovery still running after 10 s");
466 }
467 state.reloading.store(false, Ordering::SeqCst);
468 ReloadError::ExecFailed(err)
469}
470
471#[cfg(unix)]
472fn exec_self(exe: &Path, rs_path: &Path, state: &DaemonState) -> String {
473 use std::os::unix::process::CommandExt;
474 let Some(lock_fd) = state.lock_fd.lock().unwrap_or_else(|e| e.into_inner()).as_ref().map(|l| l.raw_fd()) else {
475 return "no lock fd to inherit".into();
476 };
477 unsafe {
482 let flags = libc::fcntl(lock_fd, libc::F_GETFD);
483 if flags >= 0 {
484 libc::fcntl(lock_fd, libc::F_SETFD, flags & !libc::FD_CLOEXEC);
485 }
486 }
487 let mut args: Vec<std::ffi::OsString> = std::env::args_os().skip(1).collect();
488 if args.is_empty() || args.iter().all(|a| a != "daemon") {
489 args = vec!["daemon".into(), "--foreground".into()];
490 }
491 let mut cmd = std::process::Command::new(exe);
492 cmd.args(&args)
493 .env("SYNAPS_DAEMON", "1")
494 .env(LOCK_FD_ENV, lock_fd.to_string())
495 .env(RELOAD_STATE_ENV, rs_path);
496 let e = cmd.exec();
497 unsafe {
500 let flags = libc::fcntl(lock_fd, libc::F_GETFD);
501 if flags >= 0 {
502 libc::fcntl(lock_fd, libc::F_SETFD, flags | libc::FD_CLOEXEC);
503 }
504 }
505 e.to_string()
506}
507
508#[cfg(not(unix))]
509fn exec_self(_exe: &Path, _rs_path: &Path, _state: &DaemonState) -> String {
510 "reload is unix-only".into()
511}
512
513pub fn adopt_from_env() -> anyhow::Result<Option<(DaemonLock, ReloadState, PathBuf)>> {
517 let Some(rs_path) = std::env::var_os(RELOAD_STATE_ENV) else {
518 return Ok(None);
519 };
520 let rs_path = PathBuf::from(rs_path);
521 std::env::remove_var(RELOAD_STATE_ENV);
522 let fd: i32 = std::env::var(LOCK_FD_ENV)
523 .ok()
524 .and_then(|s| s.parse().ok())
525 .ok_or_else(|| anyhow::anyhow!("{RELOAD_STATE_ENV} set without {LOCK_FD_ENV}: refusing to start (the flock is the liveness oracle)"))?;
526 std::env::remove_var(LOCK_FD_ENV);
527 let lock = DaemonLock::adopt(fd)?;
528 let body = std::fs::read_to_string(&rs_path)?;
529 let rs: ReloadState = serde_json::from_str(&body)?;
530 Ok(Some((lock, rs, rs_path)))
531}
532
533fn is_lock_held_error(e: &str) -> bool {
545 e.contains("is live in another process")
546}
547
548fn parse_holder_from_error(e: &str) -> String {
550 if let Some(start) = e.find("(pid ") {
552 let after = &e[start + 1..]; if let Some(end) = after.find(')') {
554 return after[..end].to_string();
555 }
556 }
557 "unknown holder".to_string()
558}
559
560pub async fn rehydrate(state: &Arc<DaemonState>, rs: &ReloadState) {
561 for s in &rs.sessions {
562 let created = match state.create(s.config.clone()).await {
563 Ok(h) => Ok(h),
564 Err(e) if is_lock_held_error(&e) => {
567 let holder = parse_holder_from_error(&e);
568 tracing::warn!(
569 session = %s.id, holder = %holder,
570 "daemon: journal locked by another process; registering Parked placeholder"
571 );
572 let sid = crate::session::SessionId::from(s.id.as_str());
573 let (handle, _task) = crate::session::handle::locked_placeholder::spawn(
574 sid,
575 s.config.clone(),
576 holder,
577 );
578 state.insert(handle);
579 continue;
580 }
581 Err(e) if s.config.continue_session.is_some() => {
582 tracing::warn!(session = %s.id, error = %e, "daemon: journal not continuable; recreating fresh");
583 state
584 .create(SessionConfig {
585 continue_session: None,
586 ..s.config.clone()
587 })
588 .await
589 }
590 Err(e) => Err(e),
591 };
592 match created {
593 Ok(h) => {
594 let new_id = h.id.as_str().to_string();
595 if new_id != s.id {
596 state
597 .reload_aliases
598 .lock()
599 .unwrap_or_else(|e| e.into_inner())
600 .insert(s.id.clone(), new_id.clone());
601 }
602 for (i, setting) in s.settings_replay.iter().enumerate() {
604 let _ = h
605 .send(SessionCommand::Set {
606 id: u64::MAX - i as u64,
607 setting: setting.clone(),
608 })
609 .await;
610 }
611 if s.keep_warm {
612 let _ = h.send(SessionCommand::KeepWarm { on: true }).await;
613 }
614 if matches!(s.lifecycle, SessionLifecycle::Parked | SessionLifecycle::Parking) {
615 let _ = h.send(SessionCommand::Park).await;
616 }
617 tracing::info!(old = %s.id, new = %new_id, lifecycle = ?s.lifecycle, "daemon: session rehydrated after reload");
618 }
619 Err(e) => tracing::warn!(session = %s.id, error = %e, "daemon: session could not be rehydrated"),
620 }
621 }
622}
623
624#[cfg(test)]
625mod tests {
626 use super::*;
627
628 fn pv(b: &str, p: u32) -> PrintVersion {
629 PrintVersion { binary_version: b.into(), protocol_version: p }
630 }
631
632 #[test]
633 fn gate_is_directional_newer_or_equal() {
634 let cur = pv("0.9.0", 2);
635 assert!(version_gate(&cur, &pv("0.9.0", 2)).is_ok(), "equal allowed");
636 assert!(version_gate(&cur, &pv("0.9.1", 2)).is_ok());
637 assert!(version_gate(&cur, &pv("1.0.0-rc1", 3)).is_ok());
638 assert!(version_gate(&cur, &pv("0.8.9", 2)).is_err(), "older refused");
639 assert!(version_gate(&cur, &pv("0.9.0", 1)).is_err(), "older protocol refused");
640 assert!(version_gate(&cur, &pv("garbage", 2)).is_err());
641 assert!(
644 version_gate(&pv("0.9.1", 3), &pv("0.10.0", 3)).is_ok(),
645 "0.10.0 is newer than 0.9.1"
646 );
647 assert!(
648 version_gate(&pv("0.10.0", 3), &pv("0.9.1", 3)).is_err(),
649 "0.9.1 is older than 0.10.0"
650 );
651 }
652
653 #[cfg(unix)]
654 #[test]
655 fn validate_exe_refuses_missing_dirs_non_exec_and_writable() {
656 use std::os::unix::fs::PermissionsExt;
657 let d = tempfile::tempdir().unwrap();
658 assert!(validate_exe(&d.path().join("nope")).unwrap_err().contains("nope"));
659 assert!(validate_exe(d.path()).unwrap_err().contains("not a regular file"));
660 let f = d.path().join("bin");
661 std::fs::write(&f, "#!/bin/sh\nexit 0\n").unwrap();
662 std::fs::set_permissions(&f, std::fs::Permissions::from_mode(0o644)).unwrap();
663 assert!(validate_exe(&f).unwrap_err().contains("not executable"));
664 std::fs::set_permissions(&f, std::fs::Permissions::from_mode(0o777)).unwrap();
665 assert!(validate_exe(&f).unwrap_err().contains("writable"));
666 std::fs::set_permissions(&f, std::fs::Permissions::from_mode(0o755)).unwrap();
667 assert_eq!(validate_exe(&f).unwrap(), f.canonicalize().unwrap());
668 let l = d.path().join("link");
670 std::os::unix::fs::symlink(&f, &l).unwrap();
671 assert_eq!(validate_exe(&l).unwrap(), f.canonicalize().unwrap());
672 }
673
674 #[test]
675 fn reload_state_path_follows_profile() {
676 let d = tempfile::tempdir().unwrap();
677 let p = registry::daemon_paths_in(d.path(), Some("work"));
678 assert!(reload_state_path(&p).ends_with("daemon-work.reload.json"), "{:?}", reload_state_path(&p));
679 let p = registry::daemon_paths_in(d.path(), None);
680 assert!(reload_state_path(&p).ends_with("daemon.reload.json"));
681 }
682}