1use std::path::{Path, PathBuf};
22use std::sync::atomic::Ordering;
23use std::sync::Arc;
24use std::time::Duration;
25
26use serde::{Deserialize, Serialize};
27
28use super::registry::{self, DaemonLock, DaemonPaths};
29use super::DaemonState;
30use crate::session::wire::{ClientFrame, CHECKPOINT_QUERY_ID, PROTOCOL_VERSION};
31use crate::session::{
32 CheckpointReason, SessionCommand, SessionConfig, SessionEventWire, SessionHandle,
33 SessionLifecycle, SessionReloadRecord, SessionSetting,
34};
35
36pub const RELOAD_STATE_ENV: &str = "SYNAPS_DAEMON_RELOAD_STATE";
38pub const LOCK_FD_ENV: &str = "SYNAPS_DAEMON_LOCK_FD";
40pub const DRAIN_SECS_ENV: &str = "SYNAPS_DAEMON_RELOAD_DRAIN_SECS";
42const DEFAULT_DRAIN: Duration = Duration::from_secs(30);
43const VERSION_PROBE_TIMEOUT: Duration = Duration::from_secs(5);
44const CHECKPOINT_BUDGET: Duration = Duration::from_secs(
47 crate::session::budgets::CANCEL_TURN_TIMEOUT_SECS
48 + crate::session::budgets::SAVE_TIMEOUT_SECS
49 + 1,
50);
51pub const RETRY_AFTER_MS: u64 = 500;
53
54#[derive(Debug, Clone, Default)]
55pub struct ReloadRequest {
56 pub now: bool,
57 pub drain_secs: Option<u64>,
58 pub exe: Option<PathBuf>,
59}
60
61impl ReloadRequest {
62 pub fn from_frame(frame: &ClientFrame) -> Option<Self> {
63 match frame {
64 ClientFrame::Reload { now, drain_secs, exe } => {
65 Some(Self { now: *now, drain_secs: *drain_secs, exe: exe.clone() })
66 }
67 _ => None,
68 }
69 }
70}
71
72#[derive(Debug)]
73pub enum ReloadError {
74 Refused(String),
76 ExecFailed(String),
78}
79
80impl std::fmt::Display for ReloadError {
81 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
82 match self {
83 Self::Refused(w) => write!(f, "reload refused: {w}"),
84 Self::ExecFailed(e) => write!(f, "reload exec failed: {e}"),
85 }
86 }
87}
88
89#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
91pub struct PrintVersion {
92 pub binary_version: String,
93 pub protocol_version: u32,
94}
95
96impl PrintVersion {
97 pub fn current() -> Self {
98 Self {
99 binary_version: crate::session::wire::binary_version(),
100 protocol_version: PROTOCOL_VERSION,
101 }
102 }
103}
104
105#[derive(Debug, Clone, Serialize, Deserialize)]
107pub struct ReloadSession {
108 pub id: String,
109 pub journal_id: String,
110 pub config: SessionConfig,
111 pub keep_warm: bool,
112 pub lifecycle: SessionLifecycle,
113 #[serde(default)]
115 pub settings_replay: Vec<SessionSetting>,
116 #[serde(default)]
118 pub cache_keepwarm: Option<String>,
119 pub input_owner_kind: Option<crate::session::ClientKind>,
120}
121
122#[derive(Debug, Clone, Serialize, Deserialize)]
124pub struct ReloadState {
125 pub generation: u64,
126 pub written_at: chrono::DateTime<chrono::Utc>,
127 pub sessions: Vec<ReloadSession>,
128 pub expected_clients: usize,
129}
130
131pub fn reload_state_path(paths: &DaemonPaths) -> PathBuf {
132 let stem = paths
133 .json
134 .file_stem()
135 .map(|s| s.to_string_lossy().into_owned())
136 .unwrap_or_else(|| "daemon".into());
137 paths.dir.join(format!("{stem}.reload.json"))
138}
139
140pub fn resolve_exe() -> PathBuf {
144 if let Some(a0) = std::env::args_os().next() {
145 let p = PathBuf::from(&a0);
146 if p.components().count() > 1 {
147 if let Ok(c) = p.canonicalize() {
148 return c;
149 }
150 }
151 }
152 std::env::current_exe().unwrap_or_else(|_| PathBuf::from("synaps"))
153}
154
155fn semver_tuple(s: &str) -> Option<(u64, u64, u64)> {
156 let core = s.split(['-', '+']).next()?;
157 let mut it = core.split('.').map(|p| p.parse::<u64>().ok());
158 Some((it.next()??, it.next()??, it.next()??))
159}
160
161pub fn version_gate(current: &PrintVersion, new: &PrintVersion) -> Result<(), String> {
165 if new.protocol_version < current.protocol_version {
166 return Err(format!(
167 "new binary speaks protocol {} < {}: reconnecting clients would be refused",
168 new.protocol_version, current.protocol_version
169 ));
170 }
171 match (semver_tuple(¤t.binary_version), semver_tuple(&new.binary_version)) {
172 (Some(c), Some(n)) if n < c => Err(format!(
173 "new binary {} is older than the running {} (reload is newer-or-equal only)",
174 new.binary_version, current.binary_version
175 )),
176 (Some(_), Some(_)) => Ok(()),
177 _ => Err(format!(
178 "cannot compare versions {:?} vs {:?}",
179 current.binary_version, new.binary_version
180 )),
181 }
182}
183
184#[cfg(unix)]
190pub fn validate_exe(exe: &Path) -> Result<PathBuf, String> {
191 use std::os::unix::fs::{MetadataExt, PermissionsExt};
192 let canon = exe
193 .canonicalize()
194 .map_err(|e| format!("--exe {}: {e}", exe.display()))?;
195 let md = std::fs::metadata(&canon).map_err(|e| format!("--exe {}: {e}", canon.display()))?;
196 if !md.is_file() {
197 return Err(format!("--exe {}: not a regular file", canon.display()));
198 }
199 let mode = md.permissions().mode();
200 if mode & 0o111 == 0 {
201 return Err(format!("--exe {}: not executable", canon.display()));
202 }
203 let uid = unsafe { libc::getuid() };
205 if md.uid() != uid && md.uid() != 0 {
206 return Err(format!(
207 "--exe {}: owned by uid {} (expected {} or root)",
208 canon.display(),
209 md.uid(),
210 uid
211 ));
212 }
213 if mode & 0o022 != 0 {
214 return Err(format!(
215 "--exe {}: group/world-writable (mode {:o})",
216 canon.display(),
217 mode & 0o777
218 ));
219 }
220 Ok(canon)
221}
222
223#[cfg(not(unix))]
224pub fn validate_exe(exe: &Path) -> Result<PathBuf, String> {
225 exe.canonicalize().map_err(|e| format!("--exe {}: {e}", exe.display()))
226}
227
228pub async fn probe_version(exe: &Path) -> Result<PrintVersion, String> {
230 let out = tokio::time::timeout(
231 VERSION_PROBE_TIMEOUT,
232 tokio::process::Command::new(exe)
233 .args(["daemon", "--print-version"])
234 .env("SYNAPS_DAEMON", "1")
235 .stdin(std::process::Stdio::null())
236 .stderr(std::process::Stdio::piped())
237 .stdout(std::process::Stdio::piped())
238 .output(),
239 )
240 .await
241 .map_err(|_| format!("{} --print-version timed out", exe.display()))?
242 .map_err(|e| format!("cannot run {}: {e}", exe.display()))?;
243 if !out.status.success() {
244 return Err(format!(
245 "{} --print-version exited {:?}: {}",
246 exe.display(),
247 out.status.code(),
248 String::from_utf8_lossy(&out.stderr).trim()
249 ));
250 }
251 serde_json::from_slice(&out.stdout)
252 .map_err(|e| format!("{} --print-version: unparsable output ({e})", exe.display()))
253}
254
255async fn checkpoint_one(handle: SessionHandle) -> Option<SessionReloadRecord> {
258 let mut rx = handle.subscribe();
259 if handle
260 .send(SessionCommand::Checkpoint { reason: CheckpointReason::Reload })
261 .await
262 .is_err()
263 {
264 return None;
265 }
266 let wait = async {
267 loop {
268 match rx.recv().await {
269 Ok(env) => {
270 if let SessionEventWire::QueryResult { id, value } = env.event {
271 if id == CHECKPOINT_QUERY_ID {
272 return serde_json::from_value(value["record"].clone()).ok();
273 }
274 }
275 }
276 Err(tokio::sync::broadcast::error::RecvError::Lagged(_)) => continue,
277 Err(tokio::sync::broadcast::error::RecvError::Closed) => return None,
278 }
279 }
280 };
281 tokio::time::timeout(CHECKPOINT_BUDGET, wait).await.unwrap_or(None)
282}
283
284fn record(handle: &SessionHandle, rec: Option<SessionReloadRecord>) -> ReloadSession {
288 let meta = handle.meta();
289 let journal_id = handle.journal_id();
290 let (config, keep_warm, lifecycle, settings_replay, cache_keepwarm, model) = match rec {
291 Some(r) => (
292 r.config,
293 r.keep_warm,
294 r.lifecycle,
295 r.settings_replay,
296 r.cache_keepwarm,
297 r.model,
298 ),
299 None => (
300 SessionConfig {
301 cwd: meta.cwd.clone(),
302 env: None,
303 env_stripped: Vec::new(),
304 ..SessionConfig::default()
305 },
306 false,
307 handle.lifecycle(),
308 Vec::new(),
309 handle.cache_keepwarm_override(),
312 handle.view().model.clone(),
313 ),
314 };
315 ReloadSession {
316 id: meta.id.as_str().to_string(),
317 journal_id: journal_id.clone(),
318 config: SessionConfig {
319 continue_session: Some(Some(journal_id)),
320 model_override: Some(model),
321 keep_warm,
322 name: None,
325 ..config
326 },
327 keep_warm,
328 lifecycle,
329 settings_replay,
330 cache_keepwarm,
331 input_owner_kind: None,
332 }
333}
334
335pub struct Prepared {
340 pub exe: PathBuf,
341 pub rs_path: PathBuf,
342 pub generation: u64,
343}
344
345pub async fn prepare(
346 state: &Arc<DaemonState>,
347 paths: &DaemonPaths,
348 req: ReloadRequest,
349) -> Result<Prepared, ReloadError> {
350 let exe = req.exe.clone().unwrap_or_else(|| {
352 registry::read_daemon_json(paths)
353 .and_then(|i| i.exe)
354 .unwrap_or_else(resolve_exe)
355 });
356 let exe = validate_exe(&exe).map_err(ReloadError::Refused)?;
357 let new = probe_version(&exe).await.map_err(ReloadError::Refused)?;
358 version_gate(&PrintVersion::current(), &new).map_err(ReloadError::Refused)?;
359 if state.reloading.swap(true, Ordering::SeqCst) {
360 return Err(ReloadError::Refused("a reload is already in progress".into()));
361 }
362 let generation = state.generation + 1;
363 tracing::info!(exe = %exe.display(), new = %new.binary_version, generation, "daemon: reload accepted");
364
365 let drain = if req.now {
367 Duration::ZERO
368 } else {
369 req.drain_secs
370 .map(Duration::from_secs)
371 .or_else(|| std::env::var(DRAIN_SECS_ENV).ok()?.parse().ok().map(Duration::from_secs))
372 .unwrap_or(DEFAULT_DRAIN)
373 };
374 let t0 = std::time::Instant::now();
375 while t0.elapsed() < drain {
376 let mut all_idle = true;
377 for h in state.live_sessions() {
378 if !super::session_is_idle(&h).await {
379 all_idle = false;
380 break;
381 }
382 }
383 if all_idle {
384 break;
385 }
386 tokio::time::sleep(Duration::from_millis(200)).await;
387 }
388 let handles = state.live_sessions();
389 let results = futures::future::join_all(handles.iter().cloned().map(checkpoint_one)).await;
390 let mut sessions = Vec::with_capacity(handles.len());
391 for (h, rec) in handles.iter().zip(results) {
392 if rec.is_none() {
393 tracing::warn!(session = %h.id, "reload: checkpoint did not confirm; recording from the handle");
394 }
395 sessions.push(record(h, rec));
396 }
397
398 let rs = ReloadState {
400 generation,
401 written_at: chrono::Utc::now(),
402 sessions,
403 expected_clients: state.connections.load(Ordering::SeqCst),
404 };
405 let rs_path = reload_state_path(paths);
406 let body = serde_json::to_vec_pretty(&rs).map_err(|e| {
411 state.reloading.store(false, Ordering::SeqCst);
412 ReloadError::ExecFailed(e.to_string())
413 })?;
414 registry::write_private_atomic(&rs_path, &body).map_err(|e| {
415 state.reloading.store(false, Ordering::SeqCst);
416 ReloadError::ExecFailed(e.to_string())
417 })?;
418
419 state.reload_generation.store(generation, Ordering::SeqCst);
422 state.fire_announce();
423 let t1 = std::time::Instant::now();
424 while state.connections.load(Ordering::SeqCst) > 1 && t1.elapsed() < Duration::from_secs(1) {
425 tokio::time::sleep(Duration::from_millis(20)).await;
426 }
427 Ok(Prepared { exe, rs_path, generation })
428}
429
430pub async fn exec(state: &Arc<DaemonState>, paths: &DaemonPaths, p: Prepared) -> ReloadError {
439 let mut info = registry::read_daemon_json(paths).unwrap_or_else(|| registry::DaemonInfo {
440 pid: std::process::id(),
441 protocol_version: PROTOCOL_VERSION,
442 daemon_version: crate::session::wire::binary_version(),
443 profile: state.profile.clone(),
444 started_at: chrono::Utc::now(),
445 socket: paths.sock.to_string_lossy().into_owned(),
446 exe: None,
447 generation: 1,
448 });
449 let prev_generation = info.generation;
450 info.generation = p.generation;
451 info.started_at = chrono::Utc::now();
452 info.exe = Some(p.exe.clone());
453 let _ = registry::write_daemon_json(paths, &info);
454 let _ = tokio::time::timeout(
455 Duration::from_secs(5),
456 async { state.host.ext_manager().write().await.shutdown_all().await },
457 )
458 .await;
459
460 let err = exec_self(&p.exe, &p.rs_path, state);
461 tracing::error!(error = %err, "daemon: reload exec failed; continuing on the old image");
462 info.generation = prev_generation;
463 let _ = registry::write_daemon_json(paths, &info);
464 let _ = std::fs::remove_file(&p.rs_path);
465 state.reload_generation.store(prev_generation, Ordering::SeqCst);
466 state.reset_announce();
467 let (tx, mut rx) = tokio::sync::mpsc::unbounded_channel();
470 let _loader = crate::extensions::loader::spawn_discover_and_load(
471 Arc::clone(state.host.ext_manager()),
472 tx,
473 None,
474 );
475 let wait = async {
476 while let Some(ev) = rx.recv().await {
477 if let crate::extensions::loader::ExtensionLoaderEvent::Finished { loaded, failed } = ev {
478 tracing::info!(loaded = loaded.len(), failed = failed.len(), "daemon: extensions re-discovered after failed reload");
479 break;
480 }
481 }
482 };
483 if tokio::time::timeout(Duration::from_secs(10), wait).await.is_err() {
484 tracing::warn!("daemon: extension re-discovery still running after 10 s");
485 }
486 state.reloading.store(false, Ordering::SeqCst);
487 ReloadError::ExecFailed(err)
488}
489
490#[cfg(unix)]
491fn exec_self(exe: &Path, rs_path: &Path, state: &DaemonState) -> String {
492 use std::os::unix::process::CommandExt;
493 let Some(lock_fd) = state.lock_fd.lock().unwrap_or_else(|e| e.into_inner()).as_ref().map(|l| l.raw_fd()) else {
494 return "no lock fd to inherit".into();
495 };
496 unsafe {
501 let flags = libc::fcntl(lock_fd, libc::F_GETFD);
502 if flags >= 0 {
503 libc::fcntl(lock_fd, libc::F_SETFD, flags & !libc::FD_CLOEXEC);
504 }
505 }
506 let mut args: Vec<std::ffi::OsString> = std::env::args_os().skip(1).collect();
507 if args.is_empty() || args.iter().all(|a| a != "daemon") {
508 args = vec!["daemon".into(), "--foreground".into()];
509 }
510 let mut cmd = std::process::Command::new(exe);
511 cmd.args(&args)
512 .env("SYNAPS_DAEMON", "1")
513 .env(LOCK_FD_ENV, lock_fd.to_string())
514 .env(RELOAD_STATE_ENV, rs_path);
515 let e = cmd.exec();
516 unsafe {
519 let flags = libc::fcntl(lock_fd, libc::F_GETFD);
520 if flags >= 0 {
521 libc::fcntl(lock_fd, libc::F_SETFD, flags | libc::FD_CLOEXEC);
522 }
523 }
524 e.to_string()
525}
526
527#[cfg(not(unix))]
528fn exec_self(_exe: &Path, _rs_path: &Path, _state: &DaemonState) -> String {
529 "reload is unix-only".into()
530}
531
532pub fn adopt_from_env() -> anyhow::Result<Option<(DaemonLock, ReloadState, PathBuf)>> {
536 let Some(rs_path) = std::env::var_os(RELOAD_STATE_ENV) else {
537 return Ok(None);
538 };
539 let rs_path = PathBuf::from(rs_path);
540 std::env::remove_var(RELOAD_STATE_ENV);
541 let fd: i32 = std::env::var(LOCK_FD_ENV)
542 .ok()
543 .and_then(|s| s.parse().ok())
544 .ok_or_else(|| anyhow::anyhow!("{RELOAD_STATE_ENV} set without {LOCK_FD_ENV}: refusing to start (the flock is the liveness oracle)"))?;
545 std::env::remove_var(LOCK_FD_ENV);
546 let lock = DaemonLock::adopt(fd)?;
547 let body = std::fs::read_to_string(&rs_path)?;
548 let rs: ReloadState = serde_json::from_str(&body)?;
549 Ok(Some((lock, rs, rs_path)))
550}
551
552fn is_lock_held_error(e: &str) -> bool {
564 e.contains("is live in another process")
565}
566
567fn parse_holder_from_error(e: &str) -> String {
569 if let Some(start) = e.find("(pid ") {
571 let after = &e[start + 1..]; if let Some(end) = after.find(')') {
573 return after[..end].to_string();
574 }
575 }
576 "unknown holder".to_string()
577}
578
579pub async fn rehydrate(state: &Arc<DaemonState>, rs: &ReloadState) {
580 for s in &rs.sessions {
581 let created = match state.create(s.config.clone()).await {
582 Ok(h) => Ok(h),
583 Err(e) if is_lock_held_error(&e) => {
586 let holder = parse_holder_from_error(&e);
587 tracing::warn!(
588 session = %s.id, holder = %holder,
589 "daemon: journal locked by another process; registering Parked placeholder"
590 );
591 let sid = crate::session::SessionId::from(s.id.as_str());
592 let (handle, _task) = crate::session::handle::locked_placeholder::spawn(
593 sid,
594 s.config.clone(),
595 holder,
596 );
597 state.insert(handle);
598 continue;
599 }
600 Err(e) if s.config.continue_session.is_some() => {
601 tracing::warn!(session = %s.id, error = %e, "daemon: journal not continuable; recreating fresh");
602 state
603 .create(SessionConfig {
604 continue_session: None,
605 ..s.config.clone()
606 })
607 .await
608 }
609 Err(e) => Err(e),
610 };
611 match created {
612 Ok(h) => {
613 let new_id = h.id.as_str().to_string();
614 if new_id != s.id {
615 state
616 .reload_aliases
617 .lock()
618 .unwrap_or_else(|e| e.into_inner())
619 .insert(s.id.clone(), new_id.clone());
620 }
621 for (i, setting) in s.settings_replay.iter().enumerate() {
623 let _ = h
624 .send(SessionCommand::Set {
625 id: u64::MAX - i as u64,
626 setting: setting.clone(),
627 })
628 .await;
629 }
630 if s.keep_warm {
631 let _ = h.send(SessionCommand::KeepWarm { on: true }).await;
632 }
633 if let Some(mode) = &s.cache_keepwarm {
634 let _ = h
635 .send(SessionCommand::EngineCommand {
636 id: u64::MAX,
637 name: "keepwarm".into(),
638 arg: mode.clone(),
639 })
640 .await;
641 }
642 if matches!(s.lifecycle, SessionLifecycle::Parked | SessionLifecycle::Parking) {
643 let _ = h.send(SessionCommand::Park).await;
644 }
645 tracing::info!(old = %s.id, new = %new_id, lifecycle = ?s.lifecycle, "daemon: session rehydrated after reload");
646 }
647 Err(e) => tracing::warn!(session = %s.id, error = %e, "daemon: session could not be rehydrated"),
648 }
649 }
650}
651
652#[cfg(test)]
653mod tests {
654 use super::*;
655
656 fn pv(b: &str, p: u32) -> PrintVersion {
657 PrintVersion { binary_version: b.into(), protocol_version: p }
658 }
659
660 #[test]
661 fn gate_is_directional_newer_or_equal() {
662 let cur = pv("0.9.0", 2);
663 assert!(version_gate(&cur, &pv("0.9.0", 2)).is_ok(), "equal allowed");
664 assert!(version_gate(&cur, &pv("0.9.1", 2)).is_ok());
665 assert!(version_gate(&cur, &pv("1.0.0-rc1", 3)).is_ok());
666 assert!(version_gate(&cur, &pv("0.8.9", 2)).is_err(), "older refused");
667 assert!(version_gate(&cur, &pv("0.9.0", 1)).is_err(), "older protocol refused");
668 assert!(version_gate(&cur, &pv("garbage", 2)).is_err());
669 assert!(
672 version_gate(&pv("0.9.1", 3), &pv("0.10.0", 3)).is_ok(),
673 "0.10.0 is newer than 0.9.1"
674 );
675 assert!(
676 version_gate(&pv("0.10.0", 3), &pv("0.9.1", 3)).is_err(),
677 "0.9.1 is older than 0.10.0"
678 );
679 }
680
681 #[cfg(unix)]
682 #[test]
683 fn validate_exe_refuses_missing_dirs_non_exec_and_writable() {
684 use std::os::unix::fs::PermissionsExt;
685 let d = tempfile::tempdir().unwrap();
686 assert!(validate_exe(&d.path().join("nope")).unwrap_err().contains("nope"));
687 assert!(validate_exe(d.path()).unwrap_err().contains("not a regular file"));
688 let f = d.path().join("bin");
689 std::fs::write(&f, "#!/bin/sh\nexit 0\n").unwrap();
690 std::fs::set_permissions(&f, std::fs::Permissions::from_mode(0o644)).unwrap();
691 assert!(validate_exe(&f).unwrap_err().contains("not executable"));
692 std::fs::set_permissions(&f, std::fs::Permissions::from_mode(0o777)).unwrap();
693 assert!(validate_exe(&f).unwrap_err().contains("writable"));
694 std::fs::set_permissions(&f, std::fs::Permissions::from_mode(0o755)).unwrap();
695 assert_eq!(validate_exe(&f).unwrap(), f.canonicalize().unwrap());
696 let l = d.path().join("link");
698 std::os::unix::fs::symlink(&f, &l).unwrap();
699 assert_eq!(validate_exe(&l).unwrap(), f.canonicalize().unwrap());
700 }
701
702 #[test]
703 fn reload_state_path_follows_profile() {
704 let d = tempfile::tempdir().unwrap();
705 let p = registry::daemon_paths_in(d.path(), Some("work"));
706 assert!(reload_state_path(&p).ends_with("daemon-work.reload.json"), "{:?}", reload_state_path(&p));
707 let p = registry::daemon_paths_in(d.path(), None);
708 assert!(reload_state_path(&p).ends_with("daemon.reload.json"));
709 }
710
711 #[test]
715 fn keepwarm_override_survives_an_unconfirmed_checkpoint() {
716 use crate::core::config::CacheKeepWarm;
717 use crate::session::handle::echo::{meta_for, view_for};
718 let id = crate::session::SessionId("kw-reload".into());
719 let (handle, ep) = SessionHandle::new(meta_for(&id), view_for());
720 assert_eq!(record(&handle, None).cache_keepwarm, None, "no override yet");
721 *ep.cache_keepwarm.lock().unwrap() = Some(CacheKeepWarm::Off);
723 assert_eq!(record(&handle, None).cache_keepwarm.as_deref(), Some("off"));
724 *ep.cache_keepwarm.lock().unwrap() = Some(CacheKeepWarm::Hours(6));
725 assert_eq!(record(&handle, None).cache_keepwarm.as_deref(), Some("6h"));
726 }
727}