Skip to main content

rigspark_runtime/
process_control.rs

1use crate::identity::{Listener, NativeProcessProbe, ProcessIdentity, ProcessProbe};
2use std::{
3    collections::BTreeMap,
4    future::Future,
5    path::{Path, PathBuf},
6    process::Stdio,
7    time::Duration,
8};
9use tokio_util::sync::CancellationToken;
10
11pub struct SpawnSpec {
12    pub binary: PathBuf,
13    pub args: Vec<String>,
14    pub env: BTreeMap<String, String>,
15}
16#[async_trait::async_trait]
17pub trait ChildProcess: Send {
18    fn pid(&self) -> u32;
19    fn exited(&mut self) -> Result<bool, String>;
20    async fn terminate(&mut self) -> Result<(), String>;
21    fn detach(&mut self);
22}
23#[async_trait::async_trait]
24pub trait ProcessControl: Send + Sync {
25    async fn occupied(&self, endpoint: &str) -> Result<bool, String>;
26    async fn spawn(&self, spec: &SpawnSpec) -> Result<Box<dyn ChildProcess>, String>;
27    async fn signal(&self, process: &ProcessIdentity, force: bool) -> Result<(), String>;
28    async fn alive(&self, pid: u32) -> Result<bool, String>;
29}
30struct NativeChild {
31    child: tokio::process::Child,
32    detached: bool,
33    pid: u32,
34}
35#[async_trait::async_trait]
36impl ChildProcess for NativeChild {
37    fn pid(&self) -> u32 {
38        self.pid
39    }
40    fn exited(&mut self) -> Result<bool, String> {
41        self.child
42            .try_wait()
43            .map(|value| value.is_some())
44            .map_err(|_| "cannot observe child exit".into())
45    }
46    async fn terminate(&mut self) -> Result<(), String> {
47        if self.exited()? {
48            return Ok(());
49        }
50        self.child
51            .start_kill()
52            .map_err(|_| "child termination failed")?;
53        tokio::time::timeout(Duration::from_secs(5), self.child.wait())
54            .await
55            .map_err(|_| "child termination timed out")?
56            .map_err(|_| "child wait failed")?;
57        Ok(())
58    }
59    fn detach(&mut self) {
60        self.detached = true;
61    }
62}
63impl Drop for NativeChild {
64    fn drop(&mut self) {
65        if !self.detached {
66            let _ = self.child.start_kill();
67        }
68    }
69}
70pub struct NativeProcessControl;
71#[async_trait::async_trait]
72impl ProcessControl for NativeProcessControl {
73    async fn occupied(&self, endpoint: &str) -> Result<bool, String> {
74        let url = crate::state::loopback(endpoint).map_err(|error| error.to_string())?;
75        let host = url
76            .host_str()
77            .ok_or("missing host")?
78            .trim_matches(['[', ']']);
79        let port = url.port_or_known_default().ok_or("missing port")?;
80        let addresses = if host == "localhost" {
81            vec![format!("127.0.0.1:{port}"), format!("[::1]:{port}")]
82        } else {
83            vec![format!(
84                "{}:{port}",
85                if host.contains(':') {
86                    format!("[{host}]")
87                } else {
88                    host.into()
89                }
90            )]
91        };
92        for address in addresses {
93            match tokio::time::timeout(
94                Duration::from_secs(2),
95                tokio::net::TcpStream::connect(address),
96            )
97            .await
98            {
99                Ok(Ok(_)) => return Ok(true),
100                Ok(Err(error)) if error.kind() == std::io::ErrorKind::ConnectionRefused => (),
101                _ => return Err("cannot establish port availability".into()),
102            }
103        }
104        Ok(false)
105    }
106    async fn spawn(&self, spec: &SpawnSpec) -> Result<Box<dyn ChildProcess>, String> {
107        if !spec.binary.is_absolute() || spec.args.iter().any(|arg| arg.contains('\0')) {
108            return Err("invalid spawn specification".into());
109        }
110        let mut command = tokio::process::Command::new(&spec.binary);
111        command
112            .args(&spec.args)
113            .env_clear()
114            .envs(&spec.env)
115            .stdin(Stdio::null())
116            .stdout(Stdio::null())
117            .stderr(Stdio::null());
118        let mut child = command.spawn().map_err(|_| "runtime spawn failed")?;
119        let Some(pid) = child.id().filter(|pid| *pid > 0) else {
120            let _ = child.kill().await;
121            return Err("runtime reported invalid PID".into());
122        };
123        Ok(Box::new(NativeChild {
124            child,
125            detached: false,
126            pid,
127        }))
128    }
129    async fn signal(&self, expected: &ProcessIdentity, force: bool) -> Result<(), String> {
130        let observed = NativeProcessProbe
131            .process(expected.pid)
132            .await
133            .map_err(|_| "cannot verify process before signal")?;
134        if !same_process(&observed, expected) {
135            return Err("process changed before signal".into());
136        }
137        #[cfg(unix)]
138        {
139            let pid = i32::try_from(expected.pid)
140                .ok()
141                .and_then(rustix::process::Pid::from_raw)
142                .ok_or("invalid process PID")?;
143            rustix::process::kill_process(
144                pid,
145                if force {
146                    rustix::process::Signal::KILL
147                } else {
148                    rustix::process::Signal::TERM
149                },
150            )
151            .map_err(|_| "signal failed".to_owned())
152        }
153        #[cfg(windows)]
154        {
155            let _ = force;
156            let system = sysinfo::System::new_all();
157            let process = system
158                .process(sysinfo::Pid::from_u32(expected.pid))
159                .ok_or("process unavailable")?;
160            if process.kill() {
161                Ok(())
162            } else {
163                Err("process termination failed".into())
164            }
165        }
166        #[cfg(not(any(unix, windows)))]
167        {
168            let _ = force;
169            Err("unsupported platform".into())
170        }
171    }
172    async fn alive(&self, pid: u32) -> Result<bool, String> {
173        #[cfg(unix)]
174        {
175            let pid = i32::try_from(pid)
176                .ok()
177                .and_then(rustix::process::Pid::from_raw)
178                .ok_or("invalid process PID")?;
179            match rustix::process::test_kill_process(pid) {
180                Ok(()) => Ok(true),
181                Err(rustix::io::Errno::SRCH) => Ok(false),
182                Err(rustix::io::Errno::PERM) => Ok(true),
183                Err(_) => Err("process existence unavailable".into()),
184            }
185        }
186        #[cfg(not(unix))]
187        {
188            let system = sysinfo::System::new_all();
189            Ok(system.process(sysinfo::Pid::from_u32(pid)).is_some())
190        }
191    }
192}
193pub fn same_process(actual: &ProcessIdentity, expected: &ProcessIdentity) -> bool {
194    actual.pid == expected.pid
195        && actual.executable == expected.executable
196        && actual.started == expected.started
197}
198/// Package-manager directories searched after `PATH` once
199/// [`enable_package_manager_fallback`] is called: apps opened from Finder or a launcher
200/// inherit only the system `PATH` and would otherwise miss Homebrew runtimes.
201#[cfg(target_os = "macos")]
202pub const FALLBACK_BINARY_DIRS: &[&str] = &["/opt/homebrew/bin", "/usr/local/bin"];
203#[cfg(target_os = "linux")]
204pub const FALLBACK_BINARY_DIRS: &[&str] = &["/usr/local/bin", "/home/linuxbrew/.linuxbrew/bin"];
205#[cfg(not(any(target_os = "macos", target_os = "linux")))]
206pub const FALLBACK_BINARY_DIRS: &[&str] = &[];
207
208static PACKAGE_MANAGER_FALLBACK: std::sync::atomic::AtomicBool =
209    std::sync::atomic::AtomicBool::new(false);
210
211/// Lets a GUI-launched app, whose `PATH` is the system default, also find runtimes in
212/// [`FALLBACK_BINARY_DIRS`]. Command-line use keeps the caller's `PATH` exactly.
213pub fn enable_package_manager_fallback() {
214    PACKAGE_MANAGER_FALLBACK.store(true, std::sync::atomic::Ordering::Relaxed);
215}
216
217/// Runtimes an app may ship inside its own bundle; nothing else resolves from there.
218pub const BUNDLED_RUNTIMES: &[&str] = &["llama-server"];
219
220static BUNDLED_RUNTIME_DIR: std::sync::OnceLock<PathBuf> = std::sync::OnceLock::new();
221
222/// Registers the directory holding an app's bundled [`BUNDLED_RUNTIMES`]. They are
223/// searched last, so a runtime the user installed still wins.
224pub fn register_bundled_runtime_dir(dir: PathBuf) -> Result<(), String> {
225    if !dir.is_absolute() {
226        return Err("bundled runtime directory must be absolute".into());
227    }
228    BUNDLED_RUNTIME_DIR
229        .set(dir)
230        .map_err(|_| "bundled runtime directory is already registered".into())
231}
232
233/// Whether `binary` resolves to the copy shipped in the registered app bundle.
234pub fn resolves_to_bundled(binary: &str) -> bool {
235    let (Some(dir), Ok(path)) = (BUNDLED_RUNTIME_DIR.get(), resolve_binary(binary)) else {
236        return false;
237    };
238    dir.canonicalize().is_ok_and(|dir| path.starts_with(dir))
239}
240
241/// Directories searched after `PATH` for `binary`, in order.
242pub fn fallback_directories<'a>(
243    binary: &str,
244    package_manager: &[&'a Path],
245    bundled: Option<&'a Path>,
246) -> Vec<&'a Path> {
247    let mut dirs = package_manager.to_vec();
248    if BUNDLED_RUNTIMES.contains(&binary)
249        && let Some(dir) = bundled.filter(|dir| dir.is_absolute())
250    {
251        dirs.push(dir);
252    }
253    dirs
254}
255
256pub fn resolve_binary(binary: &str) -> Result<PathBuf, String> {
257    let package_manager: Vec<&Path> =
258        if PACKAGE_MANAGER_FALLBACK.load(std::sync::atomic::Ordering::Relaxed) {
259            FALLBACK_BINARY_DIRS.iter().map(Path::new).collect()
260        } else {
261            Vec::new()
262        };
263    let fallbacks = fallback_directories(
264        binary,
265        &package_manager,
266        BUNDLED_RUNTIME_DIR.get().map(PathBuf::as_path),
267    );
268    resolve_binary_in(
269        binary,
270        &std::env::var_os("PATH").unwrap_or_default(),
271        &fallbacks,
272    )
273}
274
275pub fn resolve_binary_in(
276    binary: &str,
277    path_var: &std::ffi::OsStr,
278    fallbacks: &[&Path],
279) -> Result<PathBuf, String> {
280    let path = Path::new(binary);
281    if path.is_absolute() {
282        return path
283            .canonicalize()
284            .map_err(|_| "runtime executable unavailable".into());
285    }
286
287    let fixed = fallbacks.iter().map(|dir| dir.to_path_buf());
288    for directory in std::env::split_paths(path_var).chain(fixed) {
289        if directory.as_os_str().is_empty() || !directory.is_absolute() {
290            continue;
291        }
292        for name in [binary.to_owned(), format!("{binary}.exe")] {
293            let candidate = directory.join(name);
294            if candidate.is_file() {
295                return candidate
296                    .canonicalize()
297                    .map_err(|_| "runtime executable cannot be resolved".into());
298            }
299        }
300    }
301    Err("runtime executable not found on PATH".into())
302}
303pub fn bitnet_server_path(value: Option<&std::ffi::OsStr>) -> Result<PathBuf, String> {
304    let invalid = || {
305        "Set RIGSPARK_BITNET_SERVER to the absolute path of a compiled BitNet llama-server executable (not stock llama.cpp)".to_owned()
306    };
307    let path = Path::new(value.ok_or_else(invalid)?);
308    if !path.is_absolute()
309        || path
310            .to_str()
311            .is_none_or(|text| text.chars().any(char::is_control))
312        || !path.is_file()
313    {
314        return Err(invalid());
315    }
316    let path = path.canonicalize().map_err(|_| invalid())?;
317    #[cfg(unix)]
318    {
319        use std::os::unix::fs::PermissionsExt;
320        if path.metadata().map_err(|_| invalid())?.permissions().mode() & 0o111 == 0 {
321            return Err(invalid());
322        }
323    }
324    Ok(path)
325}
326
327pub fn resolve_bitnet_server() -> Result<PathBuf, String> {
328    bitnet_server_path(std::env::var_os("RIGSPARK_BITNET_SERVER").as_deref())
329}
330
331pub fn minimal_env() -> BTreeMap<String, String> {
332    [
333        "PATH",
334        "HOME",
335        "USER",
336        "LOGNAME",
337        "LANG",
338        "LC_ALL",
339        "LC_CTYPE",
340        "TMPDIR",
341        "TMP",
342        "TEMP",
343        "SYSTEMROOT",
344        "WINDIR",
345        "COMSPEC",
346        "PATHEXT",
347    ]
348    .into_iter()
349    .filter_map(|key| std::env::var(key).ok().map(|value| (key.into(), value)))
350    .collect()
351}
352pub async fn listener(
353    probe: &dyn ProcessProbe,
354    endpoint: &str,
355    cancel: &CancellationToken,
356) -> Result<Listener, String> {
357    let url = crate::state::loopback(endpoint).map_err(|error| error.to_string())?;
358    let host = url.host_str().ok_or("missing host")?;
359    let port = url.port_or_known_default().ok_or("missing port")?;
360    tokio::select! {biased;_=cancel.cancelled()=>Err("cancelled".into()),result=tokio::time::timeout(Duration::from_secs(5),probe.listener(port,host))=>{
361        let observed=result.map_err(|_|"listener probe timed out")?.map_err(|_|"listener identity unavailable")?;
362        crate::identity::choose_listener(std::slice::from_ref(&observed),port,host).map_err(|_|"listener address mismatch".into())
363    }}
364}
365pub async fn wait_owned<F, Fut>(
366    child: Box<dyn ChildProcess>,
367    endpoint: &str,
368    executable: &str,
369    probe: &dyn ProcessProbe,
370    cancel: &CancellationToken,
371    ready: F,
372) -> Result<Listener, String>
373where
374    F: FnOnce() -> Fut,
375    Fut: Future<Output = Result<(), String>>,
376{
377    wait_owned_with_timeout(
378        child,
379        endpoint,
380        executable,
381        probe,
382        cancel,
383        Duration::from_secs(30),
384        ready,
385    )
386    .await
387}
388pub async fn wait_owned_with_timeout<F, Fut>(
389    mut child: Box<dyn ChildProcess>,
390    endpoint: &str,
391    executable: &str,
392    probe: &dyn ProcessProbe,
393    cancel: &CancellationToken,
394    timeout: Duration,
395    ready: F,
396) -> Result<Listener, String>
397where
398    F: FnOnce() -> Fut,
399    Fut: Future<Output = Result<(), String>>,
400{
401    if timeout.is_zero() || timeout > Duration::from_secs(300) {
402        child.terminate().await?;
403        return Err("invalid readiness deadline".into());
404    }
405    let operation = async {
406        if child.pid() == 0 {
407            return Err("invalid child PID".into());
408        }
409        let readiness = ready();
410        tokio::pin!(readiness);
411        loop {
412            if child.exited()? {
413                return Err("runtime exited before readiness".into());
414            }
415            tokio::select! {result=&mut readiness=>{result?;break;},_=tokio::time::sleep(Duration::from_millis(50))=>()}
416        }
417        let observed = listener(probe, endpoint, cancel).await?;
418        if observed.identity.pid != child.pid() || observed.identity.executable != executable {
419            return Err("readiness belongs to another process".into());
420        }
421        Ok(observed)
422    };
423    let result = tokio::select! {biased;_=cancel.cancelled()=>Err("cancelled".into()),result=tokio::time::timeout(timeout,operation)=>result.map_err(|_|"runtime readiness timed out".to_owned()).and_then(|result|result)};
424    match result {
425        Ok(observed) => {
426            child.detach();
427            Ok(observed)
428        }
429        Err(error) => {
430            child
431                .terminate()
432                .await
433                .map_err(|cleanup| format!("{error}; {cleanup}"))?;
434            Err(error)
435        }
436    }
437}
438/// Polls after a graceful signal before escalating: model servers can take a few
439/// seconds to unload weights (50 ms each, so 3 s).
440const GRACEFUL_POLLS: usize = 60;
441/// Polls after a forced signal (50 ms each, so 0.5 s).
442const FORCED_POLLS: usize = 10;
443
444/// Stop a runtime RigSpark started, identified by `expected`.
445///
446/// Fails closed: a process is only signalled once its identity matches
447/// `expected`. A stale handle is not an error: if nothing listens on the
448/// endpoint and the recorded process is gone (or its PID now belongs to another
449/// program), the runtime is already stopped.
450pub async fn stop_owned(
451    endpoint: &str,
452    expected: &ProcessIdentity,
453    probe: &dyn ProcessProbe,
454    control: &dyn ProcessControl,
455    cancel: &CancellationToken,
456) -> Result<(), String> {
457    match listener(probe, endpoint, cancel).await {
458        Ok(observed) => {
459            if !same_process(&observed.identity, expected) {
460                return Err("listener changed; refusing to stop".into());
461            }
462        }
463        Err(error) if cancel.is_cancelled() => return Err(error),
464        Err(error) => {
465            // Nothing (identifiable) listens. Decide from the recorded process alone.
466            if !control.alive(expected.pid).await? {
467                return Ok(());
468            }
469            match probe.process(expected.pid).await {
470                Ok(current) if !same_process(&current, expected) => return Ok(()),
471                Ok(_) => {} // still our process, just not listening: stop it below
472                Err(_) => return Err(error),
473            }
474        }
475    }
476    for (force, polls) in [(false, GRACEFUL_POLLS), (true, FORCED_POLLS)] {
477        let probed = tokio::select! {biased;_=cancel.cancelled()=>return Err("cancelled".into()),result=tokio::time::timeout(Duration::from_secs(5),probe.process(expected.pid))=>result.map_err(|_|"process probe timed out")?};
478        let current = match probed {
479            Ok(current) => current,
480            Err(_) if !control.alive(expected.pid).await? => return Ok(()),
481            Err(_) => {
482                return Err(if force {
483                    "process identity unavailable after signal".into()
484                } else {
485                    "process identity unavailable".into()
486                });
487            }
488        };
489        if !same_process(&current, expected) {
490            if force {
491                return Ok(());
492            }
493            return Err("process changed; refusing to stop".into());
494        }
495        control.signal(expected, force).await?;
496        for _ in 0..polls {
497            if !control.alive(expected.pid).await? {
498                return Ok(());
499            }
500            // An exiting process can briefly fail identity probes (zombie, or
501            // mid-teardown). That's not a failure: keep waiting. The next pass
502            // re-checks identity before any forced signal.
503            if let Ok(current) = probe.process(expected.pid).await
504                && !same_process(&current, expected)
505            {
506                return Ok(());
507            }
508            tokio::select! {biased;_=cancel.cancelled()=>return Err("cancelled".into()),_=tokio::time::sleep(Duration::from_millis(50))=>()}
509        }
510    }
511    Err("runtime remains alive after termination".into())
512}