Skip to main content

magi/
worker.rs

1//! Durable detached execution, independent of the scheduler and web lifetimes.
2//!
3//! A reservation is published before spawn, and the child must take its execution
4//! lease and acknowledge that reservation before dispatch. Reconciliation cancels
5//! an unacknowledged startup under the same store lock the child needs: a late
6//! child can never execute a cancelled operation. Process IDs are diagnostic only.
7//! Metadata publication closes its staging handle before atomic replacement.
8//! Windows replacement/delete-pending handoffs use the short, bounded lock-file
9//! retry policy; failed publication never releases an uncertain reservation.
10
11use std::fs::{File, OpenOptions};
12use std::io::{Read, Write};
13use std::path::{Path, PathBuf};
14use std::process::{Child, Command, Stdio};
15use std::sync::{LazyLock, Mutex, OnceLock};
16use std::time::{Duration, SystemTime, UNIX_EPOCH};
17
18use anyhow::{Context, Result, bail};
19use serde::{Deserialize, Serialize};
20use sha2::{Digest, Sha256};
21
22const STARTUP_SECONDS: u64 = 30;
23const RETAIN_FINISHED: usize = 256;
24
25/// Identity and executable selection inherited by descendants of a worker.
26/// Verification commands and standalone test fixtures remove these overrides:
27/// they test the checkout, rather than act on behalf of the enclosing worker.
28/// Keep runtime storage, seat metadata and process containment context intact.
29pub const INVOCATION_ENV: [&str; 4] = [
30    "MAGI_WORKER_ID",
31    "MAGI_WORKER_GENERATION",
32    "MAGI_WORKER_HOME",
33    "MAGI_WORKER_EXE",
34];
35
36/// Durable execution state. Starting reservations count as occupied slots.
37#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
38#[serde(rename_all = "snake_case")]
39pub enum Status {
40    /// Published but not yet acknowledged by the detached child.
41    Starting,
42    /// The child acknowledged startup and owns the execution lease.
43    Running,
44    /// The handler returned successfully.
45    Finished,
46    /// Startup, handler execution, or the owning process failed.
47    Failed,
48}
49
50/// An explicit owner control; upgrades do not send these to task workers.
51#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
52#[serde(rename_all = "snake_case")]
53pub enum Control {
54    /// Park at the handler's existing safe cancellation boundary.
55    Park,
56    /// Stop the operation through its handler's cancellation path.
57    Stop,
58}
59
60#[derive(Debug)]
61struct NotStarted(anyhow::Error);
62
63impl std::fmt::Display for NotStarted {
64    fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
65        write!(formatter, "{:#}", self.0)
66    }
67}
68
69impl std::error::Error for NotStarted {
70    fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
71        Some(self.0.as_ref())
72    }
73}
74
75fn not_started_error(error: impl Into<anyhow::Error>) -> anyhow::Error {
76    anyhow::Error::new(NotStarted(error.into()))
77}
78
79/// Whether this spawn invocation provably did not execute its handler. This
80/// covers preparation failures and a failed OS spawn whose fresh reservation
81/// was exclusively leased and durably fenced before this error was returned.
82/// Only a fresh operation key makes this proof sufficient to restore a caller's
83/// publication. Unknown startup, existing work, or a failed fence is never tagged.
84pub fn not_started(error: &anyhow::Error) -> bool {
85    error.downcast_ref::<NotStarted>().is_some()
86}
87
88/// The wait timed out, but this reservation is still eligible to start.
89#[derive(Debug, thiserror::Error)]
90#[error(
91    "worker {id} startup is unacknowledged; its reservation remains occupied; inspect or retry the same operation key"
92)]
93pub(crate) struct StartupUnacknowledged {
94    pub(crate) id: String,
95}
96
97/// A durable operation record. Its payload belongs to the started generation.
98#[derive(Clone, Debug, Serialize, Deserialize)]
99pub struct Job {
100    /// Unique invocation identity.
101    pub id: String,
102    /// Durable total reservation order, independent of wall clock resolution.
103    #[serde(default)]
104    pub sequence: u64,
105    /// Dispatch handler, such as task, chat, scheduler, or service.
106    pub kind: String,
107    /// Exclusive logical operation identity, independent of executable generation.
108    pub key: String,
109    /// Handler-owned request.
110    pub payload: serde_json::Value,
111    /// SHA-256 identity of the immutable executable.
112    pub generation: String,
113    /// Version reported by the started executable, unknown before acknowledgement.
114    #[serde(default)]
115    pub version: Option<String>,
116    /// Last durable execution state.
117    pub status: Status,
118    /// Successful completion note or failure diagnostic.
119    pub result: Option<String>,
120    /// Diagnostic process ID, never used as an execution witness.
121    pub pid: Option<u32>,
122    /// Reservation publication time, seconds since the Unix epoch.
123    pub created_at: u64,
124    /// Terminal state publication time.
125    pub finished_at: Option<u64>,
126}
127
128/// An immutable, content-addressed executable.
129#[derive(Clone, Debug)]
130pub struct Generation {
131    /// SHA-256 digest of the executable bytes.
132    pub id: String,
133    /// Absolute path to the retained executable.
134    pub executable: PathBuf,
135}
136
137/// OS-held lock. Closing it, including on a crash, releases ownership.
138#[derive(Debug)]
139pub struct Lease {
140    _file: File,
141}
142
143impl Drop for Lease {
144    fn drop(&mut self) {
145        // flock's lock belongs to the open-file description; explicitly unlock
146        // so a temporarily inherited descriptor cannot prolong ownership.
147        #[cfg(unix)]
148        {
149            use std::os::fd::AsRawFd;
150            unsafe { libc::flock(self._file.as_raw_fd(), libc::LOCK_UN) };
151        }
152    }
153}
154
155fn root(home: &Path) -> PathBuf {
156    home.join("workers")
157}
158fn now() -> u64 {
159    SystemTime::now()
160        .duration_since(UNIX_EPOCH)
161        .unwrap_or_default()
162        .as_secs()
163}
164fn nonce() -> String {
165    crate::rng::SplitMix64::new(crate::rng::entropy()).uuid_v4()
166}
167fn valid_id(id: &str) -> bool {
168    !id.is_empty() && id.bytes().all(|b| b.is_ascii_hexdigit() || b == b'-')
169}
170fn job_dir(home: &Path, id: &str) -> Result<PathBuf> {
171    if !valid_id(id) {
172        bail!("invalid worker id");
173    }
174    Ok(root(home).join("jobs").join(id))
175}
176
177fn open_lock(path: &Path) -> Result<File> {
178    if let Some(parent) = path.parent() {
179        std::fs::create_dir_all(parent)
180            .with_context(|| format!("create worker lock directory {}", parent.display()))?;
181    }
182    OpenOptions::new()
183        .create(true)
184        .truncate(false)
185        .read(true)
186        .write(true)
187        .open(path)
188        .with_context(|| format!("open worker lock {}", path.display()))
189}
190
191fn try_lock(path: &Path) -> Result<Option<Lease>> {
192    let file = open_lock(path)?;
193    match crate::pr_head::try_lock_file(&file) {
194        Ok(()) => Ok(Some(Lease { _file: file })),
195        Err(e)
196            if e.kind() == std::io::ErrorKind::WouldBlock
197                || matches!(e.raw_os_error(), Some(33)) =>
198        {
199            Ok(None)
200        }
201        Err(e) => Err(e).with_context(|| format!("acquire worker lock {}", path.display())),
202    }
203}
204
205#[cfg(test)]
206pub(crate) fn execution_lease_for_test(path: &Path) -> Lease {
207    try_lock(path)
208        .expect("execution lease")
209        .expect("free execution lease")
210}
211
212/// Try to own a named cross-process singleton, without PID-based stale handling.
213/// Names are hashed, so callers may use logical keys containing paths or colons.
214pub fn try_lease(home: &Path, name: &str) -> Result<Option<Lease>> {
215    try_lock(
216        &root(home)
217            .join("leases")
218            .join(format!("{}.lock", digest(name.as_bytes()))),
219    )
220}
221
222fn store_lock(home: &Path) -> Result<Lease> {
223    let deadline = std::time::Instant::now() + Duration::from_secs(10);
224    loop {
225        if let Some(lease) = try_lock(&root(home).join("store.lock"))? {
226            return Ok(lease);
227        }
228        if std::time::Instant::now() >= deadline {
229            bail!("worker store is busy; retry later");
230        }
231        std::thread::sleep(Duration::from_millis(5));
232    }
233}
234
235#[cfg(unix)]
236fn sync_dir(path: &Path) -> Result<()> {
237    File::open(path)
238        .with_context(|| format!("open worker directory for sync {}", path.display()))?
239        .sync_all()
240        .with_context(|| format!("sync worker directory {}", path.display()))?;
241    Ok(())
242}
243
244#[cfg(not(unix))]
245fn sync_dir(_path: &Path) -> Result<()> {
246    Ok(())
247}
248
249fn save(path: &Path, value: &impl Serialize) -> Result<()> {
250    let parent = path.parent().context("worker record has no parent")?;
251    std::fs::create_dir_all(parent)
252        .with_context(|| format!("create worker record directory {}", parent.display()))?;
253    let temp = parent.join(format!(".pending-{}", nonce()));
254    let result = (|| -> Result<()> {
255        let mut file = OpenOptions::new()
256            .create_new(true)
257            .write(true)
258            .open(&temp)
259            .with_context(|| format!("create staged worker record {}", temp.display()))?;
260        serde_json::to_writer(&mut file, value)
261            .with_context(|| format!("serialize worker record {}", path.display()))?;
262        file.flush()
263            .with_context(|| format!("flush staged worker record {}", temp.display()))?;
264        file.sync_all()
265            .with_context(|| format!("sync staged worker record {}", temp.display()))?;
266        // Publication must not retain our own writable staging handle. Windows
267        // readers may briefly deny deletion; only the atomic handoff retries.
268        drop(file);
269        replace(&temp, path).with_context(|| {
270            format!(
271                "publish worker record {} -> {}",
272                temp.display(),
273                path.display()
274            )
275        })?;
276        sync_dir(parent)
277    })();
278    if result.is_err() {
279        let _ = std::fs::remove_file(&temp);
280    }
281    result
282}
283
284#[cfg(not(windows))]
285fn replace(from: &Path, to: &Path) -> Result<()> {
286    Ok(std::fs::rename(from, to)?)
287}
288
289#[cfg(windows)]
290fn replace(from: &Path, to: &Path) -> Result<()> {
291    // std uses FileRenameInfoEx/POSIX semantics where supported, unlike
292    // MoveFileExW: a FILE_SHARE_DELETE reader may retain the previous record
293    // while the replacement becomes visible atomically. Readers that briefly
294    // deny deletion still get only the existing short Windows handoff budget.
295    crate::lock_file::Retry::new(None).run(|| std::fs::rename(from, to))?;
296    Ok(())
297}
298
299// Atomic replacement can leave the old Windows name briefly delete-pending.
300// Keep that handoff bounded; malformed records and permanent IO errors propagate.
301fn open_record(path: &Path) -> std::io::Result<File> {
302    #[cfg(windows)]
303    {
304        crate::lock_file::Retry::new(None).run(|| File::open(path))
305    }
306    #[cfg(not(windows))]
307    {
308        File::open(path)
309    }
310}
311
312/// Load a specific invocation, including a retained terminal result.
313pub fn get(home: &Path, id: &str) -> Result<Job> {
314    let path = job_dir(home, id)?.join("job.json");
315    let file = match open_record(&path) {
316        Ok(file) => file,
317        Err(error) if error.kind() == std::io::ErrorKind::NotFound => {
318            let retained = root(home).join("results").join(id).join("job.json");
319            open_record(&retained)
320                .with_context(|| format!("open retained worker record {}", retained.display()))?
321        }
322        Err(error) => {
323            return Err(error).with_context(|| format!("open worker record {}", path.display()));
324        }
325    };
326    serde_json::from_reader(file).with_context(|| format!("decode worker record {id}"))
327}
328
329/// Identify this detached invocation. Helpers inherit its identity, while a
330/// newly spawned worker receives its own id before exec; no global environment
331/// mutation is required inside the asynchronous handler runtime.
332pub fn current_job(home: &Path) -> Result<Option<Job>> {
333    match std::env::var("MAGI_WORKER_ID") {
334        Ok(id) => Ok(Some(get(home, &id)?)),
335        Err(std::env::VarError::NotPresent) => Ok(None),
336        Err(error) => Err(error.into()),
337    }
338}
339
340fn next_sequence(home: &Path) -> Result<u64> {
341    let path = root(home).join("sequence.json");
342    let previous: u64 = match open_record(&path) {
343        Ok(file) => serde_json::from_reader(file)
344            .with_context(|| format!("decode worker sequence {}", path.display()))?,
345        Err(error) if error.kind() == std::io::ErrorKind::NotFound => 0,
346        Err(error) => {
347            return Err(error).with_context(|| format!("open worker sequence {}", path.display()));
348        }
349    };
350    let sequence = previous
351        .checked_add(1)
352        .context("worker reservation sequence exhausted")?;
353    save(&path, &sequence)?;
354    Ok(sequence)
355}
356
357fn records(home: &Path) -> Result<Vec<Job>> {
358    let directory = root(home).join("jobs");
359    std::fs::create_dir_all(&directory)?;
360    let mut jobs = Vec::new();
361    let mut retired = false;
362    for entry in std::fs::read_dir(directory)? {
363        let entry = entry?;
364        if !entry.file_type()?.is_dir() {
365            continue;
366        }
367        let name = entry.file_name();
368        let Some(id) = name.to_str() else {
369            continue;
370        };
371        if id.starts_with(".new-") {
372            // Reservations are staged only while holding the store lock. Any
373            // staging directory visible to the next owner was left by a crash.
374            std::fs::remove_dir_all(entry.path())?;
375            continue;
376        }
377        if id.starts_with('.') {
378            continue;
379        }
380        for pending in std::fs::read_dir(entry.path())? {
381            let pending = pending?;
382            if pending.file_type()?.is_file()
383                && pending
384                    .file_name()
385                    .to_string_lossy()
386                    .starts_with(".pending-")
387            {
388                std::fs::remove_file(pending.path())?;
389            }
390        }
391        let job = get(home, id).with_context(|| format!("read worker {id}"))?;
392        if matches!(job.status, Status::Starting | Status::Running) {
393            jobs.push(job);
394        } else {
395            retired |= retire(home, &job);
396        }
397    }
398    if retired {
399        if let Err(error) = prune_results(home) {
400            tracing::warn!(%error, "worker result retention deferred");
401        }
402    }
403    Ok(jobs)
404}
405
406fn terminal(home: &Path, job: &mut Job, status: Status, result: String) -> Result<()> {
407    job.status = status;
408    job.result = Some(result);
409    job.finished_at = Some(now());
410    save(&job_dir(home, &job.id)?.join("job.json"), job)
411        .with_context(|| format!("publish worker {} terminal status {status:?}", job.id))?;
412    // The result is authoritative now. Keep the directory in place while this
413    // process owns live.lock and its stdout/stderr handles. Another store owner
414    // retires it opportunistically after execution ownership is released.
415    clear_control_locked(home, &job.key)
416}
417
418fn retire(home: &Path, job: &Job) -> bool {
419    let attempt = || -> Result<bool> {
420        let dir = job_dir(home, &job.id)?;
421        let Some(lease) = try_lock(&dir.join("live.lock"))? else {
422            return Ok(false);
423        };
424        drop(lease);
425        archive(home, job)?;
426        Ok(true)
427    };
428    // Windows can still deny the directory rename in the short interval after
429    // lease release but before process stdout/stderr are closed. Retry on the
430    // next store read; archival failure must never turn completion into failure
431    // or terminate an otherwise healthy scheduler.
432    match attempt() {
433        Ok(retired) => retired,
434        Err(error) => {
435            tracing::warn!(
436                worker = %job.id,
437                error = %format_args!("{error:#}"),
438                "worker result retirement deferred"
439            );
440            false
441        }
442    }
443}
444
445fn archive(home: &Path, job: &Job) -> Result<()> {
446    let results = root(home).join("results");
447    std::fs::create_dir_all(&results)
448        .with_context(|| format!("create worker results directory {}", results.display()))?;
449    let from = job_dir(home, &job.id)?;
450    let to = results.join(&job.id);
451    std::fs::rename(&from, &to).with_context(|| {
452        format!(
453            "archive worker result {} -> {}",
454            from.display(),
455            to.display()
456        )
457    })?;
458    sync_dir(&root(home).join("jobs"))?;
459    sync_dir(&results)
460}
461
462// Caller owns the store lock. Acquiring the execution lease and then marking a
463// dead startup terminal fences a late child: it must recheck state under store.
464fn reconcile(home: &Path, jobs: &mut [Job]) -> Result<()> {
465    let mut retired = false;
466    for job in jobs {
467        if !matches!(job.status, Status::Starting | Status::Running) {
468            continue;
469        }
470        let lease = try_lock(&job_dir(home, &job.id)?.join("live.lock"))?;
471        if lease.is_none() {
472            continue;
473        }
474        if job.status == Status::Starting && now().saturating_sub(job.created_at) < STARTUP_SECONDS
475        {
476            continue;
477        }
478        let reason = if job.status == Status::Starting {
479            "worker startup reservation expired before acknowledgement"
480        } else {
481            "worker exited without recording a result"
482        };
483        terminal(home, job, Status::Failed, reason.to_owned())?;
484        drop(lease);
485        retired |= retire(home, job);
486    }
487    if retired {
488        if let Err(error) = prune_results(home) {
489            tracing::warn!(%error, "worker result retention deferred");
490        }
491    }
492    Ok(())
493}
494
495/// List live operations and unresolved startups. A missing lease only frees a
496/// startup after its deadline and an atomic cancellation fence; PID is irrelevant.
497pub fn active(home: &Path) -> Result<Vec<Job>> {
498    let _store = store_lock(home)?;
499    let mut jobs = records(home)?;
500    reconcile(home, &mut jobs)?;
501    Ok(jobs
502        .into_iter()
503        .filter(|job| matches!(job.status, Status::Starting | Status::Running))
504        .collect())
505}
506
507/// Observe execution without creating, repairing, archiving, or locking the
508/// writable store. Sandboxed diagnostics need only file reads and a read-only
509/// handle for the OS lease probe. Starting reservations remain protected until
510/// an authorized writer fences them; a missing running lease is unknown.
511pub fn observe_active(home: &Path) -> Result<Vec<Job>> {
512    let entries = match std::fs::read_dir(root(home).join("jobs")) {
513        Ok(entries) => entries,
514        Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(Vec::new()),
515        Err(error) => return Err(error.into()),
516    };
517    let mut active = Vec::new();
518    for entry in entries {
519        let entry = entry?;
520        if !entry.file_type()?.is_dir() {
521            continue;
522        }
523        let name = entry.file_name();
524        let id = name.to_str().context("invalid worker reservation name")?;
525        if id.starts_with('.') {
526            continue;
527        }
528        let job = get(home, id)?;
529        if observe_job_active(home, &job)? {
530            active.push(job);
531        }
532    }
533    Ok(active)
534}
535
536/// Passive lease observation for a single reservation already read by the caller.
537pub(crate) fn observe_job_active(home: &Path, job: &Job) -> Result<bool> {
538    match job.status {
539        Status::Starting => Ok(true),
540        Status::Running => {
541            let file = match File::open(job_dir(home, &job.id)?.join("live.lock")) {
542                Ok(file) => file,
543                Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(true),
544                Err(error) => return Err(error.into()),
545            };
546            match crate::pr_head::try_lock_file(&file) {
547                Ok(()) => {
548                    drop(Lease { _file: file });
549                    Ok(false)
550                }
551                Err(error)
552                    if error.kind() == std::io::ErrorKind::WouldBlock
553                        || cfg!(windows) && error.raw_os_error() == Some(33) =>
554                {
555                    Ok(true)
556                }
557                Err(error) => Err(error.into()),
558            }
559        }
560        Status::Finished | Status::Failed => Ok(false),
561    }
562}
563
564fn control_path(home: &Path, key: &str) -> PathBuf {
565    root(home)
566        .join("controls")
567        .join(format!("{}.json", digest(key.as_bytes())))
568}
569
570/// Persist a stop or park request; the operation's handler consumes it safely.
571pub fn request(home: &Path, key: &str, control: Control) -> Result<()> {
572    let _store = store_lock(home)?;
573    save(&control_path(home, key), &control)
574}
575
576/// Read the latest durable owner request for a logical operation.
577pub fn control(home: &Path, key: &str) -> Result<Option<Control>> {
578    let path = control_path(home, key);
579    match open_record(&path) {
580        Ok(file) => serde_json::from_reader(file)
581            .map(Some)
582            .with_context(|| format!("decode worker control {}", path.display())),
583        Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(None),
584        Err(e) => Err(e).with_context(|| format!("open worker control {}", path.display())),
585    }
586}
587
588fn clear_control_locked(home: &Path, key: &str) -> Result<()> {
589    let path = control_path(home, key);
590    match std::fs::remove_file(&path) {
591        Ok(()) => sync_dir(&root(home).join("controls")),
592        Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(()),
593        Err(e) => Err(e).with_context(|| format!("remove worker control {}", path.display())),
594    }
595}
596
597/// Clear controls at an explicitly owned new singleton invocation boundary.
598pub fn clear_control(home: &Path, key: &str) -> Result<()> {
599    let _store = store_lock(home)?;
600    clear_control_locked(home, key)
601}
602
603/// Resolve a retained executable strictly by its content identity.
604pub fn executable(home: &Path, generation: &str) -> Result<PathBuf> {
605    if generation.len() != 64 || !generation.bytes().all(|b| b.is_ascii_hexdigit()) {
606        bail!("invalid worker generation");
607    }
608    let path = root(home)
609        .join("generations")
610        .join(generation)
611        .join(if cfg!(windows) { "magi.exe" } else { "magi" });
612    if !path.is_file() {
613        bail!("worker generation {generation} is unavailable");
614    }
615    Ok(path)
616}
617
618/// Resolve helper invocations to this worker's started executable, not a newly
619/// installed binary. Outside a worker, pin the caller's current executable.
620pub fn helper_executable(home: &Path) -> Result<PathBuf> {
621    if let Some(generation) = std::env::var_os("MAGI_WORKER_GENERATION") {
622        return executable(
623            home,
624            generation
625                .to_str()
626                .context("invalid generation environment")?,
627        );
628    }
629    Ok(pin_current(home)?.executable)
630}
631
632/// Remember the installed executable pathname before an updater replaces it.
633/// A web process calls this at startup so Linux's deleted-executable marker or
634/// Windows's backup rename cannot redirect successor workers to the old binary.
635pub fn remember_current_executable() -> Result<()> {
636    if INSTALLED_EXECUTABLE.get().is_some() {
637        return Ok(());
638    }
639    let path = std::env::current_exe()?.canonicalize()?;
640    let _ = INSTALLED_EXECUTABLE.set(path);
641    Ok(())
642}
643
644static INSTALLED_EXECUTABLE: OnceLock<PathBuf> = OnceLock::new();
645
646/// Copy and SHA-256 address the executable. MAGI_WORKER_EXE is an explicit
647/// embedding/test override; it is never an in-process execution fallback.
648pub fn pin_current(home: &Path) -> Result<Generation> {
649    let source = match std::env::var_os("MAGI_WORKER_EXE") {
650        Some(path) => PathBuf::from(path),
651        None => match INSTALLED_EXECUTABLE.get() {
652            Some(path) => path.clone(),
653            None => std::env::current_exe()?,
654        },
655    };
656    let source = source.canonicalize().context("resolve worker executable")?;
657    // A worker already runs the immutable copy. Do not copy/hash a large binary
658    // again for every finite maintenance invocation in the same generation.
659    if let Some(directory) = source.parent() {
660        if directory.parent() == Some(root(home).join("generations").as_path()) {
661            if let Some(id) = directory.file_name().and_then(|name| name.to_str()) {
662                if id.len() == 64
663                    && id.bytes().all(|b| b.is_ascii_hexdigit())
664                    && executable(home, id)?.canonicalize()? == source
665                {
666                    return Ok(Generation {
667                        id: id.to_owned(),
668                        executable: source,
669                    });
670                }
671            }
672        }
673    }
674    let stamp = source_stamp(&source)?;
675    let mut cache = PIN_CACHE
676        .lock()
677        .map_err(|_| anyhow::anyhow!("worker generation cache poisoned"))?;
678    if let Some(entry) = cache.iter().find(|entry| {
679        entry.home == home
680            && entry.source == source
681            && entry.stamp == stamp
682            && entry.generation.executable.is_file()
683    }) {
684        return Ok(entry.generation.clone());
685    }
686    let generation = pin_source(home, &source)?;
687    if source_stamp(&source)? != stamp {
688        bail!("worker executable changed while being pinned; retry");
689    }
690    if cache.len() >= 8 {
691        cache.remove(0);
692    }
693    cache.push(CachedGeneration {
694        home: home.to_path_buf(),
695        source,
696        stamp,
697        generation: generation.clone(),
698    });
699    Ok(generation)
700}
701
702type SourceStamp = (u64, Option<SystemTime>, Option<SystemTime>, u64, u64);
703struct CachedGeneration {
704    home: PathBuf,
705    source: PathBuf,
706    stamp: SourceStamp,
707    generation: Generation,
708}
709static PIN_CACHE: LazyLock<Mutex<Vec<CachedGeneration>>> = LazyLock::new(|| Mutex::new(Vec::new()));
710
711fn source_stamp(path: &Path) -> Result<SourceStamp> {
712    let metadata = std::fs::metadata(path)?;
713    #[cfg(unix)]
714    let identity = {
715        use std::os::unix::fs::MetadataExt;
716        (metadata.dev(), metadata.ino())
717    };
718    #[cfg(not(unix))]
719    let identity = (0, 0);
720    Ok((
721        metadata.len(),
722        metadata.modified().ok(),
723        metadata.created().ok(),
724        identity.0,
725        identity.1,
726    ))
727}
728
729fn pin_source(home: &Path, source: &Path) -> Result<Generation> {
730    let generations = root(home).join("generations");
731    std::fs::create_dir_all(&generations)?;
732    let staging = generations.join(format!(".new-{}", nonce()));
733    let mut staging_lease = {
734        let _store = store_lock(home)?;
735        // Only unpublished copies are reclaimed, never executable generations.
736        for entry in std::fs::read_dir(&generations)? {
737            let entry = entry?;
738            if entry.file_type()?.is_dir()
739                && entry.file_name().to_string_lossy().starts_with(".new-")
740            {
741                if let Some(lease) = try_lock(&entry.path().join("pin.lock"))? {
742                    drop(lease);
743                    remove_staging(&entry.path())?;
744                }
745            }
746        }
747        std::fs::create_dir(&staging)?;
748        Some(try_lock(&staging.join("pin.lock"))?.context("generation staging lease unavailable")?)
749    };
750    let temp = staging.join(if cfg!(windows) { "magi.exe" } else { "magi" });
751    let result = (|| -> Result<Generation> {
752        let mut input = File::open(source)?;
753        let mut output = OpenOptions::new()
754            .create_new(true)
755            .write(true)
756            .open(&temp)?;
757        let mut hash = Sha256::new();
758        let mut buffer = [0u8; 65536];
759        loop {
760            let n = input.read(&mut buffer)?;
761            if n == 0 {
762                break;
763            }
764            hash.update(&buffer[..n]);
765            output.write_all(&buffer[..n])?;
766        }
767        let id = hex_digest(hash);
768        let mut permissions = input.metadata()?.permissions();
769        permissions.set_readonly(true);
770        output.set_permissions(permissions)?;
771        output.sync_all()?;
772        drop(output);
773        sync_dir(&staging)?;
774        let target = generations.join(&id);
775        // The store lock serializes publication with retention and other pins.
776        let _store = store_lock(home)?;
777        drop(staging_lease.take());
778        std::fs::remove_file(staging.join("pin.lock"))?;
779        if target.exists() {
780            let existing = target.join(temp.file_name().context("binary filename")?);
781            if hash_file(&existing)? != id {
782                bail!("immutable worker generation is corrupt: {id}");
783            }
784        } else {
785            std::fs::rename(&staging, &target)?;
786            sync_dir(&generations)?;
787        }
788        Ok(Generation {
789            executable: target.join(temp.file_name().context("binary filename")?),
790            id,
791        })
792    })();
793    let _store = store_lock(home)?;
794    drop(staging_lease);
795    if staging.exists() {
796        remove_staging(&staging)?;
797    }
798    result
799}
800
801fn remove_staging(staging: &Path) -> Result<()> {
802    #[cfg(windows)]
803    {
804        use std::os::windows::{ffi::OsStrExt, fs::MetadataExt};
805        #[link(name = "kernel32")]
806        unsafe extern "system" {
807            fn SetFileAttributesW(path: *const u16, attributes: u32) -> i32;
808        }
809        const READONLY: u32 = 0x1;
810        for entry in std::fs::read_dir(staging)? {
811            let entry = entry?;
812            let attributes = entry.metadata()?.file_attributes();
813            if attributes & READONLY != 0 {
814                let path: Vec<u16> = entry.path().as_os_str().encode_wide().chain([0]).collect();
815                // SAFETY: path is a live, NUL-terminated UTF-16 buffer. Clear
816                // only the Windows readonly attribute, preserving other bits.
817                if unsafe { SetFileAttributesW(path.as_ptr(), attributes & !READONLY) } == 0 {
818                    return Err(std::io::Error::last_os_error())
819                        .context("clear staging readonly attribute");
820                }
821            }
822        }
823    }
824    Ok(std::fs::remove_dir_all(staging)?)
825}
826
827fn hash_file(path: &Path) -> Result<String> {
828    let mut file = File::open(path)?;
829    let mut hash = Sha256::new();
830    let mut buffer = [0u8; 65536];
831    loop {
832        let n = file.read(&mut buffer)?;
833        if n == 0 {
834            break;
835        }
836        hash.update(&buffer[..n]);
837    }
838    Ok(hex_digest(hash))
839}
840
841fn reserve(
842    home: &Path,
843    generation: &Generation,
844    kind: &str,
845    key: &str,
846    payload: serde_json::Value,
847) -> Result<(Job, bool)> {
848    if kind.is_empty() || key.is_empty() {
849        bail!("worker kind and operation key must be nonempty");
850    }
851    let _store = store_lock(home)?;
852    let mut jobs = records(home)?;
853    reconcile(home, &mut jobs)?;
854    if let Some(job) = jobs
855        .iter()
856        .find(|job| job.key == key && matches!(job.status, Status::Starting | Status::Running))
857    {
858        if job.kind != kind || job.payload != payload {
859            bail!("worker operation key {key} is already reserved for a different request");
860        }
861        return Ok((job.clone(), false));
862    }
863    let id = nonce();
864    let job = Job {
865        id: id.clone(),
866        sequence: next_sequence(home)?,
867        kind: kind.to_owned(),
868        key: key.to_owned(),
869        payload,
870        generation: generation.id.clone(),
871        version: None,
872        status: Status::Starting,
873        result: None,
874        pid: None,
875        created_at: now(),
876        finished_at: None,
877    };
878    let jobs_dir = root(home).join("jobs");
879    let staging = jobs_dir.join(format!(".new-{id}"));
880    std::fs::create_dir(&staging)
881        .with_context(|| format!("create worker reservation staging {}", staging.display()))?;
882    save(&staging.join("job.json"), &job)?;
883    clear_control_locked(home, key)?;
884    std::fs::rename(&staging, job_dir(home, &id)?)
885        .with_context(|| format!("publish worker {id} reservation from {}", staging.display()))?;
886    sync_dir(&jobs_dir)?;
887    Ok((job, true))
888}
889
890fn prune_results(home: &Path) -> Result<()> {
891    let results = root(home).join("results");
892    std::fs::create_dir_all(&results)?;
893    let mut entries = Vec::new();
894    for entry in std::fs::read_dir(&results)? {
895        let entry = entry?;
896        if entry.file_type()?.is_dir() {
897            // Directory mtime changes when the terminal job.json is published.
898            // Retention needs no deserialization of old handler/config payloads.
899            entries.push((entry.metadata()?.modified()?, entry.path()));
900        }
901    }
902    entries.sort_unstable_by_key(|(modified, _)| std::cmp::Reverse(*modified));
903    for (_, dir) in entries.into_iter().skip(RETAIN_FINISHED) {
904        if let Some(lease) = try_lock(&dir.join("live.lock"))? {
905            drop(lease);
906            std::fs::remove_dir_all(dir)?;
907        }
908    }
909    sync_dir(&results)
910}
911
912fn detached_command(home: &Path, job: &Job, generation: &Generation) -> Result<Command> {
913    // Open before exec so argument parsing, startup panics, and task progress
914    // remain diagnosable even before the child can publish its acknowledgement.
915    // Archiving/pruning the reservation also archives/prunes its log.
916    let log_path = job_dir(home, &job.id)?.join("worker.log");
917    let log = OpenOptions::new()
918        .create(true)
919        .append(true)
920        .open(&log_path)
921        .with_context(|| format!("open worker startup log {}", log_path.display()))?;
922    let stderr = log
923        .try_clone()
924        .with_context(|| format!("clone worker startup log {}", log_path.display()))?;
925    let mut command = Command::new(&generation.executable);
926    command
927        .arg("__worker")
928        .arg("--home")
929        .arg(home)
930        .arg(&job.id)
931        .env("MAGI_HOME", home)
932        .env("MAGI_WORKER_HOME", home)
933        .env("MAGI_WORKER_ID", &job.id)
934        .env("MAGI_WORKER_GENERATION", &generation.id)
935        .env("MAGI_WORKER_EXE", &generation.executable)
936        .env_remove("MAGI_RUN")
937        .env_remove("MAGI_NODE")
938        .env_remove("MAGI_SEAT")
939        .stdin(Stdio::null())
940        .stdout(Stdio::from(log))
941        .stderr(Stdio::from(stderr));
942    crate::chat_control::bridge::clear_std_environment(&mut command);
943    let mut paths = vec![
944        generation
945            .executable
946            .parent()
947            .context("generation directory")?
948            .to_path_buf(),
949    ];
950    if let Some(path) = std::env::var_os("PATH") {
951        paths.extend(std::env::split_paths(&path));
952    }
953    command.env("PATH", std::env::join_paths(paths)?);
954    #[cfg(unix)]
955    {
956        use std::os::unix::process::CommandExt;
957        unsafe {
958            command.pre_exec(|| {
959                if libc::setsid() < 0 {
960                    return Err(std::io::Error::last_os_error());
961                }
962                // A scheduler's Ctrl-C handling must not be inherited as an
963                // ignored shutdown signal by its independent worker session.
964                libc::signal(libc::SIGINT, libc::SIG_DFL);
965                libc::signal(libc::SIGTERM, libc::SIG_DFL);
966                Ok(())
967            });
968        }
969    }
970    #[cfg(windows)]
971    {
972        use std::os::windows::process::CommandExt;
973        detached_standard_handles().context("exclude caller pipes from detached worker")?;
974        command.creation_flags(
975            crate::chat_watch::windows::detached_flags()
976                .context("prepare independent Windows worker")?,
977        );
978    }
979    Ok(command)
980}
981
982#[cfg(windows)]
983fn detached_standard_handles() -> std::io::Result<()> {
984    use std::ffi::c_void;
985    #[link(name = "kernel32")]
986    unsafe extern "system" {
987        fn GetStdHandle(kind: u32) -> *mut c_void;
988        fn SetHandleInformation(handle: *mut c_void, mask: u32, flags: u32) -> i32;
989    }
990    // Explicit Stdio entries are duplicated by Command for the child. The
991    // caller's original inherited handles must not also leak into a detached
992    // process: that keeps an external caller's stdout pipe open after exit.
993    // Leave them non-inheritable; subsequent ordinary Command launches still
994    // inherit their explicitly selected standard streams.
995    for kind in [-10_i32, -11, -12] {
996        let handle = unsafe { GetStdHandle(kind as u32) };
997        if handle.is_null() || handle as isize == -1 {
998            continue;
999        }
1000        if unsafe { SetHandleInformation(handle, 1, 0) } == 0 {
1001            let error = std::io::Error::last_os_error();
1002            // Console pseudo handles have no inheritable kernel handle flag.
1003            if error.raw_os_error() != Some(6) {
1004                return Err(error);
1005            }
1006        }
1007    }
1008    Ok(())
1009}
1010
1011/// A concurrent fork can inherit the generation's writable file descriptor.
1012/// Even after pinning closes its copy, Unix exec can return ETXTBSY until the
1013/// other child closes its copy on exec or exit. Retry only that spawn error.
1014async fn spawn_generation(
1015    mut spawn: impl FnMut() -> std::io::Result<Child>,
1016) -> std::io::Result<Child> {
1017    for delay_ms in [10, 20, 40, 80] {
1018        match spawn() {
1019            Err(error) if error.kind() == std::io::ErrorKind::ExecutableFileBusy => {
1020                tokio::time::sleep(Duration::from_millis(delay_ms)).await;
1021            }
1022            result => return result,
1023        }
1024    }
1025    spawn()
1026}
1027
1028/// Reserve an exclusive operation and detach its immutable executable. Returns
1029/// after acknowledgement; a timeout preserves the reservation and never kills
1030/// the child or silently executes in the caller. Repeated identical requests
1031/// attach to the existing reservation rather than dispatching twice.
1032pub async fn spawn(home: &Path, kind: &str, key: &str, payload: serde_json::Value) -> Result<Job> {
1033    std::fs::create_dir_all(home).map_err(not_started_error)?;
1034    let home = home
1035        .canonicalize()
1036        .context("resolve worker home")
1037        .map_err(not_started_error)?;
1038    let pin_home = home.clone();
1039    let generation = tokio::task::spawn_blocking(move || pin_current(&pin_home))
1040        .await
1041        .map_err(not_started_error)?
1042        .map_err(not_started_error)?;
1043    let (job, fresh) = reserve(&home, &generation, kind, key, payload)?;
1044    if fresh {
1045        let child = match detached_command(&home, &job, &generation) {
1046            Ok(mut command) => spawn_generation(|| command.spawn())
1047                .await
1048                .map_err(Into::into),
1049            Err(error) => Err(error),
1050        };
1051        match child {
1052            Ok(mut child) => {
1053                // std::process::Child has no kill-on-drop. The reaper is only
1054                // bookkeeping; runtime or parent exit cannot terminate a worker.
1055                std::thread::spawn(move || {
1056                    let _ = child.wait();
1057                });
1058            }
1059            Err(error) => {
1060                let _store = store_lock(&home)?;
1061                let mut job = get(&home, &job.id)?;
1062                if job.status == Status::Starting {
1063                    if let Some(_lease) = try_lock(&job_dir(&home, &job.id)?.join("live.lock"))? {
1064                        terminal(
1065                            &home,
1066                            &mut job,
1067                            Status::Failed,
1068                            format!("worker spawn failed: {error}"),
1069                        )?;
1070                        return Err(not_started_error(error));
1071                    }
1072                }
1073                return Err(error);
1074            }
1075        }
1076    }
1077    let deadline = tokio::time::Instant::now() + Duration::from_secs(10);
1078    loop {
1079        let current = get(&home, &job.id)?;
1080        match current.status {
1081            Status::Running | Status::Finished => return Ok(current),
1082            Status::Failed => bail!(
1083                "worker {} failed: {}",
1084                current.id,
1085                current.result.as_deref().unwrap_or("unknown failure")
1086            ),
1087            Status::Starting => {}
1088        }
1089        if tokio::time::Instant::now() >= deadline {
1090            return Err(StartupUnacknowledged { id: job.id }.into());
1091        }
1092        tokio::time::sleep(Duration::from_millis(20)).await;
1093    }
1094}
1095
1096/// Hidden detached entrypoint. The lease is acquired before acknowledgement and
1097/// is retained through handler result persistence. Only the recorded generation
1098/// can acknowledge a reservation, even if a newer CLI is accidentally invoked.
1099pub async fn run(home: &Path, id: &str) -> Result<()> {
1100    let dir = job_dir(home, id)?;
1101    let (job, _lease) = {
1102        let _store = store_lock(home)?;
1103        let mut job = get(home, id)?;
1104        if job.status != Status::Starting {
1105            bail!("worker reservation is no longer startable");
1106        }
1107        let lease =
1108            try_lock(&dir.join("live.lock"))?.context("worker execution lease is already held")?;
1109        let current = std::env::current_exe()
1110            .context("locate running worker executable")?
1111            .canonicalize()
1112            .context("resolve running worker executable")?;
1113        let expected = executable(home, &job.generation)?
1114            .canonicalize()
1115            .with_context(|| {
1116                format!("resolve worker {id} reserved generation {}", job.generation)
1117            })?;
1118        if current != expected {
1119            bail!("worker must execute its reserved immutable generation");
1120        }
1121        job.pid = Some(std::process::id());
1122        job.version = Some(env!("CARGO_PKG_VERSION").to_owned());
1123        job.status = Status::Running;
1124        save(&dir.join("job.json"), &job)
1125            .with_context(|| format!("acknowledge worker {id} startup"))?;
1126        (job, lease)
1127    };
1128    let result = match job.kind.as_str() {
1129        "task" => crate::daemon::run_task_worker(home, job.payload.clone()).await,
1130        "scheduler" => crate::daemon::run_scheduler_worker(home, job.payload.clone()).await,
1131        "chat" => crate::chat_worker::run_turn_worker(home, job.payload.clone()).await,
1132        "service" => crate::consult::run_service_worker(home, job.payload.clone()).await,
1133        "daemon-service" => crate::daemon::run_service_worker(home, job.payload.clone()).await,
1134        "deputy" => crate::deputy::run_worker(home, job.payload.clone()).await,
1135        kind => Err(anyhow::anyhow!("unknown worker kind {kind}")),
1136    };
1137    let _store = store_lock(home)?;
1138    let mut current = get(home, id)?;
1139    match &result {
1140        Ok(()) => terminal(home, &mut current, Status::Finished, "completed".to_owned())?,
1141        Err(error) => terminal(home, &mut current, Status::Failed, format!("{error:#}"))?,
1142    }
1143    result
1144}
1145
1146fn hex_digest(hash: Sha256) -> String {
1147    // sha2 0.11's `finalize` returns a `hybrid_array::Array`, which no
1148    // longer implements `LowerHex`, so format the bytes individually.
1149    hash.finalize()
1150        .iter()
1151        .map(|byte| format!("{byte:02x}"))
1152        .collect()
1153}
1154fn digest(bytes: &[u8]) -> String {
1155    let mut hash = Sha256::new();
1156    hash.update(bytes);
1157    hex_digest(hash)
1158}
1159
1160#[cfg(test)]
1161mod tests {
1162    use super::*;
1163
1164    #[test]
1165    fn observing_an_unused_store_requires_no_writes() {
1166        let home = tempfile::tempdir().unwrap();
1167        assert!(observe_active(home.path()).unwrap().is_empty());
1168        assert!(!home.path().join("workers").exists());
1169    }
1170
1171    #[test]
1172    fn observing_execution_probes_the_lease_without_rewriting_history() {
1173        let home = tempfile::tempdir().unwrap();
1174        let mut job = reservation(home.path(), "task:observer");
1175        let directory = job_dir(home.path(), &job.id).unwrap();
1176        let live_path = directory.join("live.lock");
1177        let lease = try_lock(&live_path).unwrap().unwrap();
1178        job.status = Status::Running;
1179        save(&directory.join("job.json"), &job).unwrap();
1180        #[cfg(unix)]
1181        {
1182            use std::os::unix::fs::PermissionsExt as _;
1183            std::fs::set_permissions(&live_path, std::fs::Permissions::from_mode(0o444)).unwrap();
1184        }
1185        let before = std::fs::read(directory.join("job.json")).unwrap();
1186        let observed = observe_active(home.path()).unwrap();
1187        assert_eq!(observed.len(), 1);
1188        assert_eq!(observed[0].id, job.id);
1189        drop(lease);
1190        assert!(observe_active(home.path()).unwrap().is_empty());
1191        assert_eq!(std::fs::read(directory.join("job.json")).unwrap(), before);
1192        assert_eq!(get(home.path(), &job.id).unwrap().status, Status::Running);
1193    }
1194    fn generation() -> Generation {
1195        Generation {
1196            id: "a".repeat(64),
1197            executable: PathBuf::from("unused"),
1198        }
1199    }
1200    fn reservation(home: &Path, key: &str) -> Job {
1201        reserve(
1202            home,
1203            &generation(),
1204            "task",
1205            key,
1206            serde_json::json!({"task":"example"}),
1207        )
1208        .unwrap()
1209        .0
1210    }
1211
1212    #[test]
1213    fn reservation_deduplicates_identical_keys_and_rejects_collision() {
1214        let home = tempfile::tempdir().unwrap();
1215        let first = reservation(home.path(), "task:one");
1216        assert_eq!(first.id, reservation(home.path(), "task:one").id);
1217        assert!(
1218            reserve(
1219                home.path(),
1220                &generation(),
1221                "chat",
1222                "task:one",
1223                serde_json::Value::Null
1224            )
1225            .is_err()
1226        );
1227        assert_ne!(first.id, reservation(home.path(), "task:two").id);
1228    }
1229
1230    #[test]
1231    fn reservation_order_is_durable_and_corruption_never_resets_it() {
1232        let home = tempfile::tempdir().unwrap();
1233        let first = reservation(home.path(), "chat:first");
1234        assert_eq!(
1235            first.sequence,
1236            reservation(home.path(), "chat:first").sequence
1237        );
1238        let second = reservation(home.path(), "chat:second");
1239        assert!(second.sequence > first.sequence);
1240        let encoded = serde_json::to_vec(&second).unwrap();
1241        assert_eq!(
1242            serde_json::from_slice::<Job>(&encoded).unwrap().sequence,
1243            second.sequence
1244        );
1245        let mut legacy = serde_json::to_value(&first).unwrap();
1246        legacy.as_object_mut().unwrap().remove("sequence");
1247        assert_eq!(serde_json::from_value::<Job>(legacy).unwrap().sequence, 0);
1248        std::fs::write(root(home.path()).join("sequence.json"), b"broken").unwrap();
1249        assert!(
1250            reserve(
1251                home.path(),
1252                &generation(),
1253                "chat",
1254                "chat:third",
1255                serde_json::Value::Null
1256            )
1257            .is_err()
1258        );
1259    }
1260
1261    #[tokio::test]
1262    async fn pre_reservation_failure_is_proven_without_history_absence_guesses() {
1263        let home = tempfile::tempdir().unwrap();
1264        let blocked = home.path().join("not-a-directory");
1265        std::fs::write(&blocked, b"file").unwrap();
1266        let error = spawn(&blocked, "chat", "fresh:chat", serde_json::Value::Null)
1267            .await
1268            .unwrap_err();
1269        assert!(not_started(&error));
1270        assert!(not_started(&error.context("publish recorded turn")));
1271        assert!(!not_started(&anyhow::anyhow!(
1272            "startup acknowledgement timed out"
1273        )));
1274        assert!(!blocked.join("workers").exists());
1275    }
1276
1277    #[test]
1278    fn failed_os_spawn_is_fenced_and_proven_not_started() {
1279        let home = tempfile::tempdir().unwrap();
1280        let executable = home.path().join("invalid-executable");
1281        std::fs::write(&executable, b"not an executable binary").unwrap();
1282        let output = Command::new(std::env::current_exe().unwrap())
1283            .args([
1284                "--exact",
1285                "worker::tests::unstarted_spawn_helper",
1286                "--ignored",
1287                "--nocapture",
1288            ])
1289            .env("MAGI_WORKER_EXE", &executable)
1290            .env("MAGI_SPAWN_FAILURE_HOME", home.path())
1291            .output()
1292            .unwrap();
1293        assert!(
1294            output.status.success(),
1295            "{}\n{}",
1296            String::from_utf8_lossy(&output.stdout),
1297            String::from_utf8_lossy(&output.stderr)
1298        );
1299    }
1300
1301    #[tokio::test]
1302    #[ignore = "isolated environment for the failed OS spawn test"]
1303    async fn unstarted_spawn_helper() {
1304        let home = PathBuf::from(std::env::var_os("MAGI_SPAWN_FAILURE_HOME").unwrap());
1305        let error = spawn(&home, "chat", "fresh:failed-exec", serde_json::Value::Null)
1306            .await
1307            .unwrap_err();
1308        assert!(not_started(&error));
1309        let entry = std::fs::read_dir(root(&home).join("jobs"))
1310            .unwrap()
1311            .next()
1312            .unwrap()
1313            .unwrap();
1314        let id = entry.file_name().to_str().unwrap().to_owned();
1315        let job = get(&home, &id).unwrap();
1316        assert_eq!(job.status, Status::Failed);
1317        assert_eq!(job.pid, None);
1318        assert!(active(&home).unwrap().is_empty());
1319        assert_eq!(get(&home, &id).unwrap().status, Status::Failed);
1320    }
1321
1322    #[test]
1323    fn unresolved_startup_remains_reserved_and_live_lease_never_expires() {
1324        let home = tempfile::tempdir().unwrap();
1325        let mut job = reservation(home.path(), "task:one");
1326        assert_eq!(active(home.path()).unwrap().len(), 1);
1327        job.created_at = 0;
1328        save(
1329            &job_dir(home.path(), &job.id).unwrap().join("job.json"),
1330            &job,
1331        )
1332        .unwrap();
1333        let lease = try_lock(&job_dir(home.path(), &job.id).unwrap().join("live.lock"))
1334            .unwrap()
1335            .unwrap();
1336        assert_eq!(active(home.path()).unwrap().len(), 1);
1337        drop(lease);
1338        assert!(active(home.path()).unwrap().is_empty());
1339        assert_eq!(get(home.path(), &job.id).unwrap().status, Status::Failed);
1340    }
1341
1342    #[tokio::test]
1343    async fn cancelled_startup_cannot_execute_late() {
1344        let home = tempfile::tempdir().unwrap();
1345        let mut job = reservation(home.path(), "task:one");
1346        job.created_at = 0;
1347        save(
1348            &job_dir(home.path(), &job.id).unwrap().join("job.json"),
1349            &job,
1350        )
1351        .unwrap();
1352        assert!(active(home.path()).unwrap().is_empty());
1353        assert!(
1354            run(home.path(), &job.id)
1355                .await
1356                .unwrap_err()
1357                .to_string()
1358                .contains("no longer startable")
1359        );
1360    }
1361
1362    #[test]
1363    fn running_without_lease_fails_regardless_of_pid() {
1364        let home = tempfile::tempdir().unwrap();
1365        let mut job = reservation(home.path(), "task:one");
1366        job.status = Status::Running;
1367        job.pid = Some(std::process::id());
1368        save(
1369            &job_dir(home.path(), &job.id).unwrap().join("job.json"),
1370            &job,
1371        )
1372        .unwrap();
1373        assert!(active(home.path()).unwrap().is_empty());
1374        assert_eq!(get(home.path(), &job.id).unwrap().status, Status::Failed);
1375    }
1376
1377    #[test]
1378    fn controls_survive_reopen_and_reset_at_new_invocation() {
1379        let home = tempfile::tempdir().unwrap();
1380        request(home.path(), "task:one", Control::Park).unwrap();
1381        assert_eq!(
1382            control(home.path(), "task:one").unwrap(),
1383            Some(Control::Park)
1384        );
1385        reservation(home.path(), "task:one");
1386        assert_eq!(control(home.path(), "task:one").unwrap(), None);
1387        request(home.path(), "task:one", Control::Stop).unwrap();
1388        assert_eq!(
1389            control(home.path(), "task:one").unwrap(),
1390            Some(Control::Stop)
1391        );
1392        assert_eq!(control(home.path(), "task:other").unwrap(), None);
1393    }
1394
1395    #[test]
1396    fn singleton_locks_release_without_pid_cleanup() {
1397        let home = tempfile::tempdir().unwrap();
1398        let lease = try_lease(home.path(), "scheduler").unwrap().unwrap();
1399        assert!(try_lease(home.path(), "scheduler").unwrap().is_none());
1400        assert!(try_lease(home.path(), "other").unwrap().is_some());
1401        drop(lease);
1402        assert!(try_lease(home.path(), "scheduler").unwrap().is_some());
1403    }
1404
1405    #[test]
1406    fn immutable_generation_keeps_old_content_after_source_replacement() {
1407        let home = tempfile::tempdir().unwrap();
1408        let source = home.path().join("source");
1409        std::fs::write(&source, b"old executable").unwrap();
1410        let old = pin_source(home.path(), &source).unwrap();
1411        std::fs::write(&source, b"new executable").unwrap();
1412        let new = pin_source(home.path(), &source).unwrap();
1413        assert_ne!(old.id, new.id);
1414        assert_eq!(
1415            std::fs::read(executable(home.path(), &old.id).unwrap()).unwrap(),
1416            b"old executable"
1417        );
1418        assert_eq!(std::fs::read(new.executable).unwrap(), b"new executable");
1419    }
1420
1421    #[tokio::test]
1422    async fn generation_spawn_does_not_retry_a_missing_executable() {
1423        let home = tempfile::tempdir().unwrap();
1424        let mut command = Command::new(home.path().join("missing-executable"));
1425        let mut attempts = 0;
1426        let error = spawn_generation(|| {
1427            attempts += 1;
1428            command.spawn()
1429        })
1430        .await
1431        .unwrap_err();
1432        assert_eq!(error.kind(), std::io::ErrorKind::NotFound);
1433        assert_eq!(attempts, 1);
1434    }
1435
1436    #[cfg(target_os = "linux")]
1437    fn busy_executable(dir: &Path) -> (Command, File) {
1438        use std::os::unix::fs::PermissionsExt;
1439
1440        let path = dir.join("busy-executable");
1441        std::fs::copy("/bin/sh", &path).unwrap();
1442        std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o700)).unwrap();
1443        let writer = OpenOptions::new().write(true).open(&path).unwrap();
1444        let mut command = Command::new(path);
1445        command.args(["-c", "exit 0"]);
1446        (command, writer)
1447    }
1448
1449    #[cfg(target_os = "linux")]
1450    #[tokio::test]
1451    async fn generation_spawn_retries_until_the_writer_closes() {
1452        let home = tempfile::tempdir().unwrap();
1453        let (mut command, writer) = busy_executable(home.path());
1454        let error = command.spawn().unwrap_err();
1455        assert_eq!(error.kind(), std::io::ErrorKind::ExecutableFileBusy);
1456
1457        let (release, requested) = std::sync::mpsc::channel();
1458        let closer = std::thread::spawn(move || {
1459            if requested.recv().is_ok() {
1460                std::thread::sleep(Duration::from_millis(20));
1461            }
1462            drop(writer);
1463        });
1464        let mut release = Some(release);
1465        let child = spawn_generation(|| {
1466            let result = command.spawn();
1467            if let Some(release) = release.take() {
1468                release.send(()).unwrap();
1469            }
1470            result
1471        })
1472        .await;
1473        closer.join().unwrap();
1474        assert!(child.unwrap().wait().unwrap().success());
1475    }
1476
1477    #[cfg(target_os = "linux")]
1478    #[tokio::test]
1479    async fn generation_spawn_stops_retrying_while_the_writer_remains_open() {
1480        let home = tempfile::tempdir().unwrap();
1481        let (mut command, _writer) = busy_executable(home.path());
1482        let mut attempts = 0;
1483        let error = spawn_generation(|| {
1484            attempts += 1;
1485            assert!(attempts <= 5, "busy generation exceeded its retry bound");
1486            command.spawn()
1487        })
1488        .await
1489        .unwrap_err();
1490        assert_eq!(error.kind(), std::io::ErrorKind::ExecutableFileBusy);
1491    }
1492
1493    #[tokio::test]
1494    async fn acknowledgement_records_own_version_and_handler_failure() {
1495        let home = tempfile::tempdir().unwrap();
1496        let generation = pin_source(home.path(), &std::env::current_exe().unwrap()).unwrap();
1497        let (job, _) = reserve(
1498            home.path(),
1499            &generation,
1500            "unknown-kind",
1501            "unknown:one",
1502            serde_json::Value::Null,
1503        )
1504        .unwrap();
1505        let invoke = || async {
1506            let mut command = Command::new(&generation.executable);
1507            command
1508                .args([
1509                    "--exact",
1510                    "worker::tests::pinned_entrypoint_helper",
1511                    "--ignored",
1512                    "--nocapture",
1513                ])
1514                .env("MAGI_WORKER_TEST_HOME", home.path())
1515                .env("MAGI_WORKER_TEST_ID", &job.id)
1516                .stdin(Stdio::null())
1517                .stdout(Stdio::piped())
1518                .stderr(Stdio::piped());
1519            let output = spawn_generation(|| command.spawn())
1520                .await
1521                .unwrap()
1522                .wait_with_output()
1523                .unwrap();
1524            assert!(
1525                output.status.success(),
1526                "{}",
1527                String::from_utf8_lossy(&output.stderr)
1528            );
1529        };
1530        invoke().await;
1531        let failed = get(home.path(), &job.id).unwrap();
1532        assert_eq!(failed.status, Status::Failed);
1533        assert_eq!(failed.version.as_deref(), Some(env!("CARGO_PKG_VERSION")));
1534        assert!(failed.result.unwrap().contains("unknown worker kind"));
1535        assert!(active(home.path()).unwrap().is_empty());
1536        assert!(!job_dir(home.path(), &job.id).unwrap().exists());
1537        invoke().await;
1538        assert!(!job_dir(home.path(), &job.id).unwrap().exists());
1539    }
1540
1541    #[tokio::test]
1542    #[ignore = "subprocess entrypoint for the immutable-generation startup test"]
1543    async fn pinned_entrypoint_helper() {
1544        let home = PathBuf::from(std::env::var_os("MAGI_WORKER_TEST_HOME").unwrap());
1545        let id = std::env::var("MAGI_WORKER_TEST_ID").unwrap();
1546        assert!(run(&home, &id).await.is_err());
1547    }
1548
1549    #[test]
1550    fn unfinished_staging_is_reclaimed_without_removing_generations() {
1551        let home = tempfile::tempdir().unwrap();
1552        let old = root(home.path()).join("generations").join(".new-abandoned");
1553        std::fs::create_dir_all(&old).unwrap();
1554        let partial = old.join("partial");
1555        std::fs::write(&partial, b"interrupted copy").unwrap();
1556        #[cfg(windows)]
1557        {
1558            let mut permissions = std::fs::metadata(&partial).unwrap().permissions();
1559            permissions.set_readonly(true);
1560            std::fs::set_permissions(&partial, permissions).unwrap();
1561        }
1562        let source = home.path().join("source");
1563        std::fs::write(&source, b"complete executable").unwrap();
1564        let generation = pin_source(home.path(), &source).unwrap();
1565        assert!(!old.exists());
1566        assert!(generation.executable.is_file());
1567    }
1568
1569    #[tokio::test]
1570    async fn unrelated_executable_cannot_acknowledge_or_consume_reservation() {
1571        let home = tempfile::tempdir().unwrap();
1572        let source = home.path().join("other-binary");
1573        std::fs::write(&source, b"different executable of the same Cargo version").unwrap();
1574        let generation = pin_source(home.path(), &source).unwrap();
1575        let (job, _) = reserve(
1576            home.path(),
1577            &generation,
1578            "task",
1579            "task:one",
1580            serde_json::Value::Null,
1581        )
1582        .unwrap();
1583        assert!(
1584            run(home.path(), &job.id)
1585                .await
1586                .unwrap_err()
1587                .to_string()
1588                .contains("immutable generation")
1589        );
1590        assert_eq!(get(home.path(), &job.id).unwrap().status, Status::Starting);
1591        assert_eq!(active(home.path()).unwrap().len(), 1);
1592    }
1593
1594    #[test]
1595    fn abandoned_record_staging_is_bounded_and_never_executes() {
1596        let home = tempfile::tempdir().unwrap();
1597        let staging = root(home.path()).join("jobs").join(".new-interrupted");
1598        std::fs::create_dir_all(&staging).unwrap();
1599        std::fs::write(staging.join("job.json"), b"partial json").unwrap();
1600        assert!(active(home.path()).unwrap().is_empty());
1601        assert!(!staging.exists());
1602    }
1603
1604    #[cfg(unix)]
1605    #[tokio::test]
1606    async fn detached_startup_preserves_stdout_and_stderr_in_record_log() {
1607        use std::os::unix::fs::PermissionsExt;
1608        let home = tempfile::tempdir().unwrap();
1609        let source = home.path().join("startup-diagnostic");
1610        std::fs::write(
1611            &source,
1612            b"#!/bin/sh\nprintf 'startup stdout\\n'\nprintf 'startup stderr\\n' >&2\nexit 7\n",
1613        )
1614        .unwrap();
1615        std::fs::set_permissions(&source, std::fs::Permissions::from_mode(0o755)).unwrap();
1616        let generation = pin_source(home.path(), &source).unwrap();
1617        let (mut job, _) = reserve(
1618            home.path(),
1619            &generation,
1620            "task",
1621            "task:log",
1622            serde_json::Value::Null,
1623        )
1624        .unwrap();
1625        let mut command = detached_command(home.path(), &job, &generation).unwrap();
1626        for key in ["MAGI_RUN", "MAGI_NODE", "MAGI_SEAT"]
1627            .into_iter()
1628            .chain(crate::chat_control::bridge::ENVIRONMENT.iter().copied())
1629        {
1630            assert!(
1631                command
1632                    .get_envs()
1633                    .any(|(name, value)| name == key && value.is_none())
1634            );
1635        }
1636        let mut child = spawn_generation(|| command.spawn()).await.unwrap();
1637        assert_eq!(child.wait().unwrap().code(), Some(7));
1638        let log = job_dir(home.path(), &job.id).unwrap().join("worker.log");
1639        let output = std::fs::read_to_string(log).unwrap();
1640        assert!(output.contains("startup stdout"));
1641        assert!(output.contains("startup stderr"));
1642        terminal(
1643            home.path(),
1644            &mut job,
1645            Status::Failed,
1646            "startup failed".into(),
1647        )
1648        .unwrap();
1649        assert!(active(home.path()).unwrap().is_empty());
1650        assert!(
1651            root(home.path())
1652                .join("results")
1653                .join(&job.id)
1654                .join("worker.log")
1655                .is_file()
1656        );
1657    }
1658
1659    #[cfg(windows)]
1660    #[test]
1661    fn record_publication_survives_readers_and_short_delete_sharing_handoffs() {
1662        use std::os::windows::fs::OpenOptionsExt as _;
1663
1664        let home = tempfile::tempdir().unwrap();
1665        let mut job = reservation(home.path(), "task:metadata-handoff");
1666        let directory = job_dir(home.path(), &job.id).unwrap();
1667        let path = directory.join("job.json");
1668        let lease = try_lock(&directory.join("live.lock")).unwrap().unwrap();
1669        let snapshot = File::open(&path).unwrap();
1670        // This reader allows ordinary IO but temporarily denies replacement.
1671        let reader = OpenOptions::new()
1672            .read(true)
1673            .share_mode(1 | 2) // FILE_SHARE_READ | FILE_SHARE_WRITE
1674            .open(&path)
1675            .unwrap();
1676        std::thread::scope(|scope| {
1677            scope.spawn(move || {
1678                std::thread::sleep(Duration::from_millis(15));
1679                drop(reader);
1680            });
1681            job.status = Status::Running;
1682            save(&path, &job).unwrap();
1683        });
1684        let original: Job = serde_json::from_reader(snapshot).unwrap();
1685        assert_eq!(original.status, Status::Starting);
1686        assert_eq!(get(home.path(), &job.id).unwrap().status, Status::Running);
1687        let running_snapshot = File::open(&path).unwrap();
1688        terminal(home.path(), &mut job, Status::Finished, "completed".into()).unwrap();
1689        assert_eq!(get(home.path(), &job.id).unwrap().status, Status::Finished);
1690        let previous: Job = serde_json::from_reader(running_snapshot).unwrap();
1691        assert_eq!(previous.status, Status::Running);
1692        drop(lease);
1693        assert!(active(home.path()).unwrap().is_empty());
1694    }
1695
1696    #[cfg(windows)]
1697    #[test]
1698    fn denied_record_publication_preserves_reservation_and_original_os_error() {
1699        use std::os::windows::fs::OpenOptionsExt as _;
1700
1701        let home = tempfile::tempdir().unwrap();
1702        let job = reservation(home.path(), "task:metadata-denied");
1703        let directory = job_dir(home.path(), &job.id).unwrap();
1704        let path = directory.join("job.json");
1705        let reader = OpenOptions::new()
1706            .read(true)
1707            .share_mode(1 | 2) // Deny deletion for the entire handoff budget.
1708            .open(&path)
1709            .unwrap();
1710        let mut acknowledged = job.clone();
1711        acknowledged.status = Status::Running;
1712        let error = save(&path, &acknowledged).unwrap_err();
1713        assert!(format!("{error:#}").contains("publish worker record"));
1714        let io = error.downcast_ref::<std::io::Error>().unwrap();
1715        assert!(matches!(io.raw_os_error(), Some(5 | 32)));
1716        assert_eq!(get(home.path(), &job.id).unwrap().status, Status::Starting);
1717        assert_eq!(reservation(home.path(), &job.key).id, job.id);
1718        assert!(std::fs::read_dir(&directory).unwrap().all(|entry| {
1719            !entry
1720                .unwrap()
1721                .file_name()
1722                .to_string_lossy()
1723                .starts_with(".pending-")
1724        }));
1725        drop(reader);
1726        save(&path, &acknowledged).unwrap();
1727        assert_eq!(get(home.path(), &job.id).unwrap().status, Status::Running);
1728    }
1729
1730    #[test]
1731    fn terminal_result_survives_open_execution_and_output_handles() {
1732        let home = tempfile::tempdir().unwrap();
1733        let mut job = reservation(home.path(), "task:open-handles");
1734        let directory = job_dir(home.path(), &job.id).unwrap();
1735        let lease = try_lock(&directory.join("live.lock")).unwrap().unwrap();
1736        let mut log = OpenOptions::new()
1737            .create(true)
1738            .append(true)
1739            .open(directory.join("worker.log"))
1740            .unwrap();
1741        writeln!(log, "before completion").unwrap();
1742        terminal(home.path(), &mut job, Status::Finished, "completed".into()).unwrap();
1743        assert_eq!(get(home.path(), &job.id).unwrap().status, Status::Finished);
1744        assert!(active(home.path()).unwrap().is_empty());
1745        assert!(
1746            directory.exists(),
1747            "a held execution lease must prevent retirement"
1748        );
1749        drop(lease);
1750        // Retirement may still be denied by the output handle on Windows.
1751        // Either way, querying active work and the durable result must succeed.
1752        assert!(active(home.path()).unwrap().is_empty());
1753        assert_eq!(get(home.path(), &job.id).unwrap().status, Status::Finished);
1754        writeln!(log, "after completion").unwrap();
1755        drop(log);
1756        assert!(active(home.path()).unwrap().is_empty());
1757        assert!(
1758            root(home.path())
1759                .join("results")
1760                .join(&job.id)
1761                .join("worker.log")
1762                .is_file()
1763        );
1764        assert!(!directory.exists());
1765    }
1766
1767    #[test]
1768    fn terminal_records_are_bounded_without_pruning_live_reservations() {
1769        let home = tempfile::tempdir().unwrap();
1770        let live = reservation(home.path(), "task:live");
1771        // Exercise retention, not hundreds of durable reservation publications.
1772        // Use a real terminal record as the on-disk template; reservation and
1773        // terminal publication have their own tests above.
1774        let mut finished = reservation(home.path(), "task:finished");
1775        terminal(home.path(), &mut finished, Status::Finished, "done".into()).unwrap();
1776        let template_dir = job_dir(home.path(), &finished.id).unwrap();
1777        std::fs::remove_dir_all(template_dir).unwrap();
1778        for i in 0..RETAIN_FINISHED + 3 {
1779            let mut job = finished.clone();
1780            job.id = format!("{i:032x}");
1781            job.key = format!("task:{i}");
1782            job.sequence += i as u64;
1783            let directory = job_dir(home.path(), &job.id).unwrap();
1784            std::fs::create_dir_all(&directory).unwrap();
1785            std::fs::write(
1786                directory.join("job.json"),
1787                serde_json::to_vec(&job).unwrap(),
1788            )
1789            .unwrap();
1790        }
1791        assert_eq!(
1792            std::fs::read_dir(root(home.path()).join("jobs"))
1793                .unwrap()
1794                .count(),
1795            RETAIN_FINISHED + 4
1796        );
1797        let jobs = records(home.path()).unwrap();
1798        assert_eq!(jobs.len(), 1);
1799        assert_eq!(
1800            std::fs::read_dir(root(home.path()).join("results"))
1801                .unwrap()
1802                .count(),
1803            RETAIN_FINISHED
1804        );
1805        assert!(jobs.iter().any(|job| job.id == live.id));
1806    }
1807}