Skip to main content

magi/
worker.rs

1//! Durable detached execution, independent of the scheduler and web lifetimes.
2//!
3//! A reservation is published before spawn, and the child must take its execution
4//! lease and acknowledge that reservation before dispatch. Reconciliation cancels
5//! an unacknowledged startup under the same store lock the child needs: a late
6//! child can never execute a cancelled operation. Process IDs are diagnostic only.
7//! Metadata publication closes its staging handle before atomic replacement.
8//! Windows replacement/delete-pending handoffs use the short, bounded lock-file
9//! retry policy; failed publication never releases an uncertain reservation.
10
11use std::fs::{File, OpenOptions};
12use std::io::{Read, Write};
13use std::path::{Path, PathBuf};
14use std::process::{Command, Stdio};
15use std::sync::{LazyLock, Mutex, OnceLock};
16use std::time::{Duration, SystemTime, UNIX_EPOCH};
17
18use anyhow::{Context, Result, bail};
19use serde::{Deserialize, Serialize};
20use sha2::{Digest, Sha256};
21
22const STARTUP_SECONDS: u64 = 30;
23const RETAIN_FINISHED: usize = 256;
24
25/// Identity and executable selection inherited by descendants of a worker.
26/// Verification commands and standalone test fixtures remove these overrides:
27/// they test the checkout, rather than act on behalf of the enclosing worker.
28/// Keep runtime storage, seat metadata and process containment context intact.
29pub const INVOCATION_ENV: [&str; 4] = [
30    "MAGI_WORKER_ID",
31    "MAGI_WORKER_GENERATION",
32    "MAGI_WORKER_HOME",
33    "MAGI_WORKER_EXE",
34];
35
36/// Durable execution state. Starting reservations count as occupied slots.
37#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
38#[serde(rename_all = "snake_case")]
39pub enum Status {
40    /// Published but not yet acknowledged by the detached child.
41    Starting,
42    /// The child acknowledged startup and owns the execution lease.
43    Running,
44    /// The handler returned successfully.
45    Finished,
46    /// Startup, handler execution, or the owning process failed.
47    Failed,
48}
49
50/// An explicit owner control; upgrades do not send these to task workers.
51#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
52#[serde(rename_all = "snake_case")]
53pub enum Control {
54    /// Park at the handler's existing safe cancellation boundary.
55    Park,
56    /// Stop the operation through its handler's cancellation path.
57    Stop,
58}
59
60#[derive(Debug)]
61struct NotStarted(anyhow::Error);
62
63impl std::fmt::Display for NotStarted {
64    fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
65        write!(formatter, "{:#}", self.0)
66    }
67}
68
69impl std::error::Error for NotStarted {
70    fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
71        Some(self.0.as_ref())
72    }
73}
74
75fn not_started_error(error: impl Into<anyhow::Error>) -> anyhow::Error {
76    anyhow::Error::new(NotStarted(error.into()))
77}
78
79/// Whether this spawn invocation provably did not execute its handler. This
80/// covers preparation failures and a failed OS spawn whose fresh reservation
81/// was exclusively leased and durably fenced before this error was returned.
82/// Only a fresh operation key makes this proof sufficient to restore a caller's
83/// publication. Unknown startup, existing work, or a failed fence is never tagged.
84pub fn not_started(error: &anyhow::Error) -> bool {
85    error.downcast_ref::<NotStarted>().is_some()
86}
87
88/// A durable operation record. Its payload belongs to the started generation.
89#[derive(Clone, Debug, Serialize, Deserialize)]
90pub struct Job {
91    /// Unique invocation identity.
92    pub id: String,
93    /// Durable total reservation order, independent of wall clock resolution.
94    #[serde(default)]
95    pub sequence: u64,
96    /// Dispatch handler, such as task, chat, scheduler, or service.
97    pub kind: String,
98    /// Exclusive logical operation identity, independent of executable generation.
99    pub key: String,
100    /// Handler-owned request.
101    pub payload: serde_json::Value,
102    /// SHA-256 identity of the immutable executable.
103    pub generation: String,
104    /// Version reported by the started executable, unknown before acknowledgement.
105    #[serde(default)]
106    pub version: Option<String>,
107    /// Last durable execution state.
108    pub status: Status,
109    /// Successful completion note or failure diagnostic.
110    pub result: Option<String>,
111    /// Diagnostic process ID, never used as an execution witness.
112    pub pid: Option<u32>,
113    /// Reservation publication time, seconds since the Unix epoch.
114    pub created_at: u64,
115    /// Terminal state publication time.
116    pub finished_at: Option<u64>,
117}
118
119/// An immutable, content-addressed executable.
120#[derive(Clone, Debug)]
121pub struct Generation {
122    /// SHA-256 digest of the executable bytes.
123    pub id: String,
124    /// Absolute path to the retained executable.
125    pub executable: PathBuf,
126}
127
128/// OS-held lock. Closing it, including on a crash, releases ownership.
129#[derive(Debug)]
130pub struct Lease {
131    _file: File,
132}
133
134impl Drop for Lease {
135    fn drop(&mut self) {
136        // flock's lock belongs to the open-file description; explicitly unlock
137        // so a temporarily inherited descriptor cannot prolong ownership.
138        #[cfg(unix)]
139        {
140            use std::os::fd::AsRawFd;
141            unsafe { libc::flock(self._file.as_raw_fd(), libc::LOCK_UN) };
142        }
143    }
144}
145
146fn root(home: &Path) -> PathBuf {
147    home.join("workers")
148}
149fn now() -> u64 {
150    SystemTime::now()
151        .duration_since(UNIX_EPOCH)
152        .unwrap_or_default()
153        .as_secs()
154}
155fn nonce() -> String {
156    crate::rng::SplitMix64::new(crate::rng::entropy()).uuid_v4()
157}
158fn valid_id(id: &str) -> bool {
159    !id.is_empty() && id.bytes().all(|b| b.is_ascii_hexdigit() || b == b'-')
160}
161fn job_dir(home: &Path, id: &str) -> Result<PathBuf> {
162    if !valid_id(id) {
163        bail!("invalid worker id");
164    }
165    Ok(root(home).join("jobs").join(id))
166}
167
168fn open_lock(path: &Path) -> Result<File> {
169    if let Some(parent) = path.parent() {
170        std::fs::create_dir_all(parent)
171            .with_context(|| format!("create worker lock directory {}", parent.display()))?;
172    }
173    OpenOptions::new()
174        .create(true)
175        .truncate(false)
176        .read(true)
177        .write(true)
178        .open(path)
179        .with_context(|| format!("open worker lock {}", path.display()))
180}
181
182fn try_lock(path: &Path) -> Result<Option<Lease>> {
183    let file = open_lock(path)?;
184    match crate::pr_head::try_lock_file(&file) {
185        Ok(()) => Ok(Some(Lease { _file: file })),
186        Err(e)
187            if e.kind() == std::io::ErrorKind::WouldBlock
188                || matches!(e.raw_os_error(), Some(33)) =>
189        {
190            Ok(None)
191        }
192        Err(e) => Err(e).with_context(|| format!("acquire worker lock {}", path.display())),
193    }
194}
195
196#[cfg(test)]
197pub(crate) fn execution_lease_for_test(path: &Path) -> Lease {
198    try_lock(path)
199        .expect("execution lease")
200        .expect("free execution lease")
201}
202
203/// Try to own a named cross-process singleton, without PID-based stale handling.
204/// Names are hashed, so callers may use logical keys containing paths or colons.
205pub fn try_lease(home: &Path, name: &str) -> Result<Option<Lease>> {
206    try_lock(
207        &root(home)
208            .join("leases")
209            .join(format!("{}.lock", digest(name.as_bytes()))),
210    )
211}
212
213fn store_lock(home: &Path) -> Result<Lease> {
214    let deadline = std::time::Instant::now() + Duration::from_secs(10);
215    loop {
216        if let Some(lease) = try_lock(&root(home).join("store.lock"))? {
217            return Ok(lease);
218        }
219        if std::time::Instant::now() >= deadline {
220            bail!("worker store is busy; retry later");
221        }
222        std::thread::sleep(Duration::from_millis(5));
223    }
224}
225
226#[cfg(unix)]
227fn sync_dir(path: &Path) -> Result<()> {
228    File::open(path)
229        .with_context(|| format!("open worker directory for sync {}", path.display()))?
230        .sync_all()
231        .with_context(|| format!("sync worker directory {}", path.display()))?;
232    Ok(())
233}
234
235#[cfg(not(unix))]
236fn sync_dir(_path: &Path) -> Result<()> {
237    Ok(())
238}
239
240fn save(path: &Path, value: &impl Serialize) -> Result<()> {
241    let parent = path.parent().context("worker record has no parent")?;
242    std::fs::create_dir_all(parent)
243        .with_context(|| format!("create worker record directory {}", parent.display()))?;
244    let temp = parent.join(format!(".pending-{}", nonce()));
245    let result = (|| -> Result<()> {
246        let mut file = OpenOptions::new()
247            .create_new(true)
248            .write(true)
249            .open(&temp)
250            .with_context(|| format!("create staged worker record {}", temp.display()))?;
251        serde_json::to_writer(&mut file, value)
252            .with_context(|| format!("serialize worker record {}", path.display()))?;
253        file.flush()
254            .with_context(|| format!("flush staged worker record {}", temp.display()))?;
255        file.sync_all()
256            .with_context(|| format!("sync staged worker record {}", temp.display()))?;
257        // Publication must not retain our own writable staging handle. Windows
258        // readers may briefly deny deletion; only the atomic handoff retries.
259        drop(file);
260        replace(&temp, path).with_context(|| {
261            format!(
262                "publish worker record {} -> {}",
263                temp.display(),
264                path.display()
265            )
266        })?;
267        sync_dir(parent)
268    })();
269    if result.is_err() {
270        let _ = std::fs::remove_file(&temp);
271    }
272    result
273}
274
275#[cfg(not(windows))]
276fn replace(from: &Path, to: &Path) -> Result<()> {
277    Ok(std::fs::rename(from, to)?)
278}
279
280#[cfg(windows)]
281fn replace(from: &Path, to: &Path) -> Result<()> {
282    // std uses FileRenameInfoEx/POSIX semantics where supported, unlike
283    // MoveFileExW: a FILE_SHARE_DELETE reader may retain the previous record
284    // while the replacement becomes visible atomically. Readers that briefly
285    // deny deletion still get only the existing short Windows handoff budget.
286    crate::lock_file::Retry::new(None).run(|| std::fs::rename(from, to))?;
287    Ok(())
288}
289
290// Atomic replacement can leave the old Windows name briefly delete-pending.
291// Keep that handoff bounded; malformed records and permanent IO errors propagate.
292fn open_record(path: &Path) -> std::io::Result<File> {
293    #[cfg(windows)]
294    {
295        crate::lock_file::Retry::new(None).run(|| File::open(path))
296    }
297    #[cfg(not(windows))]
298    {
299        File::open(path)
300    }
301}
302
303/// Load a specific invocation, including a retained terminal result.
304pub fn get(home: &Path, id: &str) -> Result<Job> {
305    let path = job_dir(home, id)?.join("job.json");
306    let file = match open_record(&path) {
307        Ok(file) => file,
308        Err(error) if error.kind() == std::io::ErrorKind::NotFound => {
309            let retained = root(home).join("results").join(id).join("job.json");
310            open_record(&retained)
311                .with_context(|| format!("open retained worker record {}", retained.display()))?
312        }
313        Err(error) => {
314            return Err(error).with_context(|| format!("open worker record {}", path.display()));
315        }
316    };
317    serde_json::from_reader(file).with_context(|| format!("decode worker record {id}"))
318}
319
320/// Identify this detached invocation. Helpers inherit its identity, while a
321/// newly spawned worker receives its own id before exec; no global environment
322/// mutation is required inside the asynchronous handler runtime.
323pub fn current_job(home: &Path) -> Result<Option<Job>> {
324    match std::env::var("MAGI_WORKER_ID") {
325        Ok(id) => Ok(Some(get(home, &id)?)),
326        Err(std::env::VarError::NotPresent) => Ok(None),
327        Err(error) => Err(error.into()),
328    }
329}
330
331fn next_sequence(home: &Path) -> Result<u64> {
332    let path = root(home).join("sequence.json");
333    let previous: u64 = match open_record(&path) {
334        Ok(file) => serde_json::from_reader(file)
335            .with_context(|| format!("decode worker sequence {}", path.display()))?,
336        Err(error) if error.kind() == std::io::ErrorKind::NotFound => 0,
337        Err(error) => {
338            return Err(error).with_context(|| format!("open worker sequence {}", path.display()));
339        }
340    };
341    let sequence = previous
342        .checked_add(1)
343        .context("worker reservation sequence exhausted")?;
344    save(&path, &sequence)?;
345    Ok(sequence)
346}
347
348fn records(home: &Path) -> Result<Vec<Job>> {
349    let directory = root(home).join("jobs");
350    std::fs::create_dir_all(&directory)?;
351    let mut jobs = Vec::new();
352    let mut retired = false;
353    for entry in std::fs::read_dir(directory)? {
354        let entry = entry?;
355        if !entry.file_type()?.is_dir() {
356            continue;
357        }
358        let name = entry.file_name();
359        let Some(id) = name.to_str() else {
360            continue;
361        };
362        if id.starts_with(".new-") {
363            // Reservations are staged only while holding the store lock. Any
364            // staging directory visible to the next owner was left by a crash.
365            std::fs::remove_dir_all(entry.path())?;
366            continue;
367        }
368        if id.starts_with('.') {
369            continue;
370        }
371        for pending in std::fs::read_dir(entry.path())? {
372            let pending = pending?;
373            if pending.file_type()?.is_file()
374                && pending
375                    .file_name()
376                    .to_string_lossy()
377                    .starts_with(".pending-")
378            {
379                std::fs::remove_file(pending.path())?;
380            }
381        }
382        let job = get(home, id).with_context(|| format!("read worker {id}"))?;
383        if matches!(job.status, Status::Starting | Status::Running) {
384            jobs.push(job);
385        } else {
386            retired |= retire(home, &job);
387        }
388    }
389    if retired {
390        if let Err(error) = prune_results(home) {
391            tracing::warn!(%error, "worker result retention deferred");
392        }
393    }
394    Ok(jobs)
395}
396
397fn terminal(home: &Path, job: &mut Job, status: Status, result: String) -> Result<()> {
398    job.status = status;
399    job.result = Some(result);
400    job.finished_at = Some(now());
401    save(&job_dir(home, &job.id)?.join("job.json"), job)
402        .with_context(|| format!("publish worker {} terminal status {status:?}", job.id))?;
403    // The result is authoritative now. Keep the directory in place while this
404    // process owns live.lock and its stdout/stderr handles. Another store owner
405    // retires it opportunistically after execution ownership is released.
406    clear_control_locked(home, &job.key)
407}
408
409fn retire(home: &Path, job: &Job) -> bool {
410    let attempt = || -> Result<bool> {
411        let dir = job_dir(home, &job.id)?;
412        let Some(lease) = try_lock(&dir.join("live.lock"))? else {
413            return Ok(false);
414        };
415        drop(lease);
416        archive(home, job)?;
417        Ok(true)
418    };
419    // Windows can still deny the directory rename in the short interval after
420    // lease release but before process stdout/stderr are closed. Retry on the
421    // next store read; archival failure must never turn completion into failure
422    // or terminate an otherwise healthy scheduler.
423    match attempt() {
424        Ok(retired) => retired,
425        Err(error) => {
426            tracing::warn!(
427                worker = %job.id,
428                error = %format_args!("{error:#}"),
429                "worker result retirement deferred"
430            );
431            false
432        }
433    }
434}
435
436fn archive(home: &Path, job: &Job) -> Result<()> {
437    let results = root(home).join("results");
438    std::fs::create_dir_all(&results)
439        .with_context(|| format!("create worker results directory {}", results.display()))?;
440    let from = job_dir(home, &job.id)?;
441    let to = results.join(&job.id);
442    std::fs::rename(&from, &to).with_context(|| {
443        format!(
444            "archive worker result {} -> {}",
445            from.display(),
446            to.display()
447        )
448    })?;
449    sync_dir(&root(home).join("jobs"))?;
450    sync_dir(&results)
451}
452
453// Caller owns the store lock. Acquiring the execution lease and then marking a
454// dead startup terminal fences a late child: it must recheck state under store.
455fn reconcile(home: &Path, jobs: &mut [Job]) -> Result<()> {
456    let mut retired = false;
457    for job in jobs {
458        if !matches!(job.status, Status::Starting | Status::Running) {
459            continue;
460        }
461        let lease = try_lock(&job_dir(home, &job.id)?.join("live.lock"))?;
462        if lease.is_none() {
463            continue;
464        }
465        if job.status == Status::Starting && now().saturating_sub(job.created_at) < STARTUP_SECONDS
466        {
467            continue;
468        }
469        let reason = if job.status == Status::Starting {
470            "worker startup reservation expired before acknowledgement"
471        } else {
472            "worker exited without recording a result"
473        };
474        terminal(home, job, Status::Failed, reason.to_owned())?;
475        drop(lease);
476        retired |= retire(home, job);
477    }
478    if retired {
479        if let Err(error) = prune_results(home) {
480            tracing::warn!(%error, "worker result retention deferred");
481        }
482    }
483    Ok(())
484}
485
486/// List live operations and unresolved startups. A missing lease only frees a
487/// startup after its deadline and an atomic cancellation fence; PID is irrelevant.
488pub fn active(home: &Path) -> Result<Vec<Job>> {
489    let _store = store_lock(home)?;
490    let mut jobs = records(home)?;
491    reconcile(home, &mut jobs)?;
492    Ok(jobs
493        .into_iter()
494        .filter(|job| matches!(job.status, Status::Starting | Status::Running))
495        .collect())
496}
497
498/// Observe execution without creating, repairing, archiving, or locking the
499/// writable store. Sandboxed diagnostics need only file reads and a read-only
500/// handle for the OS lease probe. Starting reservations remain protected until
501/// an authorized writer fences them; a missing running lease is unknown.
502pub fn observe_active(home: &Path) -> Result<Vec<Job>> {
503    let entries = match std::fs::read_dir(root(home).join("jobs")) {
504        Ok(entries) => entries,
505        Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(Vec::new()),
506        Err(error) => return Err(error.into()),
507    };
508    let mut active = Vec::new();
509    for entry in entries {
510        let entry = entry?;
511        if !entry.file_type()?.is_dir() {
512            continue;
513        }
514        let name = entry.file_name();
515        let id = name.to_str().context("invalid worker reservation name")?;
516        if id.starts_with('.') {
517            continue;
518        }
519        let job = get(home, id)?;
520        match job.status {
521            Status::Starting => active.push(job),
522            Status::Running => {
523                let file = match File::open(entry.path().join("live.lock")) {
524                    Ok(file) => file,
525                    Err(error) if error.kind() == std::io::ErrorKind::NotFound => {
526                        active.push(job);
527                        continue;
528                    }
529                    Err(error) => return Err(error.into()),
530                };
531                match crate::pr_head::try_lock_file(&file) {
532                    Ok(()) => drop(Lease { _file: file }),
533                    Err(error)
534                        if error.kind() == std::io::ErrorKind::WouldBlock
535                            || cfg!(windows) && error.raw_os_error() == Some(33) =>
536                    {
537                        active.push(job);
538                    }
539                    Err(error) => return Err(error.into()),
540                }
541            }
542            Status::Finished | Status::Failed => {}
543        }
544    }
545    Ok(active)
546}
547
548fn control_path(home: &Path, key: &str) -> PathBuf {
549    root(home)
550        .join("controls")
551        .join(format!("{}.json", digest(key.as_bytes())))
552}
553
554/// Persist a stop or park request; the operation's handler consumes it safely.
555pub fn request(home: &Path, key: &str, control: Control) -> Result<()> {
556    let _store = store_lock(home)?;
557    save(&control_path(home, key), &control)
558}
559
560/// Read the latest durable owner request for a logical operation.
561pub fn control(home: &Path, key: &str) -> Result<Option<Control>> {
562    let path = control_path(home, key);
563    match open_record(&path) {
564        Ok(file) => serde_json::from_reader(file)
565            .map(Some)
566            .with_context(|| format!("decode worker control {}", path.display())),
567        Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(None),
568        Err(e) => Err(e).with_context(|| format!("open worker control {}", path.display())),
569    }
570}
571
572fn clear_control_locked(home: &Path, key: &str) -> Result<()> {
573    let path = control_path(home, key);
574    match std::fs::remove_file(&path) {
575        Ok(()) => sync_dir(&root(home).join("controls")),
576        Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(()),
577        Err(e) => Err(e).with_context(|| format!("remove worker control {}", path.display())),
578    }
579}
580
581/// Clear controls at an explicitly owned new singleton invocation boundary.
582pub fn clear_control(home: &Path, key: &str) -> Result<()> {
583    let _store = store_lock(home)?;
584    clear_control_locked(home, key)
585}
586
587/// Resolve a retained executable strictly by its content identity.
588pub fn executable(home: &Path, generation: &str) -> Result<PathBuf> {
589    if generation.len() != 64 || !generation.bytes().all(|b| b.is_ascii_hexdigit()) {
590        bail!("invalid worker generation");
591    }
592    let path = root(home)
593        .join("generations")
594        .join(generation)
595        .join(if cfg!(windows) { "magi.exe" } else { "magi" });
596    if !path.is_file() {
597        bail!("worker generation {generation} is unavailable");
598    }
599    Ok(path)
600}
601
602/// Resolve helper invocations to this worker's started executable, not a newly
603/// installed binary. Outside a worker, pin the caller's current executable.
604pub fn helper_executable(home: &Path) -> Result<PathBuf> {
605    if let Some(generation) = std::env::var_os("MAGI_WORKER_GENERATION") {
606        return executable(
607            home,
608            generation
609                .to_str()
610                .context("invalid generation environment")?,
611        );
612    }
613    Ok(pin_current(home)?.executable)
614}
615
616/// Remember the installed executable pathname before an updater replaces it.
617/// A web process calls this at startup so Linux's deleted-executable marker or
618/// Windows's backup rename cannot redirect successor workers to the old binary.
619pub fn remember_current_executable() -> Result<()> {
620    if INSTALLED_EXECUTABLE.get().is_some() {
621        return Ok(());
622    }
623    let path = std::env::current_exe()?.canonicalize()?;
624    let _ = INSTALLED_EXECUTABLE.set(path);
625    Ok(())
626}
627
628static INSTALLED_EXECUTABLE: OnceLock<PathBuf> = OnceLock::new();
629
630/// Copy and SHA-256 address the executable. MAGI_WORKER_EXE is an explicit
631/// embedding/test override; it is never an in-process execution fallback.
632pub fn pin_current(home: &Path) -> Result<Generation> {
633    let source = match std::env::var_os("MAGI_WORKER_EXE") {
634        Some(path) => PathBuf::from(path),
635        None => match INSTALLED_EXECUTABLE.get() {
636            Some(path) => path.clone(),
637            None => std::env::current_exe()?,
638        },
639    };
640    let source = source.canonicalize().context("resolve worker executable")?;
641    // A worker already runs the immutable copy. Do not copy/hash a large binary
642    // again for every finite maintenance invocation in the same generation.
643    if let Some(directory) = source.parent() {
644        if directory.parent() == Some(root(home).join("generations").as_path()) {
645            if let Some(id) = directory.file_name().and_then(|name| name.to_str()) {
646                if id.len() == 64
647                    && id.bytes().all(|b| b.is_ascii_hexdigit())
648                    && executable(home, id)?.canonicalize()? == source
649                {
650                    return Ok(Generation {
651                        id: id.to_owned(),
652                        executable: source,
653                    });
654                }
655            }
656        }
657    }
658    let stamp = source_stamp(&source)?;
659    let mut cache = PIN_CACHE
660        .lock()
661        .map_err(|_| anyhow::anyhow!("worker generation cache poisoned"))?;
662    if let Some(entry) = cache.iter().find(|entry| {
663        entry.home == home
664            && entry.source == source
665            && entry.stamp == stamp
666            && entry.generation.executable.is_file()
667    }) {
668        return Ok(entry.generation.clone());
669    }
670    let generation = pin_source(home, &source)?;
671    if source_stamp(&source)? != stamp {
672        bail!("worker executable changed while being pinned; retry");
673    }
674    if cache.len() >= 8 {
675        cache.remove(0);
676    }
677    cache.push(CachedGeneration {
678        home: home.to_path_buf(),
679        source,
680        stamp,
681        generation: generation.clone(),
682    });
683    Ok(generation)
684}
685
686type SourceStamp = (u64, Option<SystemTime>, Option<SystemTime>, u64, u64);
687struct CachedGeneration {
688    home: PathBuf,
689    source: PathBuf,
690    stamp: SourceStamp,
691    generation: Generation,
692}
693static PIN_CACHE: LazyLock<Mutex<Vec<CachedGeneration>>> = LazyLock::new(|| Mutex::new(Vec::new()));
694
695fn source_stamp(path: &Path) -> Result<SourceStamp> {
696    let metadata = std::fs::metadata(path)?;
697    #[cfg(unix)]
698    let identity = {
699        use std::os::unix::fs::MetadataExt;
700        (metadata.dev(), metadata.ino())
701    };
702    #[cfg(not(unix))]
703    let identity = (0, 0);
704    Ok((
705        metadata.len(),
706        metadata.modified().ok(),
707        metadata.created().ok(),
708        identity.0,
709        identity.1,
710    ))
711}
712
713fn pin_source(home: &Path, source: &Path) -> Result<Generation> {
714    let generations = root(home).join("generations");
715    std::fs::create_dir_all(&generations)?;
716    let staging = generations.join(format!(".new-{}", nonce()));
717    let mut staging_lease = {
718        let _store = store_lock(home)?;
719        // Only unpublished copies are reclaimed, never executable generations.
720        for entry in std::fs::read_dir(&generations)? {
721            let entry = entry?;
722            if entry.file_type()?.is_dir()
723                && entry.file_name().to_string_lossy().starts_with(".new-")
724            {
725                if let Some(lease) = try_lock(&entry.path().join("pin.lock"))? {
726                    drop(lease);
727                    remove_staging(&entry.path())?;
728                }
729            }
730        }
731        std::fs::create_dir(&staging)?;
732        Some(try_lock(&staging.join("pin.lock"))?.context("generation staging lease unavailable")?)
733    };
734    let temp = staging.join(if cfg!(windows) { "magi.exe" } else { "magi" });
735    let result = (|| -> Result<Generation> {
736        let mut input = File::open(source)?;
737        let mut output = OpenOptions::new()
738            .create_new(true)
739            .write(true)
740            .open(&temp)?;
741        let mut hash = Sha256::new();
742        let mut buffer = [0u8; 65536];
743        loop {
744            let n = input.read(&mut buffer)?;
745            if n == 0 {
746                break;
747            }
748            hash.update(&buffer[..n]);
749            output.write_all(&buffer[..n])?;
750        }
751        let id = hex_digest(hash);
752        let mut permissions = input.metadata()?.permissions();
753        permissions.set_readonly(true);
754        output.set_permissions(permissions)?;
755        output.sync_all()?;
756        drop(output);
757        sync_dir(&staging)?;
758        let target = generations.join(&id);
759        // The store lock serializes publication with retention and other pins.
760        let _store = store_lock(home)?;
761        drop(staging_lease.take());
762        std::fs::remove_file(staging.join("pin.lock"))?;
763        if target.exists() {
764            let existing = target.join(temp.file_name().context("binary filename")?);
765            if hash_file(&existing)? != id {
766                bail!("immutable worker generation is corrupt: {id}");
767            }
768        } else {
769            std::fs::rename(&staging, &target)?;
770            sync_dir(&generations)?;
771        }
772        Ok(Generation {
773            executable: target.join(temp.file_name().context("binary filename")?),
774            id,
775        })
776    })();
777    let _store = store_lock(home)?;
778    drop(staging_lease);
779    if staging.exists() {
780        remove_staging(&staging)?;
781    }
782    result
783}
784
785fn remove_staging(staging: &Path) -> Result<()> {
786    #[cfg(windows)]
787    {
788        use std::os::windows::{ffi::OsStrExt, fs::MetadataExt};
789        #[link(name = "kernel32")]
790        unsafe extern "system" {
791            fn SetFileAttributesW(path: *const u16, attributes: u32) -> i32;
792        }
793        const READONLY: u32 = 0x1;
794        for entry in std::fs::read_dir(staging)? {
795            let entry = entry?;
796            let attributes = entry.metadata()?.file_attributes();
797            if attributes & READONLY != 0 {
798                let path: Vec<u16> = entry.path().as_os_str().encode_wide().chain([0]).collect();
799                // SAFETY: path is a live, NUL-terminated UTF-16 buffer. Clear
800                // only the Windows readonly attribute, preserving other bits.
801                if unsafe { SetFileAttributesW(path.as_ptr(), attributes & !READONLY) } == 0 {
802                    return Err(std::io::Error::last_os_error())
803                        .context("clear staging readonly attribute");
804                }
805            }
806        }
807    }
808    Ok(std::fs::remove_dir_all(staging)?)
809}
810
811fn hash_file(path: &Path) -> Result<String> {
812    let mut file = File::open(path)?;
813    let mut hash = Sha256::new();
814    let mut buffer = [0u8; 65536];
815    loop {
816        let n = file.read(&mut buffer)?;
817        if n == 0 {
818            break;
819        }
820        hash.update(&buffer[..n]);
821    }
822    Ok(hex_digest(hash))
823}
824
825fn reserve(
826    home: &Path,
827    generation: &Generation,
828    kind: &str,
829    key: &str,
830    payload: serde_json::Value,
831) -> Result<(Job, bool)> {
832    if kind.is_empty() || key.is_empty() {
833        bail!("worker kind and operation key must be nonempty");
834    }
835    let _store = store_lock(home)?;
836    let mut jobs = records(home)?;
837    reconcile(home, &mut jobs)?;
838    if let Some(job) = jobs
839        .iter()
840        .find(|job| job.key == key && matches!(job.status, Status::Starting | Status::Running))
841    {
842        if job.kind != kind || job.payload != payload {
843            bail!("worker operation key {key} is already reserved for a different request");
844        }
845        return Ok((job.clone(), false));
846    }
847    let id = nonce();
848    let job = Job {
849        id: id.clone(),
850        sequence: next_sequence(home)?,
851        kind: kind.to_owned(),
852        key: key.to_owned(),
853        payload,
854        generation: generation.id.clone(),
855        version: None,
856        status: Status::Starting,
857        result: None,
858        pid: None,
859        created_at: now(),
860        finished_at: None,
861    };
862    let jobs_dir = root(home).join("jobs");
863    let staging = jobs_dir.join(format!(".new-{id}"));
864    std::fs::create_dir(&staging)
865        .with_context(|| format!("create worker reservation staging {}", staging.display()))?;
866    save(&staging.join("job.json"), &job)?;
867    clear_control_locked(home, key)?;
868    std::fs::rename(&staging, job_dir(home, &id)?)
869        .with_context(|| format!("publish worker {id} reservation from {}", staging.display()))?;
870    sync_dir(&jobs_dir)?;
871    Ok((job, true))
872}
873
874fn prune_results(home: &Path) -> Result<()> {
875    let results = root(home).join("results");
876    std::fs::create_dir_all(&results)?;
877    let mut entries = Vec::new();
878    for entry in std::fs::read_dir(&results)? {
879        let entry = entry?;
880        if entry.file_type()?.is_dir() {
881            // Directory mtime changes when the terminal job.json is published.
882            // Retention needs no deserialization of old handler/config payloads.
883            entries.push((entry.metadata()?.modified()?, entry.path()));
884        }
885    }
886    entries.sort_unstable_by_key(|(modified, _)| std::cmp::Reverse(*modified));
887    for (_, dir) in entries.into_iter().skip(RETAIN_FINISHED) {
888        if let Some(lease) = try_lock(&dir.join("live.lock"))? {
889            drop(lease);
890            std::fs::remove_dir_all(dir)?;
891        }
892    }
893    sync_dir(&results)
894}
895
896fn detached_command(home: &Path, job: &Job, generation: &Generation) -> Result<Command> {
897    // Open before exec so argument parsing, startup panics, and task progress
898    // remain diagnosable even before the child can publish its acknowledgement.
899    // Archiving/pruning the reservation also archives/prunes its log.
900    let log_path = job_dir(home, &job.id)?.join("worker.log");
901    let log = OpenOptions::new()
902        .create(true)
903        .append(true)
904        .open(&log_path)
905        .with_context(|| format!("open worker startup log {}", log_path.display()))?;
906    let stderr = log
907        .try_clone()
908        .with_context(|| format!("clone worker startup log {}", log_path.display()))?;
909    let mut command = Command::new(&generation.executable);
910    command
911        .arg("__worker")
912        .arg("--home")
913        .arg(home)
914        .arg(&job.id)
915        .env("MAGI_HOME", home)
916        .env("MAGI_WORKER_HOME", home)
917        .env("MAGI_WORKER_ID", &job.id)
918        .env("MAGI_WORKER_GENERATION", &generation.id)
919        .env("MAGI_WORKER_EXE", &generation.executable)
920        .env_remove("MAGI_RUN")
921        .env_remove("MAGI_NODE")
922        .env_remove("MAGI_SEAT")
923        .stdin(Stdio::null())
924        .stdout(Stdio::from(log))
925        .stderr(Stdio::from(stderr));
926    crate::chat_control::bridge::clear_std_environment(&mut command);
927    let mut paths = vec![
928        generation
929            .executable
930            .parent()
931            .context("generation directory")?
932            .to_path_buf(),
933    ];
934    if let Some(path) = std::env::var_os("PATH") {
935        paths.extend(std::env::split_paths(&path));
936    }
937    command.env("PATH", std::env::join_paths(paths)?);
938    #[cfg(unix)]
939    {
940        use std::os::unix::process::CommandExt;
941        unsafe {
942            command.pre_exec(|| {
943                if libc::setsid() < 0 {
944                    return Err(std::io::Error::last_os_error());
945                }
946                // A scheduler's Ctrl-C handling must not be inherited as an
947                // ignored shutdown signal by its independent worker session.
948                libc::signal(libc::SIGINT, libc::SIG_DFL);
949                libc::signal(libc::SIGTERM, libc::SIG_DFL);
950                Ok(())
951            });
952        }
953    }
954    #[cfg(windows)]
955    {
956        use std::os::windows::process::CommandExt;
957        detached_standard_handles().context("exclude caller pipes from detached worker")?;
958        command.creation_flags(
959            crate::chat_watch::windows::detached_flags()
960                .context("prepare independent Windows worker")?,
961        );
962    }
963    Ok(command)
964}
965
966#[cfg(windows)]
967fn detached_standard_handles() -> std::io::Result<()> {
968    use std::ffi::c_void;
969    #[link(name = "kernel32")]
970    unsafe extern "system" {
971        fn GetStdHandle(kind: u32) -> *mut c_void;
972        fn SetHandleInformation(handle: *mut c_void, mask: u32, flags: u32) -> i32;
973    }
974    // Explicit Stdio entries are duplicated by Command for the child. The
975    // caller's original inherited handles must not also leak into a detached
976    // process: that keeps an external caller's stdout pipe open after exit.
977    // Leave them non-inheritable; subsequent ordinary Command launches still
978    // inherit their explicitly selected standard streams.
979    for kind in [-10_i32, -11, -12] {
980        let handle = unsafe { GetStdHandle(kind as u32) };
981        if handle.is_null() || handle as isize == -1 {
982            continue;
983        }
984        if unsafe { SetHandleInformation(handle, 1, 0) } == 0 {
985            let error = std::io::Error::last_os_error();
986            // Console pseudo handles have no inheritable kernel handle flag.
987            if error.raw_os_error() != Some(6) {
988                return Err(error);
989            }
990        }
991    }
992    Ok(())
993}
994
995/// Reserve an exclusive operation and detach its immutable executable. Returns
996/// after acknowledgement; a timeout preserves the reservation and never kills
997/// the child or silently executes in the caller. Repeated identical requests
998/// attach to the existing reservation rather than dispatching twice.
999pub async fn spawn(home: &Path, kind: &str, key: &str, payload: serde_json::Value) -> Result<Job> {
1000    std::fs::create_dir_all(home).map_err(not_started_error)?;
1001    let home = home
1002        .canonicalize()
1003        .context("resolve worker home")
1004        .map_err(not_started_error)?;
1005    let pin_home = home.clone();
1006    let generation = tokio::task::spawn_blocking(move || pin_current(&pin_home))
1007        .await
1008        .map_err(not_started_error)?
1009        .map_err(not_started_error)?;
1010    let (job, fresh) = reserve(&home, &generation, kind, key, payload)?;
1011    if fresh {
1012        match detached_command(&home, &job, &generation)
1013            .and_then(|mut command| Ok(command.spawn()?))
1014        {
1015            Ok(mut child) => {
1016                // std::process::Child has no kill-on-drop. The reaper is only
1017                // bookkeeping; runtime or parent exit cannot terminate a worker.
1018                std::thread::spawn(move || {
1019                    let _ = child.wait();
1020                });
1021            }
1022            Err(error) => {
1023                let _store = store_lock(&home)?;
1024                let mut job = get(&home, &job.id)?;
1025                if job.status == Status::Starting {
1026                    if let Some(_lease) = try_lock(&job_dir(&home, &job.id)?.join("live.lock"))? {
1027                        terminal(
1028                            &home,
1029                            &mut job,
1030                            Status::Failed,
1031                            format!("worker spawn failed: {error}"),
1032                        )?;
1033                        return Err(not_started_error(error));
1034                    }
1035                }
1036                return Err(error);
1037            }
1038        }
1039    }
1040    let deadline = tokio::time::Instant::now() + Duration::from_secs(10);
1041    loop {
1042        let current = get(&home, &job.id)?;
1043        match current.status {
1044            Status::Running | Status::Finished => return Ok(current),
1045            Status::Failed => bail!(
1046                "worker {} failed: {}",
1047                current.id,
1048                current.result.as_deref().unwrap_or("unknown failure")
1049            ),
1050            Status::Starting => {}
1051        }
1052        if tokio::time::Instant::now() >= deadline {
1053            bail!(
1054                "worker {} startup is unacknowledged; its reservation remains occupied; inspect or retry the same operation key",
1055                job.id
1056            );
1057        }
1058        tokio::time::sleep(Duration::from_millis(20)).await;
1059    }
1060}
1061
1062/// Hidden detached entrypoint. The lease is acquired before acknowledgement and
1063/// is retained through handler result persistence. Only the recorded generation
1064/// can acknowledge a reservation, even if a newer CLI is accidentally invoked.
1065pub async fn run(home: &Path, id: &str) -> Result<()> {
1066    let dir = job_dir(home, id)?;
1067    let (job, _lease) = {
1068        let _store = store_lock(home)?;
1069        let mut job = get(home, id)?;
1070        if job.status != Status::Starting {
1071            bail!("worker reservation is no longer startable");
1072        }
1073        let lease =
1074            try_lock(&dir.join("live.lock"))?.context("worker execution lease is already held")?;
1075        let current = std::env::current_exe()
1076            .context("locate running worker executable")?
1077            .canonicalize()
1078            .context("resolve running worker executable")?;
1079        let expected = executable(home, &job.generation)?
1080            .canonicalize()
1081            .with_context(|| {
1082                format!("resolve worker {id} reserved generation {}", job.generation)
1083            })?;
1084        if current != expected {
1085            bail!("worker must execute its reserved immutable generation");
1086        }
1087        job.pid = Some(std::process::id());
1088        job.version = Some(env!("CARGO_PKG_VERSION").to_owned());
1089        job.status = Status::Running;
1090        save(&dir.join("job.json"), &job)
1091            .with_context(|| format!("acknowledge worker {id} startup"))?;
1092        (job, lease)
1093    };
1094    let result = match job.kind.as_str() {
1095        "task" => crate::daemon::run_task_worker(home, job.payload.clone()).await,
1096        "scheduler" => crate::daemon::run_scheduler_worker(home, job.payload.clone()).await,
1097        "chat" => crate::chat_worker::run_turn_worker(home, job.payload.clone()).await,
1098        "service" => crate::consult::run_service_worker(home, job.payload.clone()).await,
1099        "daemon-service" => crate::daemon::run_service_worker(home, job.payload.clone()).await,
1100        "deputy" => crate::deputy::run_worker(home, job.payload.clone()).await,
1101        kind => Err(anyhow::anyhow!("unknown worker kind {kind}")),
1102    };
1103    let _store = store_lock(home)?;
1104    let mut current = get(home, id)?;
1105    match &result {
1106        Ok(()) => terminal(home, &mut current, Status::Finished, "completed".to_owned())?,
1107        Err(error) => terminal(home, &mut current, Status::Failed, format!("{error:#}"))?,
1108    }
1109    result
1110}
1111
1112fn hex_digest(hash: Sha256) -> String {
1113    // sha2 0.11's `finalize` returns a `hybrid_array::Array`, which no
1114    // longer implements `LowerHex`, so format the bytes individually.
1115    hash.finalize()
1116        .iter()
1117        .map(|byte| format!("{byte:02x}"))
1118        .collect()
1119}
1120fn digest(bytes: &[u8]) -> String {
1121    let mut hash = Sha256::new();
1122    hash.update(bytes);
1123    hex_digest(hash)
1124}
1125
1126#[cfg(test)]
1127mod tests {
1128    use super::*;
1129
1130    #[test]
1131    fn observing_an_unused_store_requires_no_writes() {
1132        let home = tempfile::tempdir().unwrap();
1133        assert!(observe_active(home.path()).unwrap().is_empty());
1134        assert!(!home.path().join("workers").exists());
1135    }
1136
1137    #[test]
1138    fn observing_execution_probes_the_lease_without_rewriting_history() {
1139        let home = tempfile::tempdir().unwrap();
1140        let mut job = reservation(home.path(), "task:observer");
1141        let directory = job_dir(home.path(), &job.id).unwrap();
1142        let live_path = directory.join("live.lock");
1143        let lease = try_lock(&live_path).unwrap().unwrap();
1144        job.status = Status::Running;
1145        save(&directory.join("job.json"), &job).unwrap();
1146        #[cfg(unix)]
1147        {
1148            use std::os::unix::fs::PermissionsExt as _;
1149            std::fs::set_permissions(&live_path, std::fs::Permissions::from_mode(0o444)).unwrap();
1150        }
1151        let before = std::fs::read(directory.join("job.json")).unwrap();
1152        let observed = observe_active(home.path()).unwrap();
1153        assert_eq!(observed.len(), 1);
1154        assert_eq!(observed[0].id, job.id);
1155        drop(lease);
1156        assert!(observe_active(home.path()).unwrap().is_empty());
1157        assert_eq!(std::fs::read(directory.join("job.json")).unwrap(), before);
1158        assert_eq!(get(home.path(), &job.id).unwrap().status, Status::Running);
1159    }
1160    fn generation() -> Generation {
1161        Generation {
1162            id: "a".repeat(64),
1163            executable: PathBuf::from("unused"),
1164        }
1165    }
1166    fn reservation(home: &Path, key: &str) -> Job {
1167        reserve(
1168            home,
1169            &generation(),
1170            "task",
1171            key,
1172            serde_json::json!({"task":"example"}),
1173        )
1174        .unwrap()
1175        .0
1176    }
1177
1178    #[test]
1179    fn reservation_deduplicates_identical_keys_and_rejects_collision() {
1180        let home = tempfile::tempdir().unwrap();
1181        let first = reservation(home.path(), "task:one");
1182        assert_eq!(first.id, reservation(home.path(), "task:one").id);
1183        assert!(
1184            reserve(
1185                home.path(),
1186                &generation(),
1187                "chat",
1188                "task:one",
1189                serde_json::Value::Null
1190            )
1191            .is_err()
1192        );
1193        assert_ne!(first.id, reservation(home.path(), "task:two").id);
1194    }
1195
1196    #[test]
1197    fn reservation_order_is_durable_and_corruption_never_resets_it() {
1198        let home = tempfile::tempdir().unwrap();
1199        let first = reservation(home.path(), "chat:first");
1200        assert_eq!(
1201            first.sequence,
1202            reservation(home.path(), "chat:first").sequence
1203        );
1204        let second = reservation(home.path(), "chat:second");
1205        assert!(second.sequence > first.sequence);
1206        let encoded = serde_json::to_vec(&second).unwrap();
1207        assert_eq!(
1208            serde_json::from_slice::<Job>(&encoded).unwrap().sequence,
1209            second.sequence
1210        );
1211        let mut legacy = serde_json::to_value(&first).unwrap();
1212        legacy.as_object_mut().unwrap().remove("sequence");
1213        assert_eq!(serde_json::from_value::<Job>(legacy).unwrap().sequence, 0);
1214        std::fs::write(root(home.path()).join("sequence.json"), b"broken").unwrap();
1215        assert!(
1216            reserve(
1217                home.path(),
1218                &generation(),
1219                "chat",
1220                "chat:third",
1221                serde_json::Value::Null
1222            )
1223            .is_err()
1224        );
1225    }
1226
1227    #[tokio::test]
1228    async fn pre_reservation_failure_is_proven_without_history_absence_guesses() {
1229        let home = tempfile::tempdir().unwrap();
1230        let blocked = home.path().join("not-a-directory");
1231        std::fs::write(&blocked, b"file").unwrap();
1232        let error = spawn(&blocked, "chat", "fresh:chat", serde_json::Value::Null)
1233            .await
1234            .unwrap_err();
1235        assert!(not_started(&error));
1236        assert!(not_started(&error.context("publish recorded turn")));
1237        assert!(!not_started(&anyhow::anyhow!(
1238            "startup acknowledgement timed out"
1239        )));
1240        assert!(!blocked.join("workers").exists());
1241    }
1242
1243    #[test]
1244    fn failed_os_spawn_is_fenced_and_proven_not_started() {
1245        let home = tempfile::tempdir().unwrap();
1246        let executable = home.path().join("invalid-executable");
1247        std::fs::write(&executable, b"not an executable binary").unwrap();
1248        let output = Command::new(std::env::current_exe().unwrap())
1249            .args([
1250                "--exact",
1251                "worker::tests::unstarted_spawn_helper",
1252                "--ignored",
1253                "--nocapture",
1254            ])
1255            .env("MAGI_WORKER_EXE", &executable)
1256            .env("MAGI_SPAWN_FAILURE_HOME", home.path())
1257            .output()
1258            .unwrap();
1259        assert!(
1260            output.status.success(),
1261            "{}\n{}",
1262            String::from_utf8_lossy(&output.stdout),
1263            String::from_utf8_lossy(&output.stderr)
1264        );
1265    }
1266
1267    #[tokio::test]
1268    #[ignore = "isolated environment for the failed OS spawn test"]
1269    async fn unstarted_spawn_helper() {
1270        let home = PathBuf::from(std::env::var_os("MAGI_SPAWN_FAILURE_HOME").unwrap());
1271        let error = spawn(&home, "chat", "fresh:failed-exec", serde_json::Value::Null)
1272            .await
1273            .unwrap_err();
1274        assert!(not_started(&error));
1275        let entry = std::fs::read_dir(root(&home).join("jobs"))
1276            .unwrap()
1277            .next()
1278            .unwrap()
1279            .unwrap();
1280        let id = entry.file_name().to_str().unwrap().to_owned();
1281        let job = get(&home, &id).unwrap();
1282        assert_eq!(job.status, Status::Failed);
1283        assert_eq!(job.pid, None);
1284        assert!(active(&home).unwrap().is_empty());
1285        assert_eq!(get(&home, &id).unwrap().status, Status::Failed);
1286    }
1287
1288    #[test]
1289    fn unresolved_startup_remains_reserved_and_live_lease_never_expires() {
1290        let home = tempfile::tempdir().unwrap();
1291        let mut job = reservation(home.path(), "task:one");
1292        assert_eq!(active(home.path()).unwrap().len(), 1);
1293        job.created_at = 0;
1294        save(
1295            &job_dir(home.path(), &job.id).unwrap().join("job.json"),
1296            &job,
1297        )
1298        .unwrap();
1299        let lease = try_lock(&job_dir(home.path(), &job.id).unwrap().join("live.lock"))
1300            .unwrap()
1301            .unwrap();
1302        assert_eq!(active(home.path()).unwrap().len(), 1);
1303        drop(lease);
1304        assert!(active(home.path()).unwrap().is_empty());
1305        assert_eq!(get(home.path(), &job.id).unwrap().status, Status::Failed);
1306    }
1307
1308    #[tokio::test]
1309    async fn cancelled_startup_cannot_execute_late() {
1310        let home = tempfile::tempdir().unwrap();
1311        let mut job = reservation(home.path(), "task:one");
1312        job.created_at = 0;
1313        save(
1314            &job_dir(home.path(), &job.id).unwrap().join("job.json"),
1315            &job,
1316        )
1317        .unwrap();
1318        assert!(active(home.path()).unwrap().is_empty());
1319        assert!(
1320            run(home.path(), &job.id)
1321                .await
1322                .unwrap_err()
1323                .to_string()
1324                .contains("no longer startable")
1325        );
1326    }
1327
1328    #[test]
1329    fn running_without_lease_fails_regardless_of_pid() {
1330        let home = tempfile::tempdir().unwrap();
1331        let mut job = reservation(home.path(), "task:one");
1332        job.status = Status::Running;
1333        job.pid = Some(std::process::id());
1334        save(
1335            &job_dir(home.path(), &job.id).unwrap().join("job.json"),
1336            &job,
1337        )
1338        .unwrap();
1339        assert!(active(home.path()).unwrap().is_empty());
1340        assert_eq!(get(home.path(), &job.id).unwrap().status, Status::Failed);
1341    }
1342
1343    #[test]
1344    fn controls_survive_reopen_and_reset_at_new_invocation() {
1345        let home = tempfile::tempdir().unwrap();
1346        request(home.path(), "task:one", Control::Park).unwrap();
1347        assert_eq!(
1348            control(home.path(), "task:one").unwrap(),
1349            Some(Control::Park)
1350        );
1351        reservation(home.path(), "task:one");
1352        assert_eq!(control(home.path(), "task:one").unwrap(), None);
1353        request(home.path(), "task:one", Control::Stop).unwrap();
1354        assert_eq!(
1355            control(home.path(), "task:one").unwrap(),
1356            Some(Control::Stop)
1357        );
1358        assert_eq!(control(home.path(), "task:other").unwrap(), None);
1359    }
1360
1361    #[test]
1362    fn singleton_locks_release_without_pid_cleanup() {
1363        let home = tempfile::tempdir().unwrap();
1364        let lease = try_lease(home.path(), "scheduler").unwrap().unwrap();
1365        assert!(try_lease(home.path(), "scheduler").unwrap().is_none());
1366        assert!(try_lease(home.path(), "other").unwrap().is_some());
1367        drop(lease);
1368        assert!(try_lease(home.path(), "scheduler").unwrap().is_some());
1369    }
1370
1371    #[test]
1372    fn immutable_generation_keeps_old_content_after_source_replacement() {
1373        let home = tempfile::tempdir().unwrap();
1374        let source = home.path().join("source");
1375        std::fs::write(&source, b"old executable").unwrap();
1376        let old = pin_source(home.path(), &source).unwrap();
1377        std::fs::write(&source, b"new executable").unwrap();
1378        let new = pin_source(home.path(), &source).unwrap();
1379        assert_ne!(old.id, new.id);
1380        assert_eq!(
1381            std::fs::read(executable(home.path(), &old.id).unwrap()).unwrap(),
1382            b"old executable"
1383        );
1384        assert_eq!(std::fs::read(new.executable).unwrap(), b"new executable");
1385    }
1386
1387    #[test]
1388    fn acknowledgement_records_own_version_and_handler_failure() {
1389        let home = tempfile::tempdir().unwrap();
1390        let generation = pin_source(home.path(), &std::env::current_exe().unwrap()).unwrap();
1391        let (job, _) = reserve(
1392            home.path(),
1393            &generation,
1394            "unknown-kind",
1395            "unknown:one",
1396            serde_json::Value::Null,
1397        )
1398        .unwrap();
1399        let invoke = || {
1400            let output = Command::new(&generation.executable)
1401                .args([
1402                    "--exact",
1403                    "worker::tests::pinned_entrypoint_helper",
1404                    "--ignored",
1405                    "--nocapture",
1406                ])
1407                .env("MAGI_WORKER_TEST_HOME", home.path())
1408                .env("MAGI_WORKER_TEST_ID", &job.id)
1409                .output()
1410                .unwrap();
1411            assert!(
1412                output.status.success(),
1413                "{}",
1414                String::from_utf8_lossy(&output.stderr)
1415            );
1416        };
1417        invoke();
1418        let failed = get(home.path(), &job.id).unwrap();
1419        assert_eq!(failed.status, Status::Failed);
1420        assert_eq!(failed.version.as_deref(), Some(env!("CARGO_PKG_VERSION")));
1421        assert!(failed.result.unwrap().contains("unknown worker kind"));
1422        assert!(active(home.path()).unwrap().is_empty());
1423        assert!(!job_dir(home.path(), &job.id).unwrap().exists());
1424        invoke();
1425        assert!(!job_dir(home.path(), &job.id).unwrap().exists());
1426    }
1427
1428    #[tokio::test]
1429    #[ignore = "subprocess entrypoint for the immutable-generation startup test"]
1430    async fn pinned_entrypoint_helper() {
1431        let home = PathBuf::from(std::env::var_os("MAGI_WORKER_TEST_HOME").unwrap());
1432        let id = std::env::var("MAGI_WORKER_TEST_ID").unwrap();
1433        assert!(run(&home, &id).await.is_err());
1434    }
1435
1436    #[test]
1437    fn unfinished_staging_is_reclaimed_without_removing_generations() {
1438        let home = tempfile::tempdir().unwrap();
1439        let old = root(home.path()).join("generations").join(".new-abandoned");
1440        std::fs::create_dir_all(&old).unwrap();
1441        let partial = old.join("partial");
1442        std::fs::write(&partial, b"interrupted copy").unwrap();
1443        #[cfg(windows)]
1444        {
1445            let mut permissions = std::fs::metadata(&partial).unwrap().permissions();
1446            permissions.set_readonly(true);
1447            std::fs::set_permissions(&partial, permissions).unwrap();
1448        }
1449        let source = home.path().join("source");
1450        std::fs::write(&source, b"complete executable").unwrap();
1451        let generation = pin_source(home.path(), &source).unwrap();
1452        assert!(!old.exists());
1453        assert!(generation.executable.is_file());
1454    }
1455
1456    #[tokio::test]
1457    async fn unrelated_executable_cannot_acknowledge_or_consume_reservation() {
1458        let home = tempfile::tempdir().unwrap();
1459        let source = home.path().join("other-binary");
1460        std::fs::write(&source, b"different executable of the same Cargo version").unwrap();
1461        let generation = pin_source(home.path(), &source).unwrap();
1462        let (job, _) = reserve(
1463            home.path(),
1464            &generation,
1465            "task",
1466            "task:one",
1467            serde_json::Value::Null,
1468        )
1469        .unwrap();
1470        assert!(
1471            run(home.path(), &job.id)
1472                .await
1473                .unwrap_err()
1474                .to_string()
1475                .contains("immutable generation")
1476        );
1477        assert_eq!(get(home.path(), &job.id).unwrap().status, Status::Starting);
1478        assert_eq!(active(home.path()).unwrap().len(), 1);
1479    }
1480
1481    #[test]
1482    fn abandoned_record_staging_is_bounded_and_never_executes() {
1483        let home = tempfile::tempdir().unwrap();
1484        let staging = root(home.path()).join("jobs").join(".new-interrupted");
1485        std::fs::create_dir_all(&staging).unwrap();
1486        std::fs::write(staging.join("job.json"), b"partial json").unwrap();
1487        assert!(active(home.path()).unwrap().is_empty());
1488        assert!(!staging.exists());
1489    }
1490
1491    #[cfg(unix)]
1492    #[test]
1493    fn detached_startup_preserves_stdout_and_stderr_in_record_log() {
1494        use std::os::unix::fs::PermissionsExt;
1495        let home = tempfile::tempdir().unwrap();
1496        let source = home.path().join("startup-diagnostic");
1497        std::fs::write(
1498            &source,
1499            b"#!/bin/sh\nprintf 'startup stdout\\n'\nprintf 'startup stderr\\n' >&2\nexit 7\n",
1500        )
1501        .unwrap();
1502        std::fs::set_permissions(&source, std::fs::Permissions::from_mode(0o755)).unwrap();
1503        let generation = pin_source(home.path(), &source).unwrap();
1504        let (mut job, _) = reserve(
1505            home.path(),
1506            &generation,
1507            "task",
1508            "task:log",
1509            serde_json::Value::Null,
1510        )
1511        .unwrap();
1512        let mut command = detached_command(home.path(), &job, &generation).unwrap();
1513        for key in ["MAGI_RUN", "MAGI_NODE", "MAGI_SEAT"]
1514            .into_iter()
1515            .chain(crate::chat_control::bridge::ENVIRONMENT.iter().copied())
1516        {
1517            assert!(
1518                command
1519                    .get_envs()
1520                    .any(|(name, value)| name == key && value.is_none())
1521            );
1522        }
1523        assert_eq!(command.spawn().unwrap().wait().unwrap().code(), Some(7));
1524        let log = job_dir(home.path(), &job.id).unwrap().join("worker.log");
1525        let output = std::fs::read_to_string(log).unwrap();
1526        assert!(output.contains("startup stdout"));
1527        assert!(output.contains("startup stderr"));
1528        terminal(
1529            home.path(),
1530            &mut job,
1531            Status::Failed,
1532            "startup failed".into(),
1533        )
1534        .unwrap();
1535        assert!(active(home.path()).unwrap().is_empty());
1536        assert!(
1537            root(home.path())
1538                .join("results")
1539                .join(&job.id)
1540                .join("worker.log")
1541                .is_file()
1542        );
1543    }
1544
1545    #[cfg(windows)]
1546    #[test]
1547    fn record_publication_survives_readers_and_short_delete_sharing_handoffs() {
1548        use std::os::windows::fs::OpenOptionsExt as _;
1549
1550        let home = tempfile::tempdir().unwrap();
1551        let mut job = reservation(home.path(), "task:metadata-handoff");
1552        let directory = job_dir(home.path(), &job.id).unwrap();
1553        let path = directory.join("job.json");
1554        let lease = try_lock(&directory.join("live.lock")).unwrap().unwrap();
1555        let snapshot = File::open(&path).unwrap();
1556        // This reader allows ordinary IO but temporarily denies replacement.
1557        let reader = OpenOptions::new()
1558            .read(true)
1559            .share_mode(1 | 2) // FILE_SHARE_READ | FILE_SHARE_WRITE
1560            .open(&path)
1561            .unwrap();
1562        std::thread::scope(|scope| {
1563            scope.spawn(move || {
1564                std::thread::sleep(Duration::from_millis(15));
1565                drop(reader);
1566            });
1567            job.status = Status::Running;
1568            save(&path, &job).unwrap();
1569        });
1570        let original: Job = serde_json::from_reader(snapshot).unwrap();
1571        assert_eq!(original.status, Status::Starting);
1572        assert_eq!(get(home.path(), &job.id).unwrap().status, Status::Running);
1573        let running_snapshot = File::open(&path).unwrap();
1574        terminal(home.path(), &mut job, Status::Finished, "completed".into()).unwrap();
1575        assert_eq!(get(home.path(), &job.id).unwrap().status, Status::Finished);
1576        let previous: Job = serde_json::from_reader(running_snapshot).unwrap();
1577        assert_eq!(previous.status, Status::Running);
1578        drop(lease);
1579        assert!(active(home.path()).unwrap().is_empty());
1580    }
1581
1582    #[cfg(windows)]
1583    #[test]
1584    fn denied_record_publication_preserves_reservation_and_original_os_error() {
1585        use std::os::windows::fs::OpenOptionsExt as _;
1586
1587        let home = tempfile::tempdir().unwrap();
1588        let job = reservation(home.path(), "task:metadata-denied");
1589        let directory = job_dir(home.path(), &job.id).unwrap();
1590        let path = directory.join("job.json");
1591        let reader = OpenOptions::new()
1592            .read(true)
1593            .share_mode(1 | 2) // Deny deletion for the entire handoff budget.
1594            .open(&path)
1595            .unwrap();
1596        let mut acknowledged = job.clone();
1597        acknowledged.status = Status::Running;
1598        let error = save(&path, &acknowledged).unwrap_err();
1599        assert!(format!("{error:#}").contains("publish worker record"));
1600        let io = error.downcast_ref::<std::io::Error>().unwrap();
1601        assert!(matches!(io.raw_os_error(), Some(5 | 32)));
1602        assert_eq!(get(home.path(), &job.id).unwrap().status, Status::Starting);
1603        assert_eq!(reservation(home.path(), &job.key).id, job.id);
1604        assert!(std::fs::read_dir(&directory).unwrap().all(|entry| {
1605            !entry
1606                .unwrap()
1607                .file_name()
1608                .to_string_lossy()
1609                .starts_with(".pending-")
1610        }));
1611        drop(reader);
1612        save(&path, &acknowledged).unwrap();
1613        assert_eq!(get(home.path(), &job.id).unwrap().status, Status::Running);
1614    }
1615
1616    #[test]
1617    fn terminal_result_survives_open_execution_and_output_handles() {
1618        let home = tempfile::tempdir().unwrap();
1619        let mut job = reservation(home.path(), "task:open-handles");
1620        let directory = job_dir(home.path(), &job.id).unwrap();
1621        let lease = try_lock(&directory.join("live.lock")).unwrap().unwrap();
1622        let mut log = OpenOptions::new()
1623            .create(true)
1624            .append(true)
1625            .open(directory.join("worker.log"))
1626            .unwrap();
1627        writeln!(log, "before completion").unwrap();
1628        terminal(home.path(), &mut job, Status::Finished, "completed".into()).unwrap();
1629        assert_eq!(get(home.path(), &job.id).unwrap().status, Status::Finished);
1630        assert!(active(home.path()).unwrap().is_empty());
1631        assert!(
1632            directory.exists(),
1633            "a held execution lease must prevent retirement"
1634        );
1635        drop(lease);
1636        // Retirement may still be denied by the output handle on Windows.
1637        // Either way, querying active work and the durable result must succeed.
1638        assert!(active(home.path()).unwrap().is_empty());
1639        assert_eq!(get(home.path(), &job.id).unwrap().status, Status::Finished);
1640        writeln!(log, "after completion").unwrap();
1641        drop(log);
1642        assert!(active(home.path()).unwrap().is_empty());
1643        assert!(
1644            root(home.path())
1645                .join("results")
1646                .join(&job.id)
1647                .join("worker.log")
1648                .is_file()
1649        );
1650        assert!(!directory.exists());
1651    }
1652
1653    #[test]
1654    fn terminal_records_are_bounded_without_pruning_live_reservations() {
1655        let home = tempfile::tempdir().unwrap();
1656        let live = reservation(home.path(), "task:live");
1657        // Exercise retention, not hundreds of durable reservation publications.
1658        // Use a real terminal record as the on-disk template; reservation and
1659        // terminal publication have their own tests above.
1660        let mut finished = reservation(home.path(), "task:finished");
1661        terminal(home.path(), &mut finished, Status::Finished, "done".into()).unwrap();
1662        let template_dir = job_dir(home.path(), &finished.id).unwrap();
1663        std::fs::remove_dir_all(template_dir).unwrap();
1664        for i in 0..RETAIN_FINISHED + 3 {
1665            let mut job = finished.clone();
1666            job.id = format!("{i:032x}");
1667            job.key = format!("task:{i}");
1668            job.sequence += i as u64;
1669            let directory = job_dir(home.path(), &job.id).unwrap();
1670            std::fs::create_dir_all(&directory).unwrap();
1671            std::fs::write(
1672                directory.join("job.json"),
1673                serde_json::to_vec(&job).unwrap(),
1674            )
1675            .unwrap();
1676        }
1677        assert_eq!(
1678            std::fs::read_dir(root(home.path()).join("jobs"))
1679                .unwrap()
1680                .count(),
1681            RETAIN_FINISHED + 4
1682        );
1683        let jobs = records(home.path()).unwrap();
1684        assert_eq!(jobs.len(), 1);
1685        assert_eq!(
1686            std::fs::read_dir(root(home.path()).join("results"))
1687                .unwrap()
1688                .count(),
1689            RETAIN_FINISHED
1690        );
1691        assert!(jobs.iter().any(|job| job.id == live.id));
1692    }
1693}