1use std::fs::{File, OpenOptions};
12use std::io::{Read, Write};
13use std::path::{Path, PathBuf};
14use std::process::{Command, Stdio};
15use std::sync::{LazyLock, Mutex, OnceLock};
16use std::time::{Duration, SystemTime, UNIX_EPOCH};
17
18use anyhow::{Context, Result, bail};
19use serde::{Deserialize, Serialize};
20use sha2::{Digest, Sha256};
21
22const STARTUP_SECONDS: u64 = 30;
23const RETAIN_FINISHED: usize = 256;
24
25pub const INVOCATION_ENV: [&str; 4] = [
30 "MAGI_WORKER_ID",
31 "MAGI_WORKER_GENERATION",
32 "MAGI_WORKER_HOME",
33 "MAGI_WORKER_EXE",
34];
35
36#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
38#[serde(rename_all = "snake_case")]
39pub enum Status {
40 Starting,
42 Running,
44 Finished,
46 Failed,
48}
49
50#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
52#[serde(rename_all = "snake_case")]
53pub enum Control {
54 Park,
56 Stop,
58}
59
60#[derive(Debug)]
61struct NotStarted(anyhow::Error);
62
63impl std::fmt::Display for NotStarted {
64 fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
65 write!(formatter, "{:#}", self.0)
66 }
67}
68
69impl std::error::Error for NotStarted {
70 fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
71 Some(self.0.as_ref())
72 }
73}
74
75fn not_started_error(error: impl Into<anyhow::Error>) -> anyhow::Error {
76 anyhow::Error::new(NotStarted(error.into()))
77}
78
79pub fn not_started(error: &anyhow::Error) -> bool {
85 error.downcast_ref::<NotStarted>().is_some()
86}
87
88#[derive(Debug, thiserror::Error)]
90#[error(
91 "worker {id} startup is unacknowledged; its reservation remains occupied; inspect or retry the same operation key"
92)]
93pub(crate) struct StartupUnacknowledged {
94 pub(crate) id: String,
95}
96
97#[derive(Clone, Debug, Serialize, Deserialize)]
99pub struct Job {
100 pub id: String,
102 #[serde(default)]
104 pub sequence: u64,
105 pub kind: String,
107 pub key: String,
109 pub payload: serde_json::Value,
111 pub generation: String,
113 #[serde(default)]
115 pub version: Option<String>,
116 pub status: Status,
118 pub result: Option<String>,
120 pub pid: Option<u32>,
122 pub created_at: u64,
124 pub finished_at: Option<u64>,
126}
127
128#[derive(Clone, Debug)]
130pub struct Generation {
131 pub id: String,
133 pub executable: PathBuf,
135}
136
137#[derive(Debug)]
139pub struct Lease {
140 _file: File,
141}
142
143impl Drop for Lease {
144 fn drop(&mut self) {
145 #[cfg(unix)]
148 {
149 use std::os::fd::AsRawFd;
150 unsafe { libc::flock(self._file.as_raw_fd(), libc::LOCK_UN) };
151 }
152 }
153}
154
155fn root(home: &Path) -> PathBuf {
156 home.join("workers")
157}
158fn now() -> u64 {
159 SystemTime::now()
160 .duration_since(UNIX_EPOCH)
161 .unwrap_or_default()
162 .as_secs()
163}
164fn nonce() -> String {
165 crate::rng::SplitMix64::new(crate::rng::entropy()).uuid_v4()
166}
167fn valid_id(id: &str) -> bool {
168 !id.is_empty() && id.bytes().all(|b| b.is_ascii_hexdigit() || b == b'-')
169}
170fn job_dir(home: &Path, id: &str) -> Result<PathBuf> {
171 if !valid_id(id) {
172 bail!("invalid worker id");
173 }
174 Ok(root(home).join("jobs").join(id))
175}
176
177fn open_lock(path: &Path) -> Result<File> {
178 if let Some(parent) = path.parent() {
179 std::fs::create_dir_all(parent)
180 .with_context(|| format!("create worker lock directory {}", parent.display()))?;
181 }
182 OpenOptions::new()
183 .create(true)
184 .truncate(false)
185 .read(true)
186 .write(true)
187 .open(path)
188 .with_context(|| format!("open worker lock {}", path.display()))
189}
190
191fn try_lock(path: &Path) -> Result<Option<Lease>> {
192 let file = open_lock(path)?;
193 match crate::pr_head::try_lock_file(&file) {
194 Ok(()) => Ok(Some(Lease { _file: file })),
195 Err(e)
196 if e.kind() == std::io::ErrorKind::WouldBlock
197 || matches!(e.raw_os_error(), Some(33)) =>
198 {
199 Ok(None)
200 }
201 Err(e) => Err(e).with_context(|| format!("acquire worker lock {}", path.display())),
202 }
203}
204
205#[cfg(test)]
206pub(crate) fn execution_lease_for_test(path: &Path) -> Lease {
207 try_lock(path)
208 .expect("execution lease")
209 .expect("free execution lease")
210}
211
212pub fn try_lease(home: &Path, name: &str) -> Result<Option<Lease>> {
215 try_lock(
216 &root(home)
217 .join("leases")
218 .join(format!("{}.lock", digest(name.as_bytes()))),
219 )
220}
221
222fn store_lock(home: &Path) -> Result<Lease> {
223 let deadline = std::time::Instant::now() + Duration::from_secs(10);
224 loop {
225 if let Some(lease) = try_lock(&root(home).join("store.lock"))? {
226 return Ok(lease);
227 }
228 if std::time::Instant::now() >= deadline {
229 bail!("worker store is busy; retry later");
230 }
231 std::thread::sleep(Duration::from_millis(5));
232 }
233}
234
235#[cfg(unix)]
236fn sync_dir(path: &Path) -> Result<()> {
237 File::open(path)
238 .with_context(|| format!("open worker directory for sync {}", path.display()))?
239 .sync_all()
240 .with_context(|| format!("sync worker directory {}", path.display()))?;
241 Ok(())
242}
243
244#[cfg(not(unix))]
245fn sync_dir(_path: &Path) -> Result<()> {
246 Ok(())
247}
248
249fn save(path: &Path, value: &impl Serialize) -> Result<()> {
250 let parent = path.parent().context("worker record has no parent")?;
251 std::fs::create_dir_all(parent)
252 .with_context(|| format!("create worker record directory {}", parent.display()))?;
253 let temp = parent.join(format!(".pending-{}", nonce()));
254 let result = (|| -> Result<()> {
255 let mut file = OpenOptions::new()
256 .create_new(true)
257 .write(true)
258 .open(&temp)
259 .with_context(|| format!("create staged worker record {}", temp.display()))?;
260 serde_json::to_writer(&mut file, value)
261 .with_context(|| format!("serialize worker record {}", path.display()))?;
262 file.flush()
263 .with_context(|| format!("flush staged worker record {}", temp.display()))?;
264 file.sync_all()
265 .with_context(|| format!("sync staged worker record {}", temp.display()))?;
266 drop(file);
269 replace(&temp, path).with_context(|| {
270 format!(
271 "publish worker record {} -> {}",
272 temp.display(),
273 path.display()
274 )
275 })?;
276 sync_dir(parent)
277 })();
278 if result.is_err() {
279 let _ = std::fs::remove_file(&temp);
280 }
281 result
282}
283
284#[cfg(not(windows))]
285fn replace(from: &Path, to: &Path) -> Result<()> {
286 Ok(std::fs::rename(from, to)?)
287}
288
289#[cfg(windows)]
290fn replace(from: &Path, to: &Path) -> Result<()> {
291 crate::lock_file::Retry::new(None).run(|| std::fs::rename(from, to))?;
296 Ok(())
297}
298
299fn open_record(path: &Path) -> std::io::Result<File> {
302 #[cfg(windows)]
303 {
304 crate::lock_file::Retry::new(None).run(|| File::open(path))
305 }
306 #[cfg(not(windows))]
307 {
308 File::open(path)
309 }
310}
311
312pub fn get(home: &Path, id: &str) -> Result<Job> {
314 let path = job_dir(home, id)?.join("job.json");
315 let file = match open_record(&path) {
316 Ok(file) => file,
317 Err(error) if error.kind() == std::io::ErrorKind::NotFound => {
318 let retained = root(home).join("results").join(id).join("job.json");
319 open_record(&retained)
320 .with_context(|| format!("open retained worker record {}", retained.display()))?
321 }
322 Err(error) => {
323 return Err(error).with_context(|| format!("open worker record {}", path.display()));
324 }
325 };
326 serde_json::from_reader(file).with_context(|| format!("decode worker record {id}"))
327}
328
329pub fn current_job(home: &Path) -> Result<Option<Job>> {
333 match std::env::var("MAGI_WORKER_ID") {
334 Ok(id) => Ok(Some(get(home, &id)?)),
335 Err(std::env::VarError::NotPresent) => Ok(None),
336 Err(error) => Err(error.into()),
337 }
338}
339
340fn next_sequence(home: &Path) -> Result<u64> {
341 let path = root(home).join("sequence.json");
342 let previous: u64 = match open_record(&path) {
343 Ok(file) => serde_json::from_reader(file)
344 .with_context(|| format!("decode worker sequence {}", path.display()))?,
345 Err(error) if error.kind() == std::io::ErrorKind::NotFound => 0,
346 Err(error) => {
347 return Err(error).with_context(|| format!("open worker sequence {}", path.display()));
348 }
349 };
350 let sequence = previous
351 .checked_add(1)
352 .context("worker reservation sequence exhausted")?;
353 save(&path, &sequence)?;
354 Ok(sequence)
355}
356
357fn records(home: &Path) -> Result<Vec<Job>> {
358 let directory = root(home).join("jobs");
359 std::fs::create_dir_all(&directory)?;
360 let mut jobs = Vec::new();
361 let mut retired = false;
362 for entry in std::fs::read_dir(directory)? {
363 let entry = entry?;
364 if !entry.file_type()?.is_dir() {
365 continue;
366 }
367 let name = entry.file_name();
368 let Some(id) = name.to_str() else {
369 continue;
370 };
371 if id.starts_with(".new-") {
372 std::fs::remove_dir_all(entry.path())?;
375 continue;
376 }
377 if id.starts_with('.') {
378 continue;
379 }
380 for pending in std::fs::read_dir(entry.path())? {
381 let pending = pending?;
382 if pending.file_type()?.is_file()
383 && pending
384 .file_name()
385 .to_string_lossy()
386 .starts_with(".pending-")
387 {
388 std::fs::remove_file(pending.path())?;
389 }
390 }
391 let job = get(home, id).with_context(|| format!("read worker {id}"))?;
392 if matches!(job.status, Status::Starting | Status::Running) {
393 jobs.push(job);
394 } else {
395 retired |= retire(home, &job);
396 }
397 }
398 if retired {
399 if let Err(error) = prune_results(home) {
400 tracing::warn!(%error, "worker result retention deferred");
401 }
402 }
403 Ok(jobs)
404}
405
406fn terminal(home: &Path, job: &mut Job, status: Status, result: String) -> Result<()> {
407 job.status = status;
408 job.result = Some(result);
409 job.finished_at = Some(now());
410 save(&job_dir(home, &job.id)?.join("job.json"), job)
411 .with_context(|| format!("publish worker {} terminal status {status:?}", job.id))?;
412 clear_control_locked(home, &job.key)
416}
417
418fn retire(home: &Path, job: &Job) -> bool {
419 let attempt = || -> Result<bool> {
420 let dir = job_dir(home, &job.id)?;
421 let Some(lease) = try_lock(&dir.join("live.lock"))? else {
422 return Ok(false);
423 };
424 drop(lease);
425 archive(home, job)?;
426 Ok(true)
427 };
428 match attempt() {
433 Ok(retired) => retired,
434 Err(error) => {
435 tracing::warn!(
436 worker = %job.id,
437 error = %format_args!("{error:#}"),
438 "worker result retirement deferred"
439 );
440 false
441 }
442 }
443}
444
445fn archive(home: &Path, job: &Job) -> Result<()> {
446 let results = root(home).join("results");
447 std::fs::create_dir_all(&results)
448 .with_context(|| format!("create worker results directory {}", results.display()))?;
449 let from = job_dir(home, &job.id)?;
450 let to = results.join(&job.id);
451 std::fs::rename(&from, &to).with_context(|| {
452 format!(
453 "archive worker result {} -> {}",
454 from.display(),
455 to.display()
456 )
457 })?;
458 sync_dir(&root(home).join("jobs"))?;
459 sync_dir(&results)
460}
461
462fn reconcile(home: &Path, jobs: &mut [Job]) -> Result<()> {
465 let mut retired = false;
466 for job in jobs {
467 if !matches!(job.status, Status::Starting | Status::Running) {
468 continue;
469 }
470 let lease = try_lock(&job_dir(home, &job.id)?.join("live.lock"))?;
471 if lease.is_none() {
472 continue;
473 }
474 if job.status == Status::Starting && now().saturating_sub(job.created_at) < STARTUP_SECONDS
475 {
476 continue;
477 }
478 let reason = if job.status == Status::Starting {
479 "worker startup reservation expired before acknowledgement"
480 } else {
481 "worker exited without recording a result"
482 };
483 terminal(home, job, Status::Failed, reason.to_owned())?;
484 drop(lease);
485 retired |= retire(home, job);
486 }
487 if retired {
488 if let Err(error) = prune_results(home) {
489 tracing::warn!(%error, "worker result retention deferred");
490 }
491 }
492 Ok(())
493}
494
495pub fn active(home: &Path) -> Result<Vec<Job>> {
498 let _store = store_lock(home)?;
499 let mut jobs = records(home)?;
500 reconcile(home, &mut jobs)?;
501 Ok(jobs
502 .into_iter()
503 .filter(|job| matches!(job.status, Status::Starting | Status::Running))
504 .collect())
505}
506
507pub fn observe_active(home: &Path) -> Result<Vec<Job>> {
512 let entries = match std::fs::read_dir(root(home).join("jobs")) {
513 Ok(entries) => entries,
514 Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(Vec::new()),
515 Err(error) => return Err(error.into()),
516 };
517 let mut active = Vec::new();
518 for entry in entries {
519 let entry = entry?;
520 if !entry.file_type()?.is_dir() {
521 continue;
522 }
523 let name = entry.file_name();
524 let id = name.to_str().context("invalid worker reservation name")?;
525 if id.starts_with('.') {
526 continue;
527 }
528 let job = get(home, id)?;
529 if observe_job_active(home, &job)? {
530 active.push(job);
531 }
532 }
533 Ok(active)
534}
535
536pub(crate) fn observe_job_active(home: &Path, job: &Job) -> Result<bool> {
538 match job.status {
539 Status::Starting => Ok(true),
540 Status::Running => {
541 let file = match File::open(job_dir(home, &job.id)?.join("live.lock")) {
542 Ok(file) => file,
543 Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(true),
544 Err(error) => return Err(error.into()),
545 };
546 match crate::pr_head::try_lock_file(&file) {
547 Ok(()) => {
548 drop(Lease { _file: file });
549 Ok(false)
550 }
551 Err(error)
552 if error.kind() == std::io::ErrorKind::WouldBlock
553 || cfg!(windows) && error.raw_os_error() == Some(33) =>
554 {
555 Ok(true)
556 }
557 Err(error) => Err(error.into()),
558 }
559 }
560 Status::Finished | Status::Failed => Ok(false),
561 }
562}
563
564fn control_path(home: &Path, key: &str) -> PathBuf {
565 root(home)
566 .join("controls")
567 .join(format!("{}.json", digest(key.as_bytes())))
568}
569
570pub fn request(home: &Path, key: &str, control: Control) -> Result<()> {
572 let _store = store_lock(home)?;
573 save(&control_path(home, key), &control)
574}
575
576pub fn control(home: &Path, key: &str) -> Result<Option<Control>> {
578 let path = control_path(home, key);
579 match open_record(&path) {
580 Ok(file) => serde_json::from_reader(file)
581 .map(Some)
582 .with_context(|| format!("decode worker control {}", path.display())),
583 Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(None),
584 Err(e) => Err(e).with_context(|| format!("open worker control {}", path.display())),
585 }
586}
587
588fn clear_control_locked(home: &Path, key: &str) -> Result<()> {
589 let path = control_path(home, key);
590 match std::fs::remove_file(&path) {
591 Ok(()) => sync_dir(&root(home).join("controls")),
592 Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(()),
593 Err(e) => Err(e).with_context(|| format!("remove worker control {}", path.display())),
594 }
595}
596
597pub fn clear_control(home: &Path, key: &str) -> Result<()> {
599 let _store = store_lock(home)?;
600 clear_control_locked(home, key)
601}
602
603pub fn executable(home: &Path, generation: &str) -> Result<PathBuf> {
605 if generation.len() != 64 || !generation.bytes().all(|b| b.is_ascii_hexdigit()) {
606 bail!("invalid worker generation");
607 }
608 let path = root(home)
609 .join("generations")
610 .join(generation)
611 .join(if cfg!(windows) { "magi.exe" } else { "magi" });
612 if !path.is_file() {
613 bail!("worker generation {generation} is unavailable");
614 }
615 Ok(path)
616}
617
618pub fn helper_executable(home: &Path) -> Result<PathBuf> {
621 if let Some(generation) = std::env::var_os("MAGI_WORKER_GENERATION") {
622 return executable(
623 home,
624 generation
625 .to_str()
626 .context("invalid generation environment")?,
627 );
628 }
629 Ok(pin_current(home)?.executable)
630}
631
632pub fn remember_current_executable() -> Result<()> {
636 if INSTALLED_EXECUTABLE.get().is_some() {
637 return Ok(());
638 }
639 let path = std::env::current_exe()?.canonicalize()?;
640 let _ = INSTALLED_EXECUTABLE.set(path);
641 Ok(())
642}
643
644static INSTALLED_EXECUTABLE: OnceLock<PathBuf> = OnceLock::new();
645
646pub fn pin_current(home: &Path) -> Result<Generation> {
649 let source = match std::env::var_os("MAGI_WORKER_EXE") {
650 Some(path) => PathBuf::from(path),
651 None => match INSTALLED_EXECUTABLE.get() {
652 Some(path) => path.clone(),
653 None => std::env::current_exe()?,
654 },
655 };
656 let source = source.canonicalize().context("resolve worker executable")?;
657 if let Some(directory) = source.parent() {
660 if directory.parent() == Some(root(home).join("generations").as_path()) {
661 if let Some(id) = directory.file_name().and_then(|name| name.to_str()) {
662 if id.len() == 64
663 && id.bytes().all(|b| b.is_ascii_hexdigit())
664 && executable(home, id)?.canonicalize()? == source
665 {
666 return Ok(Generation {
667 id: id.to_owned(),
668 executable: source,
669 });
670 }
671 }
672 }
673 }
674 let stamp = source_stamp(&source)?;
675 let mut cache = PIN_CACHE
676 .lock()
677 .map_err(|_| anyhow::anyhow!("worker generation cache poisoned"))?;
678 if let Some(entry) = cache.iter().find(|entry| {
679 entry.home == home
680 && entry.source == source
681 && entry.stamp == stamp
682 && entry.generation.executable.is_file()
683 }) {
684 return Ok(entry.generation.clone());
685 }
686 let generation = pin_source(home, &source)?;
687 if source_stamp(&source)? != stamp {
688 bail!("worker executable changed while being pinned; retry");
689 }
690 if cache.len() >= 8 {
691 cache.remove(0);
692 }
693 cache.push(CachedGeneration {
694 home: home.to_path_buf(),
695 source,
696 stamp,
697 generation: generation.clone(),
698 });
699 Ok(generation)
700}
701
702type SourceStamp = (u64, Option<SystemTime>, Option<SystemTime>, u64, u64);
703struct CachedGeneration {
704 home: PathBuf,
705 source: PathBuf,
706 stamp: SourceStamp,
707 generation: Generation,
708}
709static PIN_CACHE: LazyLock<Mutex<Vec<CachedGeneration>>> = LazyLock::new(|| Mutex::new(Vec::new()));
710
711fn source_stamp(path: &Path) -> Result<SourceStamp> {
712 let metadata = std::fs::metadata(path)?;
713 #[cfg(unix)]
714 let identity = {
715 use std::os::unix::fs::MetadataExt;
716 (metadata.dev(), metadata.ino())
717 };
718 #[cfg(not(unix))]
719 let identity = (0, 0);
720 Ok((
721 metadata.len(),
722 metadata.modified().ok(),
723 metadata.created().ok(),
724 identity.0,
725 identity.1,
726 ))
727}
728
729fn pin_source(home: &Path, source: &Path) -> Result<Generation> {
730 let generations = root(home).join("generations");
731 std::fs::create_dir_all(&generations)?;
732 let staging = generations.join(format!(".new-{}", nonce()));
733 let mut staging_lease = {
734 let _store = store_lock(home)?;
735 for entry in std::fs::read_dir(&generations)? {
737 let entry = entry?;
738 if entry.file_type()?.is_dir()
739 && entry.file_name().to_string_lossy().starts_with(".new-")
740 {
741 if let Some(lease) = try_lock(&entry.path().join("pin.lock"))? {
742 drop(lease);
743 remove_staging(&entry.path())?;
744 }
745 }
746 }
747 std::fs::create_dir(&staging)?;
748 Some(try_lock(&staging.join("pin.lock"))?.context("generation staging lease unavailable")?)
749 };
750 let temp = staging.join(if cfg!(windows) { "magi.exe" } else { "magi" });
751 let result = (|| -> Result<Generation> {
752 let mut input = File::open(source)?;
753 let mut output = OpenOptions::new()
754 .create_new(true)
755 .write(true)
756 .open(&temp)?;
757 let mut hash = Sha256::new();
758 let mut buffer = [0u8; 65536];
759 loop {
760 let n = input.read(&mut buffer)?;
761 if n == 0 {
762 break;
763 }
764 hash.update(&buffer[..n]);
765 output.write_all(&buffer[..n])?;
766 }
767 let id = hex_digest(hash);
768 let mut permissions = input.metadata()?.permissions();
769 permissions.set_readonly(true);
770 output.set_permissions(permissions)?;
771 output.sync_all()?;
772 drop(output);
773 sync_dir(&staging)?;
774 let target = generations.join(&id);
775 let _store = store_lock(home)?;
777 drop(staging_lease.take());
778 std::fs::remove_file(staging.join("pin.lock"))?;
779 if target.exists() {
780 let existing = target.join(temp.file_name().context("binary filename")?);
781 if hash_file(&existing)? != id {
782 bail!("immutable worker generation is corrupt: {id}");
783 }
784 } else {
785 std::fs::rename(&staging, &target)?;
786 sync_dir(&generations)?;
787 }
788 Ok(Generation {
789 executable: target.join(temp.file_name().context("binary filename")?),
790 id,
791 })
792 })();
793 let _store = store_lock(home)?;
794 drop(staging_lease);
795 if staging.exists() {
796 remove_staging(&staging)?;
797 }
798 result
799}
800
801fn remove_staging(staging: &Path) -> Result<()> {
802 #[cfg(windows)]
803 {
804 use std::os::windows::{ffi::OsStrExt, fs::MetadataExt};
805 #[link(name = "kernel32")]
806 unsafe extern "system" {
807 fn SetFileAttributesW(path: *const u16, attributes: u32) -> i32;
808 }
809 const READONLY: u32 = 0x1;
810 for entry in std::fs::read_dir(staging)? {
811 let entry = entry?;
812 let attributes = entry.metadata()?.file_attributes();
813 if attributes & READONLY != 0 {
814 let path: Vec<u16> = entry.path().as_os_str().encode_wide().chain([0]).collect();
815 if unsafe { SetFileAttributesW(path.as_ptr(), attributes & !READONLY) } == 0 {
818 return Err(std::io::Error::last_os_error())
819 .context("clear staging readonly attribute");
820 }
821 }
822 }
823 }
824 Ok(std::fs::remove_dir_all(staging)?)
825}
826
827fn hash_file(path: &Path) -> Result<String> {
828 let mut file = File::open(path)?;
829 let mut hash = Sha256::new();
830 let mut buffer = [0u8; 65536];
831 loop {
832 let n = file.read(&mut buffer)?;
833 if n == 0 {
834 break;
835 }
836 hash.update(&buffer[..n]);
837 }
838 Ok(hex_digest(hash))
839}
840
841fn reserve(
842 home: &Path,
843 generation: &Generation,
844 kind: &str,
845 key: &str,
846 payload: serde_json::Value,
847) -> Result<(Job, bool)> {
848 if kind.is_empty() || key.is_empty() {
849 bail!("worker kind and operation key must be nonempty");
850 }
851 let _store = store_lock(home)?;
852 let mut jobs = records(home)?;
853 reconcile(home, &mut jobs)?;
854 if let Some(job) = jobs
855 .iter()
856 .find(|job| job.key == key && matches!(job.status, Status::Starting | Status::Running))
857 {
858 if job.kind != kind || job.payload != payload {
859 bail!("worker operation key {key} is already reserved for a different request");
860 }
861 return Ok((job.clone(), false));
862 }
863 let id = nonce();
864 let job = Job {
865 id: id.clone(),
866 sequence: next_sequence(home)?,
867 kind: kind.to_owned(),
868 key: key.to_owned(),
869 payload,
870 generation: generation.id.clone(),
871 version: None,
872 status: Status::Starting,
873 result: None,
874 pid: None,
875 created_at: now(),
876 finished_at: None,
877 };
878 let jobs_dir = root(home).join("jobs");
879 let staging = jobs_dir.join(format!(".new-{id}"));
880 std::fs::create_dir(&staging)
881 .with_context(|| format!("create worker reservation staging {}", staging.display()))?;
882 save(&staging.join("job.json"), &job)?;
883 clear_control_locked(home, key)?;
884 std::fs::rename(&staging, job_dir(home, &id)?)
885 .with_context(|| format!("publish worker {id} reservation from {}", staging.display()))?;
886 sync_dir(&jobs_dir)?;
887 Ok((job, true))
888}
889
890fn prune_results(home: &Path) -> Result<()> {
891 let results = root(home).join("results");
892 std::fs::create_dir_all(&results)?;
893 let mut entries = Vec::new();
894 for entry in std::fs::read_dir(&results)? {
895 let entry = entry?;
896 if entry.file_type()?.is_dir() {
897 entries.push((entry.metadata()?.modified()?, entry.path()));
900 }
901 }
902 entries.sort_unstable_by_key(|(modified, _)| std::cmp::Reverse(*modified));
903 for (_, dir) in entries.into_iter().skip(RETAIN_FINISHED) {
904 if let Some(lease) = try_lock(&dir.join("live.lock"))? {
905 drop(lease);
906 std::fs::remove_dir_all(dir)?;
907 }
908 }
909 sync_dir(&results)
910}
911
912fn detached_command(home: &Path, job: &Job, generation: &Generation) -> Result<Command> {
913 let log_path = job_dir(home, &job.id)?.join("worker.log");
917 let log = OpenOptions::new()
918 .create(true)
919 .append(true)
920 .open(&log_path)
921 .with_context(|| format!("open worker startup log {}", log_path.display()))?;
922 let stderr = log
923 .try_clone()
924 .with_context(|| format!("clone worker startup log {}", log_path.display()))?;
925 let mut command = Command::new(&generation.executable);
926 command
927 .arg("__worker")
928 .arg("--home")
929 .arg(home)
930 .arg(&job.id)
931 .env("MAGI_HOME", home)
932 .env("MAGI_WORKER_HOME", home)
933 .env("MAGI_WORKER_ID", &job.id)
934 .env("MAGI_WORKER_GENERATION", &generation.id)
935 .env("MAGI_WORKER_EXE", &generation.executable)
936 .env_remove("MAGI_RUN")
937 .env_remove("MAGI_NODE")
938 .env_remove("MAGI_SEAT")
939 .stdin(Stdio::null())
940 .stdout(Stdio::from(log))
941 .stderr(Stdio::from(stderr));
942 crate::chat_control::bridge::clear_std_environment(&mut command);
943 let mut paths = vec![
944 generation
945 .executable
946 .parent()
947 .context("generation directory")?
948 .to_path_buf(),
949 ];
950 if let Some(path) = std::env::var_os("PATH") {
951 paths.extend(std::env::split_paths(&path));
952 }
953 command.env("PATH", std::env::join_paths(paths)?);
954 #[cfg(unix)]
955 {
956 use std::os::unix::process::CommandExt;
957 unsafe {
958 command.pre_exec(|| {
959 if libc::setsid() < 0 {
960 return Err(std::io::Error::last_os_error());
961 }
962 libc::signal(libc::SIGINT, libc::SIG_DFL);
965 libc::signal(libc::SIGTERM, libc::SIG_DFL);
966 Ok(())
967 });
968 }
969 }
970 #[cfg(windows)]
971 {
972 use std::os::windows::process::CommandExt;
973 detached_standard_handles().context("exclude caller pipes from detached worker")?;
974 command.creation_flags(
975 crate::chat_watch::windows::detached_flags()
976 .context("prepare independent Windows worker")?,
977 );
978 }
979 Ok(command)
980}
981
982#[cfg(windows)]
983fn detached_standard_handles() -> std::io::Result<()> {
984 use std::ffi::c_void;
985 #[link(name = "kernel32")]
986 unsafe extern "system" {
987 fn GetStdHandle(kind: u32) -> *mut c_void;
988 fn SetHandleInformation(handle: *mut c_void, mask: u32, flags: u32) -> i32;
989 }
990 for kind in [-10_i32, -11, -12] {
996 let handle = unsafe { GetStdHandle(kind as u32) };
997 if handle.is_null() || handle as isize == -1 {
998 continue;
999 }
1000 if unsafe { SetHandleInformation(handle, 1, 0) } == 0 {
1001 let error = std::io::Error::last_os_error();
1002 if error.raw_os_error() != Some(6) {
1004 return Err(error);
1005 }
1006 }
1007 }
1008 Ok(())
1009}
1010
1011pub async fn spawn(home: &Path, kind: &str, key: &str, payload: serde_json::Value) -> Result<Job> {
1016 std::fs::create_dir_all(home).map_err(not_started_error)?;
1017 let home = home
1018 .canonicalize()
1019 .context("resolve worker home")
1020 .map_err(not_started_error)?;
1021 let pin_home = home.clone();
1022 let generation = tokio::task::spawn_blocking(move || pin_current(&pin_home))
1023 .await
1024 .map_err(not_started_error)?
1025 .map_err(not_started_error)?;
1026 let (job, fresh) = reserve(&home, &generation, kind, key, payload)?;
1027 if fresh {
1028 match detached_command(&home, &job, &generation)
1029 .and_then(|mut command| Ok(command.spawn()?))
1030 {
1031 Ok(mut child) => {
1032 std::thread::spawn(move || {
1035 let _ = child.wait();
1036 });
1037 }
1038 Err(error) => {
1039 let _store = store_lock(&home)?;
1040 let mut job = get(&home, &job.id)?;
1041 if job.status == Status::Starting {
1042 if let Some(_lease) = try_lock(&job_dir(&home, &job.id)?.join("live.lock"))? {
1043 terminal(
1044 &home,
1045 &mut job,
1046 Status::Failed,
1047 format!("worker spawn failed: {error}"),
1048 )?;
1049 return Err(not_started_error(error));
1050 }
1051 }
1052 return Err(error);
1053 }
1054 }
1055 }
1056 let deadline = tokio::time::Instant::now() + Duration::from_secs(10);
1057 loop {
1058 let current = get(&home, &job.id)?;
1059 match current.status {
1060 Status::Running | Status::Finished => return Ok(current),
1061 Status::Failed => bail!(
1062 "worker {} failed: {}",
1063 current.id,
1064 current.result.as_deref().unwrap_or("unknown failure")
1065 ),
1066 Status::Starting => {}
1067 }
1068 if tokio::time::Instant::now() >= deadline {
1069 return Err(StartupUnacknowledged { id: job.id }.into());
1070 }
1071 tokio::time::sleep(Duration::from_millis(20)).await;
1072 }
1073}
1074
1075pub async fn run(home: &Path, id: &str) -> Result<()> {
1079 let dir = job_dir(home, id)?;
1080 let (job, _lease) = {
1081 let _store = store_lock(home)?;
1082 let mut job = get(home, id)?;
1083 if job.status != Status::Starting {
1084 bail!("worker reservation is no longer startable");
1085 }
1086 let lease =
1087 try_lock(&dir.join("live.lock"))?.context("worker execution lease is already held")?;
1088 let current = std::env::current_exe()
1089 .context("locate running worker executable")?
1090 .canonicalize()
1091 .context("resolve running worker executable")?;
1092 let expected = executable(home, &job.generation)?
1093 .canonicalize()
1094 .with_context(|| {
1095 format!("resolve worker {id} reserved generation {}", job.generation)
1096 })?;
1097 if current != expected {
1098 bail!("worker must execute its reserved immutable generation");
1099 }
1100 job.pid = Some(std::process::id());
1101 job.version = Some(env!("CARGO_PKG_VERSION").to_owned());
1102 job.status = Status::Running;
1103 save(&dir.join("job.json"), &job)
1104 .with_context(|| format!("acknowledge worker {id} startup"))?;
1105 (job, lease)
1106 };
1107 let result = match job.kind.as_str() {
1108 "task" => crate::daemon::run_task_worker(home, job.payload.clone()).await,
1109 "scheduler" => crate::daemon::run_scheduler_worker(home, job.payload.clone()).await,
1110 "chat" => crate::chat_worker::run_turn_worker(home, job.payload.clone()).await,
1111 "service" => crate::consult::run_service_worker(home, job.payload.clone()).await,
1112 "daemon-service" => crate::daemon::run_service_worker(home, job.payload.clone()).await,
1113 "deputy" => crate::deputy::run_worker(home, job.payload.clone()).await,
1114 kind => Err(anyhow::anyhow!("unknown worker kind {kind}")),
1115 };
1116 let _store = store_lock(home)?;
1117 let mut current = get(home, id)?;
1118 match &result {
1119 Ok(()) => terminal(home, &mut current, Status::Finished, "completed".to_owned())?,
1120 Err(error) => terminal(home, &mut current, Status::Failed, format!("{error:#}"))?,
1121 }
1122 result
1123}
1124
1125fn hex_digest(hash: Sha256) -> String {
1126 hash.finalize()
1129 .iter()
1130 .map(|byte| format!("{byte:02x}"))
1131 .collect()
1132}
1133fn digest(bytes: &[u8]) -> String {
1134 let mut hash = Sha256::new();
1135 hash.update(bytes);
1136 hex_digest(hash)
1137}
1138
1139#[cfg(test)]
1140mod tests {
1141 use super::*;
1142
1143 #[test]
1144 fn observing_an_unused_store_requires_no_writes() {
1145 let home = tempfile::tempdir().unwrap();
1146 assert!(observe_active(home.path()).unwrap().is_empty());
1147 assert!(!home.path().join("workers").exists());
1148 }
1149
1150 #[test]
1151 fn observing_execution_probes_the_lease_without_rewriting_history() {
1152 let home = tempfile::tempdir().unwrap();
1153 let mut job = reservation(home.path(), "task:observer");
1154 let directory = job_dir(home.path(), &job.id).unwrap();
1155 let live_path = directory.join("live.lock");
1156 let lease = try_lock(&live_path).unwrap().unwrap();
1157 job.status = Status::Running;
1158 save(&directory.join("job.json"), &job).unwrap();
1159 #[cfg(unix)]
1160 {
1161 use std::os::unix::fs::PermissionsExt as _;
1162 std::fs::set_permissions(&live_path, std::fs::Permissions::from_mode(0o444)).unwrap();
1163 }
1164 let before = std::fs::read(directory.join("job.json")).unwrap();
1165 let observed = observe_active(home.path()).unwrap();
1166 assert_eq!(observed.len(), 1);
1167 assert_eq!(observed[0].id, job.id);
1168 drop(lease);
1169 assert!(observe_active(home.path()).unwrap().is_empty());
1170 assert_eq!(std::fs::read(directory.join("job.json")).unwrap(), before);
1171 assert_eq!(get(home.path(), &job.id).unwrap().status, Status::Running);
1172 }
1173 fn generation() -> Generation {
1174 Generation {
1175 id: "a".repeat(64),
1176 executable: PathBuf::from("unused"),
1177 }
1178 }
1179 fn reservation(home: &Path, key: &str) -> Job {
1180 reserve(
1181 home,
1182 &generation(),
1183 "task",
1184 key,
1185 serde_json::json!({"task":"example"}),
1186 )
1187 .unwrap()
1188 .0
1189 }
1190
1191 #[test]
1192 fn reservation_deduplicates_identical_keys_and_rejects_collision() {
1193 let home = tempfile::tempdir().unwrap();
1194 let first = reservation(home.path(), "task:one");
1195 assert_eq!(first.id, reservation(home.path(), "task:one").id);
1196 assert!(
1197 reserve(
1198 home.path(),
1199 &generation(),
1200 "chat",
1201 "task:one",
1202 serde_json::Value::Null
1203 )
1204 .is_err()
1205 );
1206 assert_ne!(first.id, reservation(home.path(), "task:two").id);
1207 }
1208
1209 #[test]
1210 fn reservation_order_is_durable_and_corruption_never_resets_it() {
1211 let home = tempfile::tempdir().unwrap();
1212 let first = reservation(home.path(), "chat:first");
1213 assert_eq!(
1214 first.sequence,
1215 reservation(home.path(), "chat:first").sequence
1216 );
1217 let second = reservation(home.path(), "chat:second");
1218 assert!(second.sequence > first.sequence);
1219 let encoded = serde_json::to_vec(&second).unwrap();
1220 assert_eq!(
1221 serde_json::from_slice::<Job>(&encoded).unwrap().sequence,
1222 second.sequence
1223 );
1224 let mut legacy = serde_json::to_value(&first).unwrap();
1225 legacy.as_object_mut().unwrap().remove("sequence");
1226 assert_eq!(serde_json::from_value::<Job>(legacy).unwrap().sequence, 0);
1227 std::fs::write(root(home.path()).join("sequence.json"), b"broken").unwrap();
1228 assert!(
1229 reserve(
1230 home.path(),
1231 &generation(),
1232 "chat",
1233 "chat:third",
1234 serde_json::Value::Null
1235 )
1236 .is_err()
1237 );
1238 }
1239
1240 #[tokio::test]
1241 async fn pre_reservation_failure_is_proven_without_history_absence_guesses() {
1242 let home = tempfile::tempdir().unwrap();
1243 let blocked = home.path().join("not-a-directory");
1244 std::fs::write(&blocked, b"file").unwrap();
1245 let error = spawn(&blocked, "chat", "fresh:chat", serde_json::Value::Null)
1246 .await
1247 .unwrap_err();
1248 assert!(not_started(&error));
1249 assert!(not_started(&error.context("publish recorded turn")));
1250 assert!(!not_started(&anyhow::anyhow!(
1251 "startup acknowledgement timed out"
1252 )));
1253 assert!(!blocked.join("workers").exists());
1254 }
1255
1256 #[test]
1257 fn failed_os_spawn_is_fenced_and_proven_not_started() {
1258 let home = tempfile::tempdir().unwrap();
1259 let executable = home.path().join("invalid-executable");
1260 std::fs::write(&executable, b"not an executable binary").unwrap();
1261 let output = Command::new(std::env::current_exe().unwrap())
1262 .args([
1263 "--exact",
1264 "worker::tests::unstarted_spawn_helper",
1265 "--ignored",
1266 "--nocapture",
1267 ])
1268 .env("MAGI_WORKER_EXE", &executable)
1269 .env("MAGI_SPAWN_FAILURE_HOME", home.path())
1270 .output()
1271 .unwrap();
1272 assert!(
1273 output.status.success(),
1274 "{}\n{}",
1275 String::from_utf8_lossy(&output.stdout),
1276 String::from_utf8_lossy(&output.stderr)
1277 );
1278 }
1279
1280 #[tokio::test]
1281 #[ignore = "isolated environment for the failed OS spawn test"]
1282 async fn unstarted_spawn_helper() {
1283 let home = PathBuf::from(std::env::var_os("MAGI_SPAWN_FAILURE_HOME").unwrap());
1284 let error = spawn(&home, "chat", "fresh:failed-exec", serde_json::Value::Null)
1285 .await
1286 .unwrap_err();
1287 assert!(not_started(&error));
1288 let entry = std::fs::read_dir(root(&home).join("jobs"))
1289 .unwrap()
1290 .next()
1291 .unwrap()
1292 .unwrap();
1293 let id = entry.file_name().to_str().unwrap().to_owned();
1294 let job = get(&home, &id).unwrap();
1295 assert_eq!(job.status, Status::Failed);
1296 assert_eq!(job.pid, None);
1297 assert!(active(&home).unwrap().is_empty());
1298 assert_eq!(get(&home, &id).unwrap().status, Status::Failed);
1299 }
1300
1301 #[test]
1302 fn unresolved_startup_remains_reserved_and_live_lease_never_expires() {
1303 let home = tempfile::tempdir().unwrap();
1304 let mut job = reservation(home.path(), "task:one");
1305 assert_eq!(active(home.path()).unwrap().len(), 1);
1306 job.created_at = 0;
1307 save(
1308 &job_dir(home.path(), &job.id).unwrap().join("job.json"),
1309 &job,
1310 )
1311 .unwrap();
1312 let lease = try_lock(&job_dir(home.path(), &job.id).unwrap().join("live.lock"))
1313 .unwrap()
1314 .unwrap();
1315 assert_eq!(active(home.path()).unwrap().len(), 1);
1316 drop(lease);
1317 assert!(active(home.path()).unwrap().is_empty());
1318 assert_eq!(get(home.path(), &job.id).unwrap().status, Status::Failed);
1319 }
1320
1321 #[tokio::test]
1322 async fn cancelled_startup_cannot_execute_late() {
1323 let home = tempfile::tempdir().unwrap();
1324 let mut job = reservation(home.path(), "task:one");
1325 job.created_at = 0;
1326 save(
1327 &job_dir(home.path(), &job.id).unwrap().join("job.json"),
1328 &job,
1329 )
1330 .unwrap();
1331 assert!(active(home.path()).unwrap().is_empty());
1332 assert!(
1333 run(home.path(), &job.id)
1334 .await
1335 .unwrap_err()
1336 .to_string()
1337 .contains("no longer startable")
1338 );
1339 }
1340
1341 #[test]
1342 fn running_without_lease_fails_regardless_of_pid() {
1343 let home = tempfile::tempdir().unwrap();
1344 let mut job = reservation(home.path(), "task:one");
1345 job.status = Status::Running;
1346 job.pid = Some(std::process::id());
1347 save(
1348 &job_dir(home.path(), &job.id).unwrap().join("job.json"),
1349 &job,
1350 )
1351 .unwrap();
1352 assert!(active(home.path()).unwrap().is_empty());
1353 assert_eq!(get(home.path(), &job.id).unwrap().status, Status::Failed);
1354 }
1355
1356 #[test]
1357 fn controls_survive_reopen_and_reset_at_new_invocation() {
1358 let home = tempfile::tempdir().unwrap();
1359 request(home.path(), "task:one", Control::Park).unwrap();
1360 assert_eq!(
1361 control(home.path(), "task:one").unwrap(),
1362 Some(Control::Park)
1363 );
1364 reservation(home.path(), "task:one");
1365 assert_eq!(control(home.path(), "task:one").unwrap(), None);
1366 request(home.path(), "task:one", Control::Stop).unwrap();
1367 assert_eq!(
1368 control(home.path(), "task:one").unwrap(),
1369 Some(Control::Stop)
1370 );
1371 assert_eq!(control(home.path(), "task:other").unwrap(), None);
1372 }
1373
1374 #[test]
1375 fn singleton_locks_release_without_pid_cleanup() {
1376 let home = tempfile::tempdir().unwrap();
1377 let lease = try_lease(home.path(), "scheduler").unwrap().unwrap();
1378 assert!(try_lease(home.path(), "scheduler").unwrap().is_none());
1379 assert!(try_lease(home.path(), "other").unwrap().is_some());
1380 drop(lease);
1381 assert!(try_lease(home.path(), "scheduler").unwrap().is_some());
1382 }
1383
1384 #[test]
1385 fn immutable_generation_keeps_old_content_after_source_replacement() {
1386 let home = tempfile::tempdir().unwrap();
1387 let source = home.path().join("source");
1388 std::fs::write(&source, b"old executable").unwrap();
1389 let old = pin_source(home.path(), &source).unwrap();
1390 std::fs::write(&source, b"new executable").unwrap();
1391 let new = pin_source(home.path(), &source).unwrap();
1392 assert_ne!(old.id, new.id);
1393 assert_eq!(
1394 std::fs::read(executable(home.path(), &old.id).unwrap()).unwrap(),
1395 b"old executable"
1396 );
1397 assert_eq!(std::fs::read(new.executable).unwrap(), b"new executable");
1398 }
1399
1400 #[test]
1401 fn acknowledgement_records_own_version_and_handler_failure() {
1402 let home = tempfile::tempdir().unwrap();
1403 let generation = pin_source(home.path(), &std::env::current_exe().unwrap()).unwrap();
1404 let (job, _) = reserve(
1405 home.path(),
1406 &generation,
1407 "unknown-kind",
1408 "unknown:one",
1409 serde_json::Value::Null,
1410 )
1411 .unwrap();
1412 let invoke = || {
1413 let output = Command::new(&generation.executable)
1414 .args([
1415 "--exact",
1416 "worker::tests::pinned_entrypoint_helper",
1417 "--ignored",
1418 "--nocapture",
1419 ])
1420 .env("MAGI_WORKER_TEST_HOME", home.path())
1421 .env("MAGI_WORKER_TEST_ID", &job.id)
1422 .output()
1423 .unwrap();
1424 assert!(
1425 output.status.success(),
1426 "{}",
1427 String::from_utf8_lossy(&output.stderr)
1428 );
1429 };
1430 invoke();
1431 let failed = get(home.path(), &job.id).unwrap();
1432 assert_eq!(failed.status, Status::Failed);
1433 assert_eq!(failed.version.as_deref(), Some(env!("CARGO_PKG_VERSION")));
1434 assert!(failed.result.unwrap().contains("unknown worker kind"));
1435 assert!(active(home.path()).unwrap().is_empty());
1436 assert!(!job_dir(home.path(), &job.id).unwrap().exists());
1437 invoke();
1438 assert!(!job_dir(home.path(), &job.id).unwrap().exists());
1439 }
1440
1441 #[tokio::test]
1442 #[ignore = "subprocess entrypoint for the immutable-generation startup test"]
1443 async fn pinned_entrypoint_helper() {
1444 let home = PathBuf::from(std::env::var_os("MAGI_WORKER_TEST_HOME").unwrap());
1445 let id = std::env::var("MAGI_WORKER_TEST_ID").unwrap();
1446 assert!(run(&home, &id).await.is_err());
1447 }
1448
1449 #[test]
1450 fn unfinished_staging_is_reclaimed_without_removing_generations() {
1451 let home = tempfile::tempdir().unwrap();
1452 let old = root(home.path()).join("generations").join(".new-abandoned");
1453 std::fs::create_dir_all(&old).unwrap();
1454 let partial = old.join("partial");
1455 std::fs::write(&partial, b"interrupted copy").unwrap();
1456 #[cfg(windows)]
1457 {
1458 let mut permissions = std::fs::metadata(&partial).unwrap().permissions();
1459 permissions.set_readonly(true);
1460 std::fs::set_permissions(&partial, permissions).unwrap();
1461 }
1462 let source = home.path().join("source");
1463 std::fs::write(&source, b"complete executable").unwrap();
1464 let generation = pin_source(home.path(), &source).unwrap();
1465 assert!(!old.exists());
1466 assert!(generation.executable.is_file());
1467 }
1468
1469 #[tokio::test]
1470 async fn unrelated_executable_cannot_acknowledge_or_consume_reservation() {
1471 let home = tempfile::tempdir().unwrap();
1472 let source = home.path().join("other-binary");
1473 std::fs::write(&source, b"different executable of the same Cargo version").unwrap();
1474 let generation = pin_source(home.path(), &source).unwrap();
1475 let (job, _) = reserve(
1476 home.path(),
1477 &generation,
1478 "task",
1479 "task:one",
1480 serde_json::Value::Null,
1481 )
1482 .unwrap();
1483 assert!(
1484 run(home.path(), &job.id)
1485 .await
1486 .unwrap_err()
1487 .to_string()
1488 .contains("immutable generation")
1489 );
1490 assert_eq!(get(home.path(), &job.id).unwrap().status, Status::Starting);
1491 assert_eq!(active(home.path()).unwrap().len(), 1);
1492 }
1493
1494 #[test]
1495 fn abandoned_record_staging_is_bounded_and_never_executes() {
1496 let home = tempfile::tempdir().unwrap();
1497 let staging = root(home.path()).join("jobs").join(".new-interrupted");
1498 std::fs::create_dir_all(&staging).unwrap();
1499 std::fs::write(staging.join("job.json"), b"partial json").unwrap();
1500 assert!(active(home.path()).unwrap().is_empty());
1501 assert!(!staging.exists());
1502 }
1503
1504 #[cfg(unix)]
1505 #[test]
1506 fn detached_startup_preserves_stdout_and_stderr_in_record_log() {
1507 use std::os::unix::fs::PermissionsExt;
1508 let home = tempfile::tempdir().unwrap();
1509 let source = home.path().join("startup-diagnostic");
1510 std::fs::write(
1511 &source,
1512 b"#!/bin/sh\nprintf 'startup stdout\\n'\nprintf 'startup stderr\\n' >&2\nexit 7\n",
1513 )
1514 .unwrap();
1515 std::fs::set_permissions(&source, std::fs::Permissions::from_mode(0o755)).unwrap();
1516 let generation = pin_source(home.path(), &source).unwrap();
1517 let (mut job, _) = reserve(
1518 home.path(),
1519 &generation,
1520 "task",
1521 "task:log",
1522 serde_json::Value::Null,
1523 )
1524 .unwrap();
1525 let mut command = detached_command(home.path(), &job, &generation).unwrap();
1526 for key in ["MAGI_RUN", "MAGI_NODE", "MAGI_SEAT"]
1527 .into_iter()
1528 .chain(crate::chat_control::bridge::ENVIRONMENT.iter().copied())
1529 {
1530 assert!(
1531 command
1532 .get_envs()
1533 .any(|(name, value)| name == key && value.is_none())
1534 );
1535 }
1536 assert_eq!(command.spawn().unwrap().wait().unwrap().code(), Some(7));
1537 let log = job_dir(home.path(), &job.id).unwrap().join("worker.log");
1538 let output = std::fs::read_to_string(log).unwrap();
1539 assert!(output.contains("startup stdout"));
1540 assert!(output.contains("startup stderr"));
1541 terminal(
1542 home.path(),
1543 &mut job,
1544 Status::Failed,
1545 "startup failed".into(),
1546 )
1547 .unwrap();
1548 assert!(active(home.path()).unwrap().is_empty());
1549 assert!(
1550 root(home.path())
1551 .join("results")
1552 .join(&job.id)
1553 .join("worker.log")
1554 .is_file()
1555 );
1556 }
1557
1558 #[cfg(windows)]
1559 #[test]
1560 fn record_publication_survives_readers_and_short_delete_sharing_handoffs() {
1561 use std::os::windows::fs::OpenOptionsExt as _;
1562
1563 let home = tempfile::tempdir().unwrap();
1564 let mut job = reservation(home.path(), "task:metadata-handoff");
1565 let directory = job_dir(home.path(), &job.id).unwrap();
1566 let path = directory.join("job.json");
1567 let lease = try_lock(&directory.join("live.lock")).unwrap().unwrap();
1568 let snapshot = File::open(&path).unwrap();
1569 let reader = OpenOptions::new()
1571 .read(true)
1572 .share_mode(1 | 2) .open(&path)
1574 .unwrap();
1575 std::thread::scope(|scope| {
1576 scope.spawn(move || {
1577 std::thread::sleep(Duration::from_millis(15));
1578 drop(reader);
1579 });
1580 job.status = Status::Running;
1581 save(&path, &job).unwrap();
1582 });
1583 let original: Job = serde_json::from_reader(snapshot).unwrap();
1584 assert_eq!(original.status, Status::Starting);
1585 assert_eq!(get(home.path(), &job.id).unwrap().status, Status::Running);
1586 let running_snapshot = File::open(&path).unwrap();
1587 terminal(home.path(), &mut job, Status::Finished, "completed".into()).unwrap();
1588 assert_eq!(get(home.path(), &job.id).unwrap().status, Status::Finished);
1589 let previous: Job = serde_json::from_reader(running_snapshot).unwrap();
1590 assert_eq!(previous.status, Status::Running);
1591 drop(lease);
1592 assert!(active(home.path()).unwrap().is_empty());
1593 }
1594
1595 #[cfg(windows)]
1596 #[test]
1597 fn denied_record_publication_preserves_reservation_and_original_os_error() {
1598 use std::os::windows::fs::OpenOptionsExt as _;
1599
1600 let home = tempfile::tempdir().unwrap();
1601 let job = reservation(home.path(), "task:metadata-denied");
1602 let directory = job_dir(home.path(), &job.id).unwrap();
1603 let path = directory.join("job.json");
1604 let reader = OpenOptions::new()
1605 .read(true)
1606 .share_mode(1 | 2) .open(&path)
1608 .unwrap();
1609 let mut acknowledged = job.clone();
1610 acknowledged.status = Status::Running;
1611 let error = save(&path, &acknowledged).unwrap_err();
1612 assert!(format!("{error:#}").contains("publish worker record"));
1613 let io = error.downcast_ref::<std::io::Error>().unwrap();
1614 assert!(matches!(io.raw_os_error(), Some(5 | 32)));
1615 assert_eq!(get(home.path(), &job.id).unwrap().status, Status::Starting);
1616 assert_eq!(reservation(home.path(), &job.key).id, job.id);
1617 assert!(std::fs::read_dir(&directory).unwrap().all(|entry| {
1618 !entry
1619 .unwrap()
1620 .file_name()
1621 .to_string_lossy()
1622 .starts_with(".pending-")
1623 }));
1624 drop(reader);
1625 save(&path, &acknowledged).unwrap();
1626 assert_eq!(get(home.path(), &job.id).unwrap().status, Status::Running);
1627 }
1628
1629 #[test]
1630 fn terminal_result_survives_open_execution_and_output_handles() {
1631 let home = tempfile::tempdir().unwrap();
1632 let mut job = reservation(home.path(), "task:open-handles");
1633 let directory = job_dir(home.path(), &job.id).unwrap();
1634 let lease = try_lock(&directory.join("live.lock")).unwrap().unwrap();
1635 let mut log = OpenOptions::new()
1636 .create(true)
1637 .append(true)
1638 .open(directory.join("worker.log"))
1639 .unwrap();
1640 writeln!(log, "before completion").unwrap();
1641 terminal(home.path(), &mut job, Status::Finished, "completed".into()).unwrap();
1642 assert_eq!(get(home.path(), &job.id).unwrap().status, Status::Finished);
1643 assert!(active(home.path()).unwrap().is_empty());
1644 assert!(
1645 directory.exists(),
1646 "a held execution lease must prevent retirement"
1647 );
1648 drop(lease);
1649 assert!(active(home.path()).unwrap().is_empty());
1652 assert_eq!(get(home.path(), &job.id).unwrap().status, Status::Finished);
1653 writeln!(log, "after completion").unwrap();
1654 drop(log);
1655 assert!(active(home.path()).unwrap().is_empty());
1656 assert!(
1657 root(home.path())
1658 .join("results")
1659 .join(&job.id)
1660 .join("worker.log")
1661 .is_file()
1662 );
1663 assert!(!directory.exists());
1664 }
1665
1666 #[test]
1667 fn terminal_records_are_bounded_without_pruning_live_reservations() {
1668 let home = tempfile::tempdir().unwrap();
1669 let live = reservation(home.path(), "task:live");
1670 let mut finished = reservation(home.path(), "task:finished");
1674 terminal(home.path(), &mut finished, Status::Finished, "done".into()).unwrap();
1675 let template_dir = job_dir(home.path(), &finished.id).unwrap();
1676 std::fs::remove_dir_all(template_dir).unwrap();
1677 for i in 0..RETAIN_FINISHED + 3 {
1678 let mut job = finished.clone();
1679 job.id = format!("{i:032x}");
1680 job.key = format!("task:{i}");
1681 job.sequence += i as u64;
1682 let directory = job_dir(home.path(), &job.id).unwrap();
1683 std::fs::create_dir_all(&directory).unwrap();
1684 std::fs::write(
1685 directory.join("job.json"),
1686 serde_json::to_vec(&job).unwrap(),
1687 )
1688 .unwrap();
1689 }
1690 assert_eq!(
1691 std::fs::read_dir(root(home.path()).join("jobs"))
1692 .unwrap()
1693 .count(),
1694 RETAIN_FINISHED + 4
1695 );
1696 let jobs = records(home.path()).unwrap();
1697 assert_eq!(jobs.len(), 1);
1698 assert_eq!(
1699 std::fs::read_dir(root(home.path()).join("results"))
1700 .unwrap()
1701 .count(),
1702 RETAIN_FINISHED
1703 );
1704 assert!(jobs.iter().any(|job| job.id == live.id));
1705 }
1706}