1use std::fs::{File, OpenOptions};
12use std::io::{Read, Write};
13use std::path::{Path, PathBuf};
14use std::process::{Child, Command, Stdio};
15use std::sync::{LazyLock, Mutex, OnceLock};
16use std::time::{Duration, SystemTime, UNIX_EPOCH};
17
18use anyhow::{Context, Result, bail};
19use serde::{Deserialize, Serialize};
20use sha2::{Digest, Sha256};
21
22const STARTUP_SECONDS: u64 = 30;
23const RETAIN_FINISHED: usize = 256;
24
25pub const INVOCATION_ENV: [&str; 4] = [
30 "MAGI_WORKER_ID",
31 "MAGI_WORKER_GENERATION",
32 "MAGI_WORKER_HOME",
33 "MAGI_WORKER_EXE",
34];
35
36#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
38#[serde(rename_all = "snake_case")]
39pub enum Status {
40 Starting,
42 Running,
44 Finished,
46 Failed,
48}
49
50#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
52#[serde(rename_all = "snake_case")]
53pub enum Control {
54 Park,
56 Stop,
58}
59
60#[derive(Debug)]
61struct NotStarted(anyhow::Error);
62
63impl std::fmt::Display for NotStarted {
64 fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
65 write!(formatter, "{:#}", self.0)
66 }
67}
68
69impl std::error::Error for NotStarted {
70 fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
71 Some(self.0.as_ref())
72 }
73}
74
75fn not_started_error(error: impl Into<anyhow::Error>) -> anyhow::Error {
76 anyhow::Error::new(NotStarted(error.into()))
77}
78
79pub fn not_started(error: &anyhow::Error) -> bool {
85 error.downcast_ref::<NotStarted>().is_some()
86}
87
88#[derive(Debug, thiserror::Error)]
90#[error(
91 "worker {id} startup is unacknowledged; its reservation remains occupied; inspect or retry the same operation key"
92)]
93pub(crate) struct StartupUnacknowledged {
94 pub(crate) id: String,
95}
96
97#[derive(Clone, Debug, Serialize, Deserialize)]
99pub struct Job {
100 pub id: String,
102 #[serde(default)]
104 pub sequence: u64,
105 pub kind: String,
107 pub key: String,
109 pub payload: serde_json::Value,
111 pub generation: String,
113 #[serde(default)]
115 pub version: Option<String>,
116 pub status: Status,
118 pub result: Option<String>,
120 pub pid: Option<u32>,
122 pub created_at: u64,
124 pub finished_at: Option<u64>,
126}
127
128#[derive(Clone, Debug)]
130pub struct Generation {
131 pub id: String,
133 pub executable: PathBuf,
135}
136
137#[derive(Debug)]
139pub struct Lease {
140 _file: File,
141}
142
143impl Drop for Lease {
144 fn drop(&mut self) {
145 #[cfg(unix)]
148 {
149 use std::os::fd::AsRawFd;
150 unsafe { libc::flock(self._file.as_raw_fd(), libc::LOCK_UN) };
151 }
152 }
153}
154
155fn root(home: &Path) -> PathBuf {
156 home.join("workers")
157}
158fn now() -> u64 {
159 SystemTime::now()
160 .duration_since(UNIX_EPOCH)
161 .unwrap_or_default()
162 .as_secs()
163}
164fn nonce() -> String {
165 crate::rng::SplitMix64::new(crate::rng::entropy()).uuid_v4()
166}
167fn valid_id(id: &str) -> bool {
168 !id.is_empty() && id.bytes().all(|b| b.is_ascii_hexdigit() || b == b'-')
169}
170fn job_dir(home: &Path, id: &str) -> Result<PathBuf> {
171 if !valid_id(id) {
172 bail!("invalid worker id");
173 }
174 Ok(root(home).join("jobs").join(id))
175}
176
177fn open_lock(path: &Path) -> Result<File> {
178 if let Some(parent) = path.parent() {
179 std::fs::create_dir_all(parent)
180 .with_context(|| format!("create worker lock directory {}", parent.display()))?;
181 }
182 OpenOptions::new()
183 .create(true)
184 .truncate(false)
185 .read(true)
186 .write(true)
187 .open(path)
188 .with_context(|| format!("open worker lock {}", path.display()))
189}
190
191fn try_lock(path: &Path) -> Result<Option<Lease>> {
192 let file = open_lock(path)?;
193 match crate::pr_head::try_lock_file(&file) {
194 Ok(()) => Ok(Some(Lease { _file: file })),
195 Err(e)
196 if e.kind() == std::io::ErrorKind::WouldBlock
197 || matches!(e.raw_os_error(), Some(33)) =>
198 {
199 Ok(None)
200 }
201 Err(e) => Err(e).with_context(|| format!("acquire worker lock {}", path.display())),
202 }
203}
204
205#[cfg(test)]
206pub(crate) fn execution_lease_for_test(path: &Path) -> Lease {
207 try_lock(path)
208 .expect("execution lease")
209 .expect("free execution lease")
210}
211
212pub fn try_lease(home: &Path, name: &str) -> Result<Option<Lease>> {
215 try_lock(
216 &root(home)
217 .join("leases")
218 .join(format!("{}.lock", digest(name.as_bytes()))),
219 )
220}
221
222fn store_lock(home: &Path) -> Result<Lease> {
223 let deadline = std::time::Instant::now() + Duration::from_secs(10);
224 loop {
225 if let Some(lease) = try_lock(&root(home).join("store.lock"))? {
226 return Ok(lease);
227 }
228 if std::time::Instant::now() >= deadline {
229 bail!("worker store is busy; retry later");
230 }
231 std::thread::sleep(Duration::from_millis(5));
232 }
233}
234
235#[cfg(unix)]
236fn sync_dir(path: &Path) -> Result<()> {
237 File::open(path)
238 .with_context(|| format!("open worker directory for sync {}", path.display()))?
239 .sync_all()
240 .with_context(|| format!("sync worker directory {}", path.display()))?;
241 Ok(())
242}
243
244#[cfg(not(unix))]
245fn sync_dir(_path: &Path) -> Result<()> {
246 Ok(())
247}
248
249fn save(path: &Path, value: &impl Serialize) -> Result<()> {
250 let parent = path.parent().context("worker record has no parent")?;
251 std::fs::create_dir_all(parent)
252 .with_context(|| format!("create worker record directory {}", parent.display()))?;
253 let temp = parent.join(format!(".pending-{}", nonce()));
254 let result = (|| -> Result<()> {
255 let mut file = OpenOptions::new()
256 .create_new(true)
257 .write(true)
258 .open(&temp)
259 .with_context(|| format!("create staged worker record {}", temp.display()))?;
260 serde_json::to_writer(&mut file, value)
261 .with_context(|| format!("serialize worker record {}", path.display()))?;
262 file.flush()
263 .with_context(|| format!("flush staged worker record {}", temp.display()))?;
264 file.sync_all()
265 .with_context(|| format!("sync staged worker record {}", temp.display()))?;
266 drop(file);
269 replace(&temp, path).with_context(|| {
270 format!(
271 "publish worker record {} -> {}",
272 temp.display(),
273 path.display()
274 )
275 })?;
276 sync_dir(parent)
277 })();
278 if result.is_err() {
279 let _ = std::fs::remove_file(&temp);
280 }
281 result
282}
283
284#[cfg(not(windows))]
285fn replace(from: &Path, to: &Path) -> Result<()> {
286 Ok(std::fs::rename(from, to)?)
287}
288
289#[cfg(windows)]
290fn replace(from: &Path, to: &Path) -> Result<()> {
291 crate::lock_file::Retry::new(None).run(|| std::fs::rename(from, to))?;
296 Ok(())
297}
298
299fn open_record(path: &Path) -> std::io::Result<File> {
302 #[cfg(windows)]
303 {
304 crate::lock_file::Retry::new(None).run(|| File::open(path))
305 }
306 #[cfg(not(windows))]
307 {
308 File::open(path)
309 }
310}
311
312pub fn get(home: &Path, id: &str) -> Result<Job> {
314 let path = job_dir(home, id)?.join("job.json");
315 let file = match open_record(&path) {
316 Ok(file) => file,
317 Err(error) if error.kind() == std::io::ErrorKind::NotFound => {
318 let retained = root(home).join("results").join(id).join("job.json");
319 open_record(&retained)
320 .with_context(|| format!("open retained worker record {}", retained.display()))?
321 }
322 Err(error) => {
323 return Err(error).with_context(|| format!("open worker record {}", path.display()));
324 }
325 };
326 serde_json::from_reader(file).with_context(|| format!("decode worker record {id}"))
327}
328
329pub fn current_job(home: &Path) -> Result<Option<Job>> {
333 match std::env::var("MAGI_WORKER_ID") {
334 Ok(id) => Ok(Some(get(home, &id)?)),
335 Err(std::env::VarError::NotPresent) => Ok(None),
336 Err(error) => Err(error.into()),
337 }
338}
339
340fn next_sequence(home: &Path) -> Result<u64> {
341 let path = root(home).join("sequence.json");
342 let previous: u64 = match open_record(&path) {
343 Ok(file) => serde_json::from_reader(file)
344 .with_context(|| format!("decode worker sequence {}", path.display()))?,
345 Err(error) if error.kind() == std::io::ErrorKind::NotFound => 0,
346 Err(error) => {
347 return Err(error).with_context(|| format!("open worker sequence {}", path.display()));
348 }
349 };
350 let sequence = previous
351 .checked_add(1)
352 .context("worker reservation sequence exhausted")?;
353 save(&path, &sequence)?;
354 Ok(sequence)
355}
356
357fn records(home: &Path) -> Result<Vec<Job>> {
358 let directory = root(home).join("jobs");
359 std::fs::create_dir_all(&directory)?;
360 let mut jobs = Vec::new();
361 let mut retired = false;
362 for entry in std::fs::read_dir(directory)? {
363 let entry = entry?;
364 if !entry.file_type()?.is_dir() {
365 continue;
366 }
367 let name = entry.file_name();
368 let Some(id) = name.to_str() else {
369 continue;
370 };
371 if id.starts_with(".new-") {
372 std::fs::remove_dir_all(entry.path())?;
375 continue;
376 }
377 if id.starts_with('.') {
378 continue;
379 }
380 for pending in std::fs::read_dir(entry.path())? {
381 let pending = pending?;
382 if pending.file_type()?.is_file()
383 && pending
384 .file_name()
385 .to_string_lossy()
386 .starts_with(".pending-")
387 {
388 std::fs::remove_file(pending.path())?;
389 }
390 }
391 let job = get(home, id).with_context(|| format!("read worker {id}"))?;
392 if matches!(job.status, Status::Starting | Status::Running) {
393 jobs.push(job);
394 } else {
395 retired |= retire(home, &job);
396 }
397 }
398 if retired {
399 if let Err(error) = prune_results(home) {
400 tracing::warn!(%error, "worker result retention deferred");
401 }
402 }
403 Ok(jobs)
404}
405
406fn terminal(home: &Path, job: &mut Job, status: Status, result: String) -> Result<()> {
407 job.status = status;
408 job.result = Some(result);
409 job.finished_at = Some(now());
410 save(&job_dir(home, &job.id)?.join("job.json"), job)
411 .with_context(|| format!("publish worker {} terminal status {status:?}", job.id))?;
412 clear_control_locked(home, &job.key)
416}
417
418fn retire(home: &Path, job: &Job) -> bool {
419 let attempt = || -> Result<bool> {
420 let dir = job_dir(home, &job.id)?;
421 let Some(lease) = try_lock(&dir.join("live.lock"))? else {
422 return Ok(false);
423 };
424 drop(lease);
425 archive(home, job)?;
426 Ok(true)
427 };
428 match attempt() {
433 Ok(retired) => retired,
434 Err(error) => {
435 tracing::warn!(
436 worker = %job.id,
437 error = %format_args!("{error:#}"),
438 "worker result retirement deferred"
439 );
440 false
441 }
442 }
443}
444
445fn archive(home: &Path, job: &Job) -> Result<()> {
446 let results = root(home).join("results");
447 std::fs::create_dir_all(&results)
448 .with_context(|| format!("create worker results directory {}", results.display()))?;
449 let from = job_dir(home, &job.id)?;
450 let to = results.join(&job.id);
451 std::fs::rename(&from, &to).with_context(|| {
452 format!(
453 "archive worker result {} -> {}",
454 from.display(),
455 to.display()
456 )
457 })?;
458 sync_dir(&root(home).join("jobs"))?;
459 sync_dir(&results)
460}
461
462fn reconcile(home: &Path, jobs: &mut [Job]) -> Result<()> {
465 let mut retired = false;
466 for job in jobs {
467 if !matches!(job.status, Status::Starting | Status::Running) {
468 continue;
469 }
470 let lease = try_lock(&job_dir(home, &job.id)?.join("live.lock"))?;
471 if lease.is_none() {
472 continue;
473 }
474 if job.status == Status::Starting && now().saturating_sub(job.created_at) < STARTUP_SECONDS
475 {
476 continue;
477 }
478 let reason = if job.status == Status::Starting {
479 "worker startup reservation expired before acknowledgement"
480 } else {
481 "worker exited without recording a result"
482 };
483 terminal(home, job, Status::Failed, reason.to_owned())?;
484 drop(lease);
485 retired |= retire(home, job);
486 }
487 if retired {
488 if let Err(error) = prune_results(home) {
489 tracing::warn!(%error, "worker result retention deferred");
490 }
491 }
492 Ok(())
493}
494
495pub fn active(home: &Path) -> Result<Vec<Job>> {
498 let _store = store_lock(home)?;
499 let mut jobs = records(home)?;
500 reconcile(home, &mut jobs)?;
501 Ok(jobs
502 .into_iter()
503 .filter(|job| matches!(job.status, Status::Starting | Status::Running))
504 .collect())
505}
506
507pub fn observe_active(home: &Path) -> Result<Vec<Job>> {
512 let entries = match std::fs::read_dir(root(home).join("jobs")) {
513 Ok(entries) => entries,
514 Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(Vec::new()),
515 Err(error) => return Err(error.into()),
516 };
517 let mut active = Vec::new();
518 for entry in entries {
519 let entry = entry?;
520 if !entry.file_type()?.is_dir() {
521 continue;
522 }
523 let name = entry.file_name();
524 let id = name.to_str().context("invalid worker reservation name")?;
525 if id.starts_with('.') {
526 continue;
527 }
528 let job = get(home, id)?;
529 if observe_job_active(home, &job)? {
530 active.push(job);
531 }
532 }
533 Ok(active)
534}
535
536pub(crate) fn observe_job_active(home: &Path, job: &Job) -> Result<bool> {
538 match job.status {
539 Status::Starting => Ok(true),
540 Status::Running => {
541 let file = match File::open(job_dir(home, &job.id)?.join("live.lock")) {
542 Ok(file) => file,
543 Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(true),
544 Err(error) => return Err(error.into()),
545 };
546 match crate::pr_head::try_lock_file(&file) {
547 Ok(()) => {
548 drop(Lease { _file: file });
549 Ok(false)
550 }
551 Err(error)
552 if error.kind() == std::io::ErrorKind::WouldBlock
553 || cfg!(windows) && error.raw_os_error() == Some(33) =>
554 {
555 Ok(true)
556 }
557 Err(error) => Err(error.into()),
558 }
559 }
560 Status::Finished | Status::Failed => Ok(false),
561 }
562}
563
564fn control_path(home: &Path, key: &str) -> PathBuf {
565 root(home)
566 .join("controls")
567 .join(format!("{}.json", digest(key.as_bytes())))
568}
569
570pub fn request(home: &Path, key: &str, control: Control) -> Result<()> {
572 let _store = store_lock(home)?;
573 save(&control_path(home, key), &control)
574}
575
576pub fn control(home: &Path, key: &str) -> Result<Option<Control>> {
578 let path = control_path(home, key);
579 match open_record(&path) {
580 Ok(file) => serde_json::from_reader(file)
581 .map(Some)
582 .with_context(|| format!("decode worker control {}", path.display())),
583 Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(None),
584 Err(e) => Err(e).with_context(|| format!("open worker control {}", path.display())),
585 }
586}
587
588fn clear_control_locked(home: &Path, key: &str) -> Result<()> {
589 let path = control_path(home, key);
590 match std::fs::remove_file(&path) {
591 Ok(()) => sync_dir(&root(home).join("controls")),
592 Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(()),
593 Err(e) => Err(e).with_context(|| format!("remove worker control {}", path.display())),
594 }
595}
596
597pub fn clear_control(home: &Path, key: &str) -> Result<()> {
599 let _store = store_lock(home)?;
600 clear_control_locked(home, key)
601}
602
603pub fn executable(home: &Path, generation: &str) -> Result<PathBuf> {
605 if generation.len() != 64 || !generation.bytes().all(|b| b.is_ascii_hexdigit()) {
606 bail!("invalid worker generation");
607 }
608 let path = root(home)
609 .join("generations")
610 .join(generation)
611 .join(if cfg!(windows) { "magi.exe" } else { "magi" });
612 if !path.is_file() {
613 bail!("worker generation {generation} is unavailable");
614 }
615 Ok(path)
616}
617
618pub fn helper_executable(home: &Path) -> Result<PathBuf> {
621 if let Some(generation) = std::env::var_os("MAGI_WORKER_GENERATION") {
622 return executable(
623 home,
624 generation
625 .to_str()
626 .context("invalid generation environment")?,
627 );
628 }
629 Ok(pin_current(home)?.executable)
630}
631
632pub fn remember_current_executable() -> Result<()> {
636 if INSTALLED_EXECUTABLE.get().is_some() {
637 return Ok(());
638 }
639 let path = std::env::current_exe()?.canonicalize()?;
640 let _ = INSTALLED_EXECUTABLE.set(path);
641 Ok(())
642}
643
644static INSTALLED_EXECUTABLE: OnceLock<PathBuf> = OnceLock::new();
645
646pub fn pin_current(home: &Path) -> Result<Generation> {
649 let source = match std::env::var_os("MAGI_WORKER_EXE") {
650 Some(path) => PathBuf::from(path),
651 None => match INSTALLED_EXECUTABLE.get() {
652 Some(path) => path.clone(),
653 None => std::env::current_exe()?,
654 },
655 };
656 let source = source.canonicalize().context("resolve worker executable")?;
657 if let Some(directory) = source.parent() {
660 if directory.parent() == Some(root(home).join("generations").as_path()) {
661 if let Some(id) = directory.file_name().and_then(|name| name.to_str()) {
662 if id.len() == 64
663 && id.bytes().all(|b| b.is_ascii_hexdigit())
664 && executable(home, id)?.canonicalize()? == source
665 {
666 return Ok(Generation {
667 id: id.to_owned(),
668 executable: source,
669 });
670 }
671 }
672 }
673 }
674 let stamp = source_stamp(&source)?;
675 let mut cache = PIN_CACHE
676 .lock()
677 .map_err(|_| anyhow::anyhow!("worker generation cache poisoned"))?;
678 if let Some(entry) = cache.iter().find(|entry| {
679 entry.home == home
680 && entry.source == source
681 && entry.stamp == stamp
682 && entry.generation.executable.is_file()
683 }) {
684 return Ok(entry.generation.clone());
685 }
686 let generation = pin_source(home, &source)?;
687 if source_stamp(&source)? != stamp {
688 bail!("worker executable changed while being pinned; retry");
689 }
690 if cache.len() >= 8 {
691 cache.remove(0);
692 }
693 cache.push(CachedGeneration {
694 home: home.to_path_buf(),
695 source,
696 stamp,
697 generation: generation.clone(),
698 });
699 Ok(generation)
700}
701
702type SourceStamp = (u64, Option<SystemTime>, Option<SystemTime>, u64, u64);
703struct CachedGeneration {
704 home: PathBuf,
705 source: PathBuf,
706 stamp: SourceStamp,
707 generation: Generation,
708}
709static PIN_CACHE: LazyLock<Mutex<Vec<CachedGeneration>>> = LazyLock::new(|| Mutex::new(Vec::new()));
710
711fn source_stamp(path: &Path) -> Result<SourceStamp> {
712 let metadata = std::fs::metadata(path)?;
713 #[cfg(unix)]
714 let identity = {
715 use std::os::unix::fs::MetadataExt;
716 (metadata.dev(), metadata.ino())
717 };
718 #[cfg(not(unix))]
719 let identity = (0, 0);
720 Ok((
721 metadata.len(),
722 metadata.modified().ok(),
723 metadata.created().ok(),
724 identity.0,
725 identity.1,
726 ))
727}
728
729fn pin_source(home: &Path, source: &Path) -> Result<Generation> {
730 let generations = root(home).join("generations");
731 std::fs::create_dir_all(&generations)?;
732 let staging = generations.join(format!(".new-{}", nonce()));
733 let mut staging_lease = {
734 let _store = store_lock(home)?;
735 for entry in std::fs::read_dir(&generations)? {
737 let entry = entry?;
738 if entry.file_type()?.is_dir()
739 && entry.file_name().to_string_lossy().starts_with(".new-")
740 {
741 if let Some(lease) = try_lock(&entry.path().join("pin.lock"))? {
742 drop(lease);
743 remove_staging(&entry.path())?;
744 }
745 }
746 }
747 std::fs::create_dir(&staging)?;
748 Some(try_lock(&staging.join("pin.lock"))?.context("generation staging lease unavailable")?)
749 };
750 let temp = staging.join(if cfg!(windows) { "magi.exe" } else { "magi" });
751 let result = (|| -> Result<Generation> {
752 let mut input = File::open(source)?;
753 let mut output = OpenOptions::new()
754 .create_new(true)
755 .write(true)
756 .open(&temp)?;
757 let mut hash = Sha256::new();
758 let mut buffer = [0u8; 65536];
759 loop {
760 let n = input.read(&mut buffer)?;
761 if n == 0 {
762 break;
763 }
764 hash.update(&buffer[..n]);
765 output.write_all(&buffer[..n])?;
766 }
767 let id = hex_digest(hash);
768 let mut permissions = input.metadata()?.permissions();
769 permissions.set_readonly(true);
770 output.set_permissions(permissions)?;
771 output.sync_all()?;
772 drop(output);
773 sync_dir(&staging)?;
774 let target = generations.join(&id);
775 let _store = store_lock(home)?;
777 drop(staging_lease.take());
778 std::fs::remove_file(staging.join("pin.lock"))?;
779 if target.exists() {
780 let existing = target.join(temp.file_name().context("binary filename")?);
781 if hash_file(&existing)? != id {
782 bail!("immutable worker generation is corrupt: {id}");
783 }
784 } else {
785 std::fs::rename(&staging, &target)?;
786 sync_dir(&generations)?;
787 }
788 Ok(Generation {
789 executable: target.join(temp.file_name().context("binary filename")?),
790 id,
791 })
792 })();
793 let _store = store_lock(home)?;
794 drop(staging_lease);
795 if staging.exists() {
796 remove_staging(&staging)?;
797 }
798 result
799}
800
801fn remove_staging(staging: &Path) -> Result<()> {
802 #[cfg(windows)]
803 {
804 use std::os::windows::{ffi::OsStrExt, fs::MetadataExt};
805 #[link(name = "kernel32")]
806 unsafe extern "system" {
807 fn SetFileAttributesW(path: *const u16, attributes: u32) -> i32;
808 }
809 const READONLY: u32 = 0x1;
810 for entry in std::fs::read_dir(staging)? {
811 let entry = entry?;
812 let attributes = entry.metadata()?.file_attributes();
813 if attributes & READONLY != 0 {
814 let path: Vec<u16> = entry.path().as_os_str().encode_wide().chain([0]).collect();
815 if unsafe { SetFileAttributesW(path.as_ptr(), attributes & !READONLY) } == 0 {
818 return Err(std::io::Error::last_os_error())
819 .context("clear staging readonly attribute");
820 }
821 }
822 }
823 }
824 Ok(std::fs::remove_dir_all(staging)?)
825}
826
827fn hash_file(path: &Path) -> Result<String> {
828 let mut file = File::open(path)?;
829 let mut hash = Sha256::new();
830 let mut buffer = [0u8; 65536];
831 loop {
832 let n = file.read(&mut buffer)?;
833 if n == 0 {
834 break;
835 }
836 hash.update(&buffer[..n]);
837 }
838 Ok(hex_digest(hash))
839}
840
841fn reserve(
842 home: &Path,
843 generation: &Generation,
844 kind: &str,
845 key: &str,
846 payload: serde_json::Value,
847) -> Result<(Job, bool)> {
848 if kind.is_empty() || key.is_empty() {
849 bail!("worker kind and operation key must be nonempty");
850 }
851 let _store = store_lock(home)?;
852 let mut jobs = records(home)?;
853 reconcile(home, &mut jobs)?;
854 if let Some(job) = jobs
855 .iter()
856 .find(|job| job.key == key && matches!(job.status, Status::Starting | Status::Running))
857 {
858 if job.kind != kind || job.payload != payload {
859 bail!("worker operation key {key} is already reserved for a different request");
860 }
861 return Ok((job.clone(), false));
862 }
863 let id = nonce();
864 let job = Job {
865 id: id.clone(),
866 sequence: next_sequence(home)?,
867 kind: kind.to_owned(),
868 key: key.to_owned(),
869 payload,
870 generation: generation.id.clone(),
871 version: None,
872 status: Status::Starting,
873 result: None,
874 pid: None,
875 created_at: now(),
876 finished_at: None,
877 };
878 let jobs_dir = root(home).join("jobs");
879 let staging = jobs_dir.join(format!(".new-{id}"));
880 std::fs::create_dir(&staging)
881 .with_context(|| format!("create worker reservation staging {}", staging.display()))?;
882 save(&staging.join("job.json"), &job)?;
883 clear_control_locked(home, key)?;
884 std::fs::rename(&staging, job_dir(home, &id)?)
885 .with_context(|| format!("publish worker {id} reservation from {}", staging.display()))?;
886 sync_dir(&jobs_dir)?;
887 Ok((job, true))
888}
889
890fn prune_results(home: &Path) -> Result<()> {
891 let results = root(home).join("results");
892 std::fs::create_dir_all(&results)?;
893 let mut entries = Vec::new();
894 for entry in std::fs::read_dir(&results)? {
895 let entry = entry?;
896 if entry.file_type()?.is_dir() {
897 entries.push((entry.metadata()?.modified()?, entry.path()));
900 }
901 }
902 entries.sort_unstable_by_key(|(modified, _)| std::cmp::Reverse(*modified));
903 for (_, dir) in entries.into_iter().skip(RETAIN_FINISHED) {
904 if let Some(lease) = try_lock(&dir.join("live.lock"))? {
905 drop(lease);
906 std::fs::remove_dir_all(dir)?;
907 }
908 }
909 sync_dir(&results)
910}
911
912fn detached_command(home: &Path, job: &Job, generation: &Generation) -> Result<Command> {
913 let log_path = job_dir(home, &job.id)?.join("worker.log");
917 let log = OpenOptions::new()
918 .create(true)
919 .append(true)
920 .open(&log_path)
921 .with_context(|| format!("open worker startup log {}", log_path.display()))?;
922 let stderr = log
923 .try_clone()
924 .with_context(|| format!("clone worker startup log {}", log_path.display()))?;
925 let mut command = Command::new(&generation.executable);
926 command
927 .arg("__worker")
928 .arg("--home")
929 .arg(home)
930 .arg(&job.id)
931 .env("MAGI_HOME", home)
932 .env("MAGI_WORKER_HOME", home)
933 .env("MAGI_WORKER_ID", &job.id)
934 .env("MAGI_WORKER_GENERATION", &generation.id)
935 .env("MAGI_WORKER_EXE", &generation.executable)
936 .env_remove("MAGI_RUN")
937 .env_remove("MAGI_NODE")
938 .env_remove("MAGI_SEAT")
939 .stdin(Stdio::null())
940 .stdout(Stdio::from(log))
941 .stderr(Stdio::from(stderr));
942 crate::chat_control::bridge::clear_std_environment(&mut command);
943 let mut paths = vec![
944 generation
945 .executable
946 .parent()
947 .context("generation directory")?
948 .to_path_buf(),
949 ];
950 if let Some(path) = std::env::var_os("PATH") {
951 paths.extend(std::env::split_paths(&path));
952 }
953 command.env("PATH", std::env::join_paths(paths)?);
954 #[cfg(unix)]
955 {
956 use std::os::unix::process::CommandExt;
957 unsafe {
958 command.pre_exec(|| {
959 if libc::setsid() < 0 {
960 return Err(std::io::Error::last_os_error());
961 }
962 libc::signal(libc::SIGINT, libc::SIG_DFL);
965 libc::signal(libc::SIGTERM, libc::SIG_DFL);
966 Ok(())
967 });
968 }
969 }
970 #[cfg(windows)]
971 {
972 use std::os::windows::process::CommandExt;
973 detached_standard_handles().context("exclude caller pipes from detached worker")?;
974 command.creation_flags(
975 crate::chat_watch::windows::detached_flags()
976 .context("prepare independent Windows worker")?,
977 );
978 }
979 Ok(command)
980}
981
982#[cfg(windows)]
983fn detached_standard_handles() -> std::io::Result<()> {
984 use std::ffi::c_void;
985 #[link(name = "kernel32")]
986 unsafe extern "system" {
987 fn GetStdHandle(kind: u32) -> *mut c_void;
988 fn SetHandleInformation(handle: *mut c_void, mask: u32, flags: u32) -> i32;
989 }
990 for kind in [-10_i32, -11, -12] {
996 let handle = unsafe { GetStdHandle(kind as u32) };
997 if handle.is_null() || handle as isize == -1 {
998 continue;
999 }
1000 if unsafe { SetHandleInformation(handle, 1, 0) } == 0 {
1001 let error = std::io::Error::last_os_error();
1002 if error.raw_os_error() != Some(6) {
1004 return Err(error);
1005 }
1006 }
1007 }
1008 Ok(())
1009}
1010
1011async fn spawn_generation(
1015 mut spawn: impl FnMut() -> std::io::Result<Child>,
1016) -> std::io::Result<Child> {
1017 for delay_ms in [10, 20, 40, 80] {
1018 match spawn() {
1019 Err(error) if error.kind() == std::io::ErrorKind::ExecutableFileBusy => {
1020 tokio::time::sleep(Duration::from_millis(delay_ms)).await;
1021 }
1022 result => return result,
1023 }
1024 }
1025 spawn()
1026}
1027
1028pub async fn spawn(home: &Path, kind: &str, key: &str, payload: serde_json::Value) -> Result<Job> {
1033 std::fs::create_dir_all(home).map_err(not_started_error)?;
1034 let home = home
1035 .canonicalize()
1036 .context("resolve worker home")
1037 .map_err(not_started_error)?;
1038 let pin_home = home.clone();
1039 let generation = tokio::task::spawn_blocking(move || pin_current(&pin_home))
1040 .await
1041 .map_err(not_started_error)?
1042 .map_err(not_started_error)?;
1043 let (job, fresh) = reserve(&home, &generation, kind, key, payload)?;
1044 if fresh {
1045 let child = match detached_command(&home, &job, &generation) {
1046 Ok(mut command) => spawn_generation(|| command.spawn())
1047 .await
1048 .map_err(Into::into),
1049 Err(error) => Err(error),
1050 };
1051 match child {
1052 Ok(mut child) => {
1053 std::thread::spawn(move || {
1056 let _ = child.wait();
1057 });
1058 }
1059 Err(error) => {
1060 let _store = store_lock(&home)?;
1061 let mut job = get(&home, &job.id)?;
1062 if job.status == Status::Starting {
1063 if let Some(_lease) = try_lock(&job_dir(&home, &job.id)?.join("live.lock"))? {
1064 terminal(
1065 &home,
1066 &mut job,
1067 Status::Failed,
1068 format!("worker spawn failed: {error}"),
1069 )?;
1070 return Err(not_started_error(error));
1071 }
1072 }
1073 return Err(error);
1074 }
1075 }
1076 }
1077 let deadline = tokio::time::Instant::now() + Duration::from_secs(10);
1078 loop {
1079 let current = get(&home, &job.id)?;
1080 match current.status {
1081 Status::Running | Status::Finished => return Ok(current),
1082 Status::Failed => bail!(
1083 "worker {} failed: {}",
1084 current.id,
1085 current.result.as_deref().unwrap_or("unknown failure")
1086 ),
1087 Status::Starting => {}
1088 }
1089 if tokio::time::Instant::now() >= deadline {
1090 return Err(StartupUnacknowledged { id: job.id }.into());
1091 }
1092 tokio::time::sleep(Duration::from_millis(20)).await;
1093 }
1094}
1095
1096pub async fn run(home: &Path, id: &str) -> Result<()> {
1100 let dir = job_dir(home, id)?;
1101 let (job, _lease) = {
1102 let _store = store_lock(home)?;
1103 let mut job = get(home, id)?;
1104 if job.status != Status::Starting {
1105 bail!("worker reservation is no longer startable");
1106 }
1107 let lease =
1108 try_lock(&dir.join("live.lock"))?.context("worker execution lease is already held")?;
1109 let current = std::env::current_exe()
1110 .context("locate running worker executable")?
1111 .canonicalize()
1112 .context("resolve running worker executable")?;
1113 let expected = executable(home, &job.generation)?
1114 .canonicalize()
1115 .with_context(|| {
1116 format!("resolve worker {id} reserved generation {}", job.generation)
1117 })?;
1118 if current != expected {
1119 bail!("worker must execute its reserved immutable generation");
1120 }
1121 job.pid = Some(std::process::id());
1122 job.version = Some(env!("CARGO_PKG_VERSION").to_owned());
1123 job.status = Status::Running;
1124 save(&dir.join("job.json"), &job)
1125 .with_context(|| format!("acknowledge worker {id} startup"))?;
1126 (job, lease)
1127 };
1128 let result = match job.kind.as_str() {
1129 "task" => crate::daemon::run_task_worker(home, job.payload.clone()).await,
1130 "scheduler" => crate::daemon::run_scheduler_worker(home, job.payload.clone()).await,
1131 "chat" => crate::chat_worker::run_turn_worker(home, job.payload.clone()).await,
1132 "service" => crate::consult::run_service_worker(home, job.payload.clone()).await,
1133 "daemon-service" => crate::daemon::run_service_worker(home, job.payload.clone()).await,
1134 "deputy" => crate::deputy::run_worker(home, job.payload.clone()).await,
1135 kind => Err(anyhow::anyhow!("unknown worker kind {kind}")),
1136 };
1137 let _store = store_lock(home)?;
1138 let mut current = get(home, id)?;
1139 match &result {
1140 Ok(()) => terminal(home, &mut current, Status::Finished, "completed".to_owned())?,
1141 Err(error) => terminal(home, &mut current, Status::Failed, format!("{error:#}"))?,
1142 }
1143 result
1144}
1145
1146fn hex_digest(hash: Sha256) -> String {
1147 hash.finalize()
1150 .iter()
1151 .map(|byte| format!("{byte:02x}"))
1152 .collect()
1153}
1154fn digest(bytes: &[u8]) -> String {
1155 let mut hash = Sha256::new();
1156 hash.update(bytes);
1157 hex_digest(hash)
1158}
1159
1160#[cfg(test)]
1161mod tests {
1162 use super::*;
1163
1164 #[test]
1165 fn observing_an_unused_store_requires_no_writes() {
1166 let home = tempfile::tempdir().unwrap();
1167 assert!(observe_active(home.path()).unwrap().is_empty());
1168 assert!(!home.path().join("workers").exists());
1169 }
1170
1171 #[test]
1172 fn observing_execution_probes_the_lease_without_rewriting_history() {
1173 let home = tempfile::tempdir().unwrap();
1174 let mut job = reservation(home.path(), "task:observer");
1175 let directory = job_dir(home.path(), &job.id).unwrap();
1176 let live_path = directory.join("live.lock");
1177 let lease = try_lock(&live_path).unwrap().unwrap();
1178 job.status = Status::Running;
1179 save(&directory.join("job.json"), &job).unwrap();
1180 #[cfg(unix)]
1181 {
1182 use std::os::unix::fs::PermissionsExt as _;
1183 std::fs::set_permissions(&live_path, std::fs::Permissions::from_mode(0o444)).unwrap();
1184 }
1185 let before = std::fs::read(directory.join("job.json")).unwrap();
1186 let observed = observe_active(home.path()).unwrap();
1187 assert_eq!(observed.len(), 1);
1188 assert_eq!(observed[0].id, job.id);
1189 drop(lease);
1190 assert!(observe_active(home.path()).unwrap().is_empty());
1191 assert_eq!(std::fs::read(directory.join("job.json")).unwrap(), before);
1192 assert_eq!(get(home.path(), &job.id).unwrap().status, Status::Running);
1193 }
1194 fn generation() -> Generation {
1195 Generation {
1196 id: "a".repeat(64),
1197 executable: PathBuf::from("unused"),
1198 }
1199 }
1200 fn reservation(home: &Path, key: &str) -> Job {
1201 reserve(
1202 home,
1203 &generation(),
1204 "task",
1205 key,
1206 serde_json::json!({"task":"example"}),
1207 )
1208 .unwrap()
1209 .0
1210 }
1211
1212 #[test]
1213 fn reservation_deduplicates_identical_keys_and_rejects_collision() {
1214 let home = tempfile::tempdir().unwrap();
1215 let first = reservation(home.path(), "task:one");
1216 assert_eq!(first.id, reservation(home.path(), "task:one").id);
1217 assert!(
1218 reserve(
1219 home.path(),
1220 &generation(),
1221 "chat",
1222 "task:one",
1223 serde_json::Value::Null
1224 )
1225 .is_err()
1226 );
1227 assert_ne!(first.id, reservation(home.path(), "task:two").id);
1228 }
1229
1230 #[test]
1231 fn reservation_order_is_durable_and_corruption_never_resets_it() {
1232 let home = tempfile::tempdir().unwrap();
1233 let first = reservation(home.path(), "chat:first");
1234 assert_eq!(
1235 first.sequence,
1236 reservation(home.path(), "chat:first").sequence
1237 );
1238 let second = reservation(home.path(), "chat:second");
1239 assert!(second.sequence > first.sequence);
1240 let encoded = serde_json::to_vec(&second).unwrap();
1241 assert_eq!(
1242 serde_json::from_slice::<Job>(&encoded).unwrap().sequence,
1243 second.sequence
1244 );
1245 let mut legacy = serde_json::to_value(&first).unwrap();
1246 legacy.as_object_mut().unwrap().remove("sequence");
1247 assert_eq!(serde_json::from_value::<Job>(legacy).unwrap().sequence, 0);
1248 std::fs::write(root(home.path()).join("sequence.json"), b"broken").unwrap();
1249 assert!(
1250 reserve(
1251 home.path(),
1252 &generation(),
1253 "chat",
1254 "chat:third",
1255 serde_json::Value::Null
1256 )
1257 .is_err()
1258 );
1259 }
1260
1261 #[tokio::test]
1262 async fn pre_reservation_failure_is_proven_without_history_absence_guesses() {
1263 let home = tempfile::tempdir().unwrap();
1264 let blocked = home.path().join("not-a-directory");
1265 std::fs::write(&blocked, b"file").unwrap();
1266 let error = spawn(&blocked, "chat", "fresh:chat", serde_json::Value::Null)
1267 .await
1268 .unwrap_err();
1269 assert!(not_started(&error));
1270 assert!(not_started(&error.context("publish recorded turn")));
1271 assert!(!not_started(&anyhow::anyhow!(
1272 "startup acknowledgement timed out"
1273 )));
1274 assert!(!blocked.join("workers").exists());
1275 }
1276
1277 #[test]
1278 fn failed_os_spawn_is_fenced_and_proven_not_started() {
1279 let home = tempfile::tempdir().unwrap();
1280 let executable = home.path().join("invalid-executable");
1281 std::fs::write(&executable, b"not an executable binary").unwrap();
1282 let output = Command::new(std::env::current_exe().unwrap())
1283 .args([
1284 "--exact",
1285 "worker::tests::unstarted_spawn_helper",
1286 "--ignored",
1287 "--nocapture",
1288 ])
1289 .env("MAGI_WORKER_EXE", &executable)
1290 .env("MAGI_SPAWN_FAILURE_HOME", home.path())
1291 .output()
1292 .unwrap();
1293 assert!(
1294 output.status.success(),
1295 "{}\n{}",
1296 String::from_utf8_lossy(&output.stdout),
1297 String::from_utf8_lossy(&output.stderr)
1298 );
1299 }
1300
1301 #[tokio::test]
1302 #[ignore = "isolated environment for the failed OS spawn test"]
1303 async fn unstarted_spawn_helper() {
1304 let home = PathBuf::from(std::env::var_os("MAGI_SPAWN_FAILURE_HOME").unwrap());
1305 let error = spawn(&home, "chat", "fresh:failed-exec", serde_json::Value::Null)
1306 .await
1307 .unwrap_err();
1308 assert!(not_started(&error));
1309 let entry = std::fs::read_dir(root(&home).join("jobs"))
1310 .unwrap()
1311 .next()
1312 .unwrap()
1313 .unwrap();
1314 let id = entry.file_name().to_str().unwrap().to_owned();
1315 let job = get(&home, &id).unwrap();
1316 assert_eq!(job.status, Status::Failed);
1317 assert_eq!(job.pid, None);
1318 assert!(active(&home).unwrap().is_empty());
1319 assert_eq!(get(&home, &id).unwrap().status, Status::Failed);
1320 }
1321
1322 #[test]
1323 fn unresolved_startup_remains_reserved_and_live_lease_never_expires() {
1324 let home = tempfile::tempdir().unwrap();
1325 let mut job = reservation(home.path(), "task:one");
1326 assert_eq!(active(home.path()).unwrap().len(), 1);
1327 job.created_at = 0;
1328 save(
1329 &job_dir(home.path(), &job.id).unwrap().join("job.json"),
1330 &job,
1331 )
1332 .unwrap();
1333 let lease = try_lock(&job_dir(home.path(), &job.id).unwrap().join("live.lock"))
1334 .unwrap()
1335 .unwrap();
1336 assert_eq!(active(home.path()).unwrap().len(), 1);
1337 drop(lease);
1338 assert!(active(home.path()).unwrap().is_empty());
1339 assert_eq!(get(home.path(), &job.id).unwrap().status, Status::Failed);
1340 }
1341
1342 #[tokio::test]
1343 async fn cancelled_startup_cannot_execute_late() {
1344 let home = tempfile::tempdir().unwrap();
1345 let mut job = reservation(home.path(), "task:one");
1346 job.created_at = 0;
1347 save(
1348 &job_dir(home.path(), &job.id).unwrap().join("job.json"),
1349 &job,
1350 )
1351 .unwrap();
1352 assert!(active(home.path()).unwrap().is_empty());
1353 assert!(
1354 run(home.path(), &job.id)
1355 .await
1356 .unwrap_err()
1357 .to_string()
1358 .contains("no longer startable")
1359 );
1360 }
1361
1362 #[test]
1363 fn running_without_lease_fails_regardless_of_pid() {
1364 let home = tempfile::tempdir().unwrap();
1365 let mut job = reservation(home.path(), "task:one");
1366 job.status = Status::Running;
1367 job.pid = Some(std::process::id());
1368 save(
1369 &job_dir(home.path(), &job.id).unwrap().join("job.json"),
1370 &job,
1371 )
1372 .unwrap();
1373 assert!(active(home.path()).unwrap().is_empty());
1374 assert_eq!(get(home.path(), &job.id).unwrap().status, Status::Failed);
1375 }
1376
1377 #[test]
1378 fn controls_survive_reopen_and_reset_at_new_invocation() {
1379 let home = tempfile::tempdir().unwrap();
1380 request(home.path(), "task:one", Control::Park).unwrap();
1381 assert_eq!(
1382 control(home.path(), "task:one").unwrap(),
1383 Some(Control::Park)
1384 );
1385 reservation(home.path(), "task:one");
1386 assert_eq!(control(home.path(), "task:one").unwrap(), None);
1387 request(home.path(), "task:one", Control::Stop).unwrap();
1388 assert_eq!(
1389 control(home.path(), "task:one").unwrap(),
1390 Some(Control::Stop)
1391 );
1392 assert_eq!(control(home.path(), "task:other").unwrap(), None);
1393 }
1394
1395 #[test]
1396 fn singleton_locks_release_without_pid_cleanup() {
1397 let home = tempfile::tempdir().unwrap();
1398 let lease = try_lease(home.path(), "scheduler").unwrap().unwrap();
1399 assert!(try_lease(home.path(), "scheduler").unwrap().is_none());
1400 assert!(try_lease(home.path(), "other").unwrap().is_some());
1401 drop(lease);
1402 assert!(try_lease(home.path(), "scheduler").unwrap().is_some());
1403 }
1404
1405 #[test]
1406 fn immutable_generation_keeps_old_content_after_source_replacement() {
1407 let home = tempfile::tempdir().unwrap();
1408 let source = home.path().join("source");
1409 std::fs::write(&source, b"old executable").unwrap();
1410 let old = pin_source(home.path(), &source).unwrap();
1411 std::fs::write(&source, b"new executable").unwrap();
1412 let new = pin_source(home.path(), &source).unwrap();
1413 assert_ne!(old.id, new.id);
1414 assert_eq!(
1415 std::fs::read(executable(home.path(), &old.id).unwrap()).unwrap(),
1416 b"old executable"
1417 );
1418 assert_eq!(std::fs::read(new.executable).unwrap(), b"new executable");
1419 }
1420
1421 #[tokio::test]
1422 async fn generation_spawn_does_not_retry_a_missing_executable() {
1423 let home = tempfile::tempdir().unwrap();
1424 let mut command = Command::new(home.path().join("missing-executable"));
1425 let mut attempts = 0;
1426 let error = spawn_generation(|| {
1427 attempts += 1;
1428 command.spawn()
1429 })
1430 .await
1431 .unwrap_err();
1432 assert_eq!(error.kind(), std::io::ErrorKind::NotFound);
1433 assert_eq!(attempts, 1);
1434 }
1435
1436 #[cfg(target_os = "linux")]
1437 fn busy_executable(dir: &Path) -> (Command, File) {
1438 use std::os::unix::fs::PermissionsExt;
1439
1440 let path = dir.join("busy-executable");
1441 std::fs::copy("/bin/sh", &path).unwrap();
1442 std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o700)).unwrap();
1443 let writer = OpenOptions::new().write(true).open(&path).unwrap();
1444 let mut command = Command::new(path);
1445 command.args(["-c", "exit 0"]);
1446 (command, writer)
1447 }
1448
1449 #[cfg(target_os = "linux")]
1450 #[tokio::test]
1451 async fn generation_spawn_retries_until_the_writer_closes() {
1452 let home = tempfile::tempdir().unwrap();
1453 let (mut command, writer) = busy_executable(home.path());
1454 let error = command.spawn().unwrap_err();
1455 assert_eq!(error.kind(), std::io::ErrorKind::ExecutableFileBusy);
1456
1457 let (release, requested) = std::sync::mpsc::channel();
1458 let closer = std::thread::spawn(move || {
1459 if requested.recv().is_ok() {
1460 std::thread::sleep(Duration::from_millis(20));
1461 }
1462 drop(writer);
1463 });
1464 let mut release = Some(release);
1465 let child = spawn_generation(|| {
1466 let result = command.spawn();
1467 if let Some(release) = release.take() {
1468 release.send(()).unwrap();
1469 }
1470 result
1471 })
1472 .await;
1473 closer.join().unwrap();
1474 assert!(child.unwrap().wait().unwrap().success());
1475 }
1476
1477 #[cfg(target_os = "linux")]
1478 #[tokio::test]
1479 async fn generation_spawn_stops_retrying_while_the_writer_remains_open() {
1480 let home = tempfile::tempdir().unwrap();
1481 let (mut command, _writer) = busy_executable(home.path());
1482 let mut attempts = 0;
1483 let error = spawn_generation(|| {
1484 attempts += 1;
1485 assert!(attempts <= 5, "busy generation exceeded its retry bound");
1486 command.spawn()
1487 })
1488 .await
1489 .unwrap_err();
1490 assert_eq!(error.kind(), std::io::ErrorKind::ExecutableFileBusy);
1491 }
1492
1493 #[tokio::test]
1494 async fn acknowledgement_records_own_version_and_handler_failure() {
1495 let home = tempfile::tempdir().unwrap();
1496 let generation = pin_source(home.path(), &std::env::current_exe().unwrap()).unwrap();
1497 let (job, _) = reserve(
1498 home.path(),
1499 &generation,
1500 "unknown-kind",
1501 "unknown:one",
1502 serde_json::Value::Null,
1503 )
1504 .unwrap();
1505 let invoke = || async {
1506 let mut command = Command::new(&generation.executable);
1507 command
1508 .args([
1509 "--exact",
1510 "worker::tests::pinned_entrypoint_helper",
1511 "--ignored",
1512 "--nocapture",
1513 ])
1514 .env("MAGI_WORKER_TEST_HOME", home.path())
1515 .env("MAGI_WORKER_TEST_ID", &job.id)
1516 .stdin(Stdio::null())
1517 .stdout(Stdio::piped())
1518 .stderr(Stdio::piped());
1519 let output = spawn_generation(|| command.spawn())
1520 .await
1521 .unwrap()
1522 .wait_with_output()
1523 .unwrap();
1524 assert!(
1525 output.status.success(),
1526 "{}",
1527 String::from_utf8_lossy(&output.stderr)
1528 );
1529 };
1530 invoke().await;
1531 let failed = get(home.path(), &job.id).unwrap();
1532 assert_eq!(failed.status, Status::Failed);
1533 assert_eq!(failed.version.as_deref(), Some(env!("CARGO_PKG_VERSION")));
1534 assert!(failed.result.unwrap().contains("unknown worker kind"));
1535 assert!(active(home.path()).unwrap().is_empty());
1536 assert!(!job_dir(home.path(), &job.id).unwrap().exists());
1537 invoke().await;
1538 assert!(!job_dir(home.path(), &job.id).unwrap().exists());
1539 }
1540
1541 #[tokio::test]
1542 #[ignore = "subprocess entrypoint for the immutable-generation startup test"]
1543 async fn pinned_entrypoint_helper() {
1544 let home = PathBuf::from(std::env::var_os("MAGI_WORKER_TEST_HOME").unwrap());
1545 let id = std::env::var("MAGI_WORKER_TEST_ID").unwrap();
1546 assert!(run(&home, &id).await.is_err());
1547 }
1548
1549 #[test]
1550 fn unfinished_staging_is_reclaimed_without_removing_generations() {
1551 let home = tempfile::tempdir().unwrap();
1552 let old = root(home.path()).join("generations").join(".new-abandoned");
1553 std::fs::create_dir_all(&old).unwrap();
1554 let partial = old.join("partial");
1555 std::fs::write(&partial, b"interrupted copy").unwrap();
1556 #[cfg(windows)]
1557 {
1558 let mut permissions = std::fs::metadata(&partial).unwrap().permissions();
1559 permissions.set_readonly(true);
1560 std::fs::set_permissions(&partial, permissions).unwrap();
1561 }
1562 let source = home.path().join("source");
1563 std::fs::write(&source, b"complete executable").unwrap();
1564 let generation = pin_source(home.path(), &source).unwrap();
1565 assert!(!old.exists());
1566 assert!(generation.executable.is_file());
1567 }
1568
1569 #[tokio::test]
1570 async fn unrelated_executable_cannot_acknowledge_or_consume_reservation() {
1571 let home = tempfile::tempdir().unwrap();
1572 let source = home.path().join("other-binary");
1573 std::fs::write(&source, b"different executable of the same Cargo version").unwrap();
1574 let generation = pin_source(home.path(), &source).unwrap();
1575 let (job, _) = reserve(
1576 home.path(),
1577 &generation,
1578 "task",
1579 "task:one",
1580 serde_json::Value::Null,
1581 )
1582 .unwrap();
1583 assert!(
1584 run(home.path(), &job.id)
1585 .await
1586 .unwrap_err()
1587 .to_string()
1588 .contains("immutable generation")
1589 );
1590 assert_eq!(get(home.path(), &job.id).unwrap().status, Status::Starting);
1591 assert_eq!(active(home.path()).unwrap().len(), 1);
1592 }
1593
1594 #[test]
1595 fn abandoned_record_staging_is_bounded_and_never_executes() {
1596 let home = tempfile::tempdir().unwrap();
1597 let staging = root(home.path()).join("jobs").join(".new-interrupted");
1598 std::fs::create_dir_all(&staging).unwrap();
1599 std::fs::write(staging.join("job.json"), b"partial json").unwrap();
1600 assert!(active(home.path()).unwrap().is_empty());
1601 assert!(!staging.exists());
1602 }
1603
1604 #[cfg(unix)]
1605 #[tokio::test]
1606 async fn detached_startup_preserves_stdout_and_stderr_in_record_log() {
1607 use std::os::unix::fs::PermissionsExt;
1608 let home = tempfile::tempdir().unwrap();
1609 let source = home.path().join("startup-diagnostic");
1610 std::fs::write(
1611 &source,
1612 b"#!/bin/sh\nprintf 'startup stdout\\n'\nprintf 'startup stderr\\n' >&2\nexit 7\n",
1613 )
1614 .unwrap();
1615 std::fs::set_permissions(&source, std::fs::Permissions::from_mode(0o755)).unwrap();
1616 let generation = pin_source(home.path(), &source).unwrap();
1617 let (mut job, _) = reserve(
1618 home.path(),
1619 &generation,
1620 "task",
1621 "task:log",
1622 serde_json::Value::Null,
1623 )
1624 .unwrap();
1625 let mut command = detached_command(home.path(), &job, &generation).unwrap();
1626 for key in ["MAGI_RUN", "MAGI_NODE", "MAGI_SEAT"]
1627 .into_iter()
1628 .chain(crate::chat_control::bridge::ENVIRONMENT.iter().copied())
1629 {
1630 assert!(
1631 command
1632 .get_envs()
1633 .any(|(name, value)| name == key && value.is_none())
1634 );
1635 }
1636 let mut child = spawn_generation(|| command.spawn()).await.unwrap();
1637 assert_eq!(child.wait().unwrap().code(), Some(7));
1638 let log = job_dir(home.path(), &job.id).unwrap().join("worker.log");
1639 let output = std::fs::read_to_string(log).unwrap();
1640 assert!(output.contains("startup stdout"));
1641 assert!(output.contains("startup stderr"));
1642 terminal(
1643 home.path(),
1644 &mut job,
1645 Status::Failed,
1646 "startup failed".into(),
1647 )
1648 .unwrap();
1649 assert!(active(home.path()).unwrap().is_empty());
1650 assert!(
1651 root(home.path())
1652 .join("results")
1653 .join(&job.id)
1654 .join("worker.log")
1655 .is_file()
1656 );
1657 }
1658
1659 #[cfg(windows)]
1660 #[test]
1661 fn record_publication_survives_readers_and_short_delete_sharing_handoffs() {
1662 use std::os::windows::fs::OpenOptionsExt as _;
1663
1664 let home = tempfile::tempdir().unwrap();
1665 let mut job = reservation(home.path(), "task:metadata-handoff");
1666 let directory = job_dir(home.path(), &job.id).unwrap();
1667 let path = directory.join("job.json");
1668 let lease = try_lock(&directory.join("live.lock")).unwrap().unwrap();
1669 let snapshot = File::open(&path).unwrap();
1670 let reader = OpenOptions::new()
1672 .read(true)
1673 .share_mode(1 | 2) .open(&path)
1675 .unwrap();
1676 std::thread::scope(|scope| {
1677 scope.spawn(move || {
1678 std::thread::sleep(Duration::from_millis(15));
1679 drop(reader);
1680 });
1681 job.status = Status::Running;
1682 save(&path, &job).unwrap();
1683 });
1684 let original: Job = serde_json::from_reader(snapshot).unwrap();
1685 assert_eq!(original.status, Status::Starting);
1686 assert_eq!(get(home.path(), &job.id).unwrap().status, Status::Running);
1687 let running_snapshot = File::open(&path).unwrap();
1688 terminal(home.path(), &mut job, Status::Finished, "completed".into()).unwrap();
1689 assert_eq!(get(home.path(), &job.id).unwrap().status, Status::Finished);
1690 let previous: Job = serde_json::from_reader(running_snapshot).unwrap();
1691 assert_eq!(previous.status, Status::Running);
1692 drop(lease);
1693 assert!(active(home.path()).unwrap().is_empty());
1694 }
1695
1696 #[cfg(windows)]
1697 #[test]
1698 fn denied_record_publication_preserves_reservation_and_original_os_error() {
1699 use std::os::windows::fs::OpenOptionsExt as _;
1700
1701 let home = tempfile::tempdir().unwrap();
1702 let job = reservation(home.path(), "task:metadata-denied");
1703 let directory = job_dir(home.path(), &job.id).unwrap();
1704 let path = directory.join("job.json");
1705 let reader = OpenOptions::new()
1706 .read(true)
1707 .share_mode(1 | 2) .open(&path)
1709 .unwrap();
1710 let mut acknowledged = job.clone();
1711 acknowledged.status = Status::Running;
1712 let error = save(&path, &acknowledged).unwrap_err();
1713 assert!(format!("{error:#}").contains("publish worker record"));
1714 let io = error.downcast_ref::<std::io::Error>().unwrap();
1715 assert!(matches!(io.raw_os_error(), Some(5 | 32)));
1716 assert_eq!(get(home.path(), &job.id).unwrap().status, Status::Starting);
1717 assert_eq!(reservation(home.path(), &job.key).id, job.id);
1718 assert!(std::fs::read_dir(&directory).unwrap().all(|entry| {
1719 !entry
1720 .unwrap()
1721 .file_name()
1722 .to_string_lossy()
1723 .starts_with(".pending-")
1724 }));
1725 drop(reader);
1726 save(&path, &acknowledged).unwrap();
1727 assert_eq!(get(home.path(), &job.id).unwrap().status, Status::Running);
1728 }
1729
1730 #[test]
1731 fn terminal_result_survives_open_execution_and_output_handles() {
1732 let home = tempfile::tempdir().unwrap();
1733 let mut job = reservation(home.path(), "task:open-handles");
1734 let directory = job_dir(home.path(), &job.id).unwrap();
1735 let lease = try_lock(&directory.join("live.lock")).unwrap().unwrap();
1736 let mut log = OpenOptions::new()
1737 .create(true)
1738 .append(true)
1739 .open(directory.join("worker.log"))
1740 .unwrap();
1741 writeln!(log, "before completion").unwrap();
1742 terminal(home.path(), &mut job, Status::Finished, "completed".into()).unwrap();
1743 assert_eq!(get(home.path(), &job.id).unwrap().status, Status::Finished);
1744 assert!(active(home.path()).unwrap().is_empty());
1745 assert!(
1746 directory.exists(),
1747 "a held execution lease must prevent retirement"
1748 );
1749 drop(lease);
1750 assert!(active(home.path()).unwrap().is_empty());
1753 assert_eq!(get(home.path(), &job.id).unwrap().status, Status::Finished);
1754 writeln!(log, "after completion").unwrap();
1755 drop(log);
1756 assert!(active(home.path()).unwrap().is_empty());
1757 assert!(
1758 root(home.path())
1759 .join("results")
1760 .join(&job.id)
1761 .join("worker.log")
1762 .is_file()
1763 );
1764 assert!(!directory.exists());
1765 }
1766
1767 #[test]
1768 fn terminal_records_are_bounded_without_pruning_live_reservations() {
1769 let home = tempfile::tempdir().unwrap();
1770 let live = reservation(home.path(), "task:live");
1771 let mut finished = reservation(home.path(), "task:finished");
1775 terminal(home.path(), &mut finished, Status::Finished, "done".into()).unwrap();
1776 let template_dir = job_dir(home.path(), &finished.id).unwrap();
1777 std::fs::remove_dir_all(template_dir).unwrap();
1778 for i in 0..RETAIN_FINISHED + 3 {
1779 let mut job = finished.clone();
1780 job.id = format!("{i:032x}");
1781 job.key = format!("task:{i}");
1782 job.sequence += i as u64;
1783 let directory = job_dir(home.path(), &job.id).unwrap();
1784 std::fs::create_dir_all(&directory).unwrap();
1785 std::fs::write(
1786 directory.join("job.json"),
1787 serde_json::to_vec(&job).unwrap(),
1788 )
1789 .unwrap();
1790 }
1791 assert_eq!(
1792 std::fs::read_dir(root(home.path()).join("jobs"))
1793 .unwrap()
1794 .count(),
1795 RETAIN_FINISHED + 4
1796 );
1797 let jobs = records(home.path()).unwrap();
1798 assert_eq!(jobs.len(), 1);
1799 assert_eq!(
1800 std::fs::read_dir(root(home.path()).join("results"))
1801 .unwrap()
1802 .count(),
1803 RETAIN_FINISHED
1804 );
1805 assert!(jobs.iter().any(|job| job.id == live.id));
1806 }
1807}