bob/
build.rs

1/*
2 * Copyright (c) 2025 Jonathan Perkin <jonathan@perkin.org.uk>
3 *
4 * Permission to use, copy, modify, and distribute this software for any
5 * purpose with or without fee is hereby granted, provided that the above
6 * copyright notice and this permission notice appear in all copies.
7 *
8 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
9 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
10 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
11 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
12 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
13 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
14 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
15 */
16
17//! Parallel package builds.
18//!
19//! This module provides the [`Build`] struct for building packages in parallel
20//! across multiple sandboxes. Packages are scheduled using a dependency graph
21//! to ensure correct build order.
22//!
23//! # Build Process
24//!
25//! 1. Create build sandboxes (one per `build_threads`)
26//! 2. Execute pre-build script in each sandbox
27//! 3. Build packages in parallel, respecting dependencies
28//! 4. Execute post-build script after each package
29//! 5. Destroy sandboxes and generate report
30//!
31//! # Build Phases
32//!
33//! Each package goes through these phases (as defined in `pkg-build` script):
34//!
35//! - `pre-clean` - Clean any previous build artifacts
36//! - `depends` - Install required dependencies
37//! - `checksum` - Verify distfile checksums
38//! - `configure` - Configure the build
39//! - `build` - Compile the package
40//! - `install` - Install to staging area
41//! - `package` - Create binary package
42//! - `deinstall` - Test package removal (non-bootstrap only)
43//! - `clean` - Clean up build artifacts
44//!
45//! # Example
46//!
47//! ```no_run
48//! use bob::{Build, Config, RunContext, Scan};
49//! use std::sync::Arc;
50//! use std::sync::atomic::AtomicBool;
51//!
52//! let config = Config::load(None, false)?;
53//! let mut scan = Scan::new(&config);
54//! // Add packages...
55//! let ctx = RunContext::new(Arc::new(AtomicBool::new(false)));
56//! scan.start(&ctx)?;
57//! let result = scan.resolve()?;
58//!
59//! let mut build = Build::new(&config, result.buildable);
60//! let summary = build.start(&ctx)?;
61//!
62//! println!("Built {} packages", summary.success_count());
63//! # Ok::<(), anyhow::Error>(())
64//! ```
65
66use crate::scan::ResolvedIndex;
67use crate::scan::ScanFailure;
68use crate::status::{self, StatusMessage};
69use crate::tui::{MultiProgress, format_duration};
70use crate::{Config, RunContext, Sandbox};
71use anyhow::{Context, bail};
72use glob::Pattern;
73use indexmap::IndexMap;
74use pkgsrc::{PkgName, PkgPath};
75use std::collections::{HashMap, HashSet};
76use std::fs;
77use std::path::{Path, PathBuf};
78use std::process::Command;
79use std::sync::atomic::{AtomicBool, Ordering};
80use std::sync::{Arc, Mutex, mpsc, mpsc::Sender};
81use std::time::{Duration, Instant};
82use tracing::{debug, error, info, trace, warn};
83
84/// Format a ResolvedIndex as pbulk-index output for piping to scripts.
85fn format_scan_index(idx: &ResolvedIndex) -> String {
86    idx.to_string()
87}
88
89/// Outcome of a package build attempt.
90///
91/// Used in [`BuildResult`] to indicate whether the build succeeded, failed,
92/// or was skipped.
93#[derive(Clone, Debug, serde::Serialize, serde::Deserialize, PartialEq, Eq)]
94pub enum BuildOutcome {
95    /// Package built and packaged successfully.
96    Success,
97    /// Package build failed.
98    ///
99    /// The string contains the failure reason (e.g., "Failed in build phase").
100    Failed(String),
101    /// Package did not need to be built - we already have a binary package
102    /// for this revision.
103    UpToDate,
104    /// Package is marked with PKG_SKIP_REASON or PKG_FAIL_REASON so cannot
105    /// be built.
106    ///
107    /// The string contains the skip/fail reason.
108    PreFailed(String),
109    /// Package depends on a different package that has Failed.
110    ///
111    /// The string contains the name of the failed dependency.
112    IndirectFailed(String),
113    /// Package depends on a different package that has PreFailed.
114    ///
115    /// The string contains the name of the pre-failed dependency.
116    IndirectPreFailed(String),
117}
118
119/// Result of building a single package.
120///
121/// Contains the outcome, timing, and log location for a package build.
122#[derive(Clone, Debug, serde::Serialize, serde::Deserialize)]
123pub struct BuildResult {
124    /// Package name with version (e.g., `mutt-2.2.12`).
125    pub pkgname: PkgName,
126    /// Package path in pkgsrc (e.g., `mail/mutt`).
127    pub pkgpath: Option<PkgPath>,
128    /// Build outcome (success, failure, or skipped).
129    pub outcome: BuildOutcome,
130    /// Time spent building this package.
131    pub duration: Duration,
132    /// Path to build logs directory, if available.
133    ///
134    /// For failed builds, this contains `pre-clean.log`, `build.log`, etc.
135    /// Successful builds clean up their log directories.
136    pub log_dir: Option<PathBuf>,
137}
138
139/// Summary of an entire build run.
140///
141/// Contains timing information and results for all packages.
142///
143/// # Example
144///
145/// ```no_run
146/// # use bob::BuildSummary;
147/// # fn example(summary: &BuildSummary) {
148/// println!("Succeeded: {}", summary.success_count());
149/// println!("Failed: {}", summary.failed_count());
150/// println!("Skipped: {}", summary.skipped_count());
151/// println!("Duration: {:?}", summary.duration);
152///
153/// for result in summary.failed() {
154///     println!("  {} failed", result.pkgname.pkgname());
155/// }
156/// # }
157/// ```
158#[derive(Clone, Debug)]
159pub struct BuildSummary {
160    /// Total duration of the build run.
161    pub duration: Duration,
162    /// Results for each package.
163    pub results: Vec<BuildResult>,
164    /// Packages that failed to scan (bmake pbulk-index failed).
165    pub scan_failed: Vec<ScanFailure>,
166}
167
168impl BuildSummary {
169    /// Count of successfully built packages.
170    pub fn success_count(&self) -> usize {
171        self.results
172            .iter()
173            .filter(|r| matches!(r.outcome, BuildOutcome::Success))
174            .count()
175    }
176
177    /// Count of failed packages (direct build failures only).
178    pub fn failed_count(&self) -> usize {
179        self.results
180            .iter()
181            .filter(|r| matches!(r.outcome, BuildOutcome::Failed(_)))
182            .count()
183    }
184
185    /// Count of up-to-date packages (already have binary package).
186    pub fn up_to_date_count(&self) -> usize {
187        self.results
188            .iter()
189            .filter(|r| matches!(r.outcome, BuildOutcome::UpToDate))
190            .count()
191    }
192
193    /// Count of pre-failed packages (PKG_SKIP_REASON/PKG_FAIL_REASON).
194    pub fn prefailed_count(&self) -> usize {
195        self.results
196            .iter()
197            .filter(|r| matches!(r.outcome, BuildOutcome::PreFailed(_)))
198            .count()
199    }
200
201    /// Count of indirect failed packages (depend on Failed).
202    pub fn indirect_failed_count(&self) -> usize {
203        self.results
204            .iter()
205            .filter(|r| matches!(r.outcome, BuildOutcome::IndirectFailed(_)))
206            .count()
207    }
208
209    /// Count of indirect pre-failed packages (depend on PreFailed).
210    pub fn indirect_prefailed_count(&self) -> usize {
211        self.results
212            .iter()
213            .filter(|r| matches!(r.outcome, BuildOutcome::IndirectPreFailed(_)))
214            .count()
215    }
216
217    /// Count of packages that failed to scan.
218    pub fn scan_failed_count(&self) -> usize {
219        self.scan_failed.len()
220    }
221
222    /// Get all failed results (direct build failures only).
223    pub fn failed(&self) -> Vec<&BuildResult> {
224        self.results
225            .iter()
226            .filter(|r| matches!(r.outcome, BuildOutcome::Failed(_)))
227            .collect()
228    }
229
230    /// Get all successful results.
231    pub fn succeeded(&self) -> Vec<&BuildResult> {
232        self.results
233            .iter()
234            .filter(|r| matches!(r.outcome, BuildOutcome::Success))
235            .collect()
236    }
237
238    /// Get all up-to-date results.
239    pub fn up_to_date(&self) -> Vec<&BuildResult> {
240        self.results
241            .iter()
242            .filter(|r| matches!(r.outcome, BuildOutcome::UpToDate))
243            .collect()
244    }
245
246    /// Get all pre-failed results.
247    pub fn prefailed(&self) -> Vec<&BuildResult> {
248        self.results
249            .iter()
250            .filter(|r| matches!(r.outcome, BuildOutcome::PreFailed(_)))
251            .collect()
252    }
253
254    /// Get all indirect failed results.
255    pub fn indirect_failed(&self) -> Vec<&BuildResult> {
256        self.results
257            .iter()
258            .filter(|r| matches!(r.outcome, BuildOutcome::IndirectFailed(_)))
259            .collect()
260    }
261
262    /// Get all indirect pre-failed results.
263    pub fn indirect_prefailed(&self) -> Vec<&BuildResult> {
264        self.results
265            .iter()
266            .filter(|r| matches!(r.outcome, BuildOutcome::IndirectPreFailed(_)))
267            .collect()
268    }
269}
270
271#[derive(Debug, Default)]
272pub struct Build {
273    /// Parsed [`Config`].
274    config: Config,
275    /// [`Sandbox`] configuration.
276    sandbox: Sandbox,
277    /// List of packages to build, as input from Scan::resolve.
278    scanpkgs: IndexMap<PkgName, ResolvedIndex>,
279    /// Cached build results from previous run.
280    cached: IndexMap<PkgName, BuildResult>,
281}
282
283#[derive(Debug)]
284struct PackageBuild {
285    id: usize,
286    config: Config,
287    pkginfo: ResolvedIndex,
288    sandbox: Sandbox,
289}
290
291/// Helper for querying bmake variables with the correct environment.
292struct MakeQuery<'a> {
293    config: &'a Config,
294    sandbox: &'a Sandbox,
295    sandbox_id: usize,
296    pkgpath: &'a PkgPath,
297    env: &'a HashMap<String, String>,
298}
299
300impl<'a> MakeQuery<'a> {
301    fn new(
302        config: &'a Config,
303        sandbox: &'a Sandbox,
304        sandbox_id: usize,
305        pkgpath: &'a PkgPath,
306        env: &'a HashMap<String, String>,
307    ) -> Self {
308        Self { config, sandbox, sandbox_id, pkgpath, env }
309    }
310
311    /// Query a bmake variable value.
312    fn var(&self, name: &str) -> Option<String> {
313        let pkgdir = self.config.pkgsrc().join(self.pkgpath.as_path());
314
315        let mut cmd = if self.sandbox.enabled() {
316            let mut c = Command::new("/usr/sbin/chroot");
317            c.arg(self.sandbox.path(self.sandbox_id)).arg(self.config.make());
318            c
319        } else {
320            Command::new(self.config.make())
321        };
322
323        cmd.arg("-C")
324            .arg(&pkgdir)
325            .arg("show-var")
326            .arg(format!("VARNAME={}", name));
327
328        // Pass env vars that may affect the variable value
329        for (key, value) in self.env {
330            cmd.env(key, value);
331        }
332
333        let output = cmd.output().ok()?;
334
335        if !output.status.success() {
336            return None;
337        }
338
339        let value = String::from_utf8_lossy(&output.stdout).trim().to_string();
340
341        if value.is_empty() { None } else { Some(value) }
342    }
343
344    /// Query a bmake variable and return as PathBuf.
345    fn var_path(&self, name: &str) -> Option<PathBuf> {
346        self.var(name).map(PathBuf::from)
347    }
348
349    /// Get the WRKDIR for this package.
350    fn wrkdir(&self) -> Option<PathBuf> {
351        self.var_path("WRKDIR")
352    }
353
354    /// Get the WRKSRC for this package.
355    #[allow(dead_code)]
356    fn wrksrc(&self) -> Option<PathBuf> {
357        self.var_path("WRKSRC")
358    }
359
360    /// Get the DESTDIR for this package.
361    #[allow(dead_code)]
362    fn destdir(&self) -> Option<PathBuf> {
363        self.var_path("DESTDIR")
364    }
365
366    /// Get the PREFIX for this package.
367    #[allow(dead_code)]
368    fn prefix(&self) -> Option<PathBuf> {
369        self.var_path("PREFIX")
370    }
371
372    /// Resolve a path to its actual location on the host filesystem.
373    /// If sandboxed, prepends the sandbox root path.
374    fn resolve_path(&self, path: &Path) -> PathBuf {
375        if self.sandbox.enabled() {
376            self.sandbox
377                .path(self.sandbox_id)
378                .join(path.strip_prefix("/").unwrap_or(path))
379        } else {
380            path.to_path_buf()
381        }
382    }
383}
384
385/// Result of a single package build attempt.
386#[derive(Debug)]
387enum PackageBuildResult {
388    /// Build succeeded
389    Success,
390    /// Build failed
391    Failed,
392    /// Package was up-to-date, skipped
393    Skipped,
394}
395
396impl PackageBuild {
397    fn build(
398        &self,
399        status_tx: &Sender<ChannelCommand>,
400    ) -> anyhow::Result<PackageBuildResult> {
401        let pkgname = self.pkginfo.pkgname.pkgname();
402        info!(pkgname = %pkgname,
403            sandbox_id = self.id,
404            "Starting package build"
405        );
406
407        let Some(pkgpath) = &self.pkginfo.pkg_location else {
408            error!(pkgname = %pkgname, "Could not get PKGPATH for package");
409            bail!("Could not get PKGPATH for {}", pkgname);
410        };
411
412        let logdir = self.config.logdir();
413
414        // Core environment vars
415        let mut envs = self.config.script_env();
416
417        // Add script paths
418        if let Some(path) = self.config.script("pkg-up-to-date") {
419            envs.push((
420                "PKG_UP_TO_DATE".to_string(),
421                format!("{}", path.display()),
422            ));
423        }
424
425        // Get env vars from Lua config (function or table)
426        let pkg_env = match self.config.get_pkg_env(&self.pkginfo) {
427            Ok(env) => {
428                for (key, value) in &env {
429                    envs.push((key.clone(), value.clone()));
430                }
431                env
432            }
433            Err(e) => {
434                error!(pkgname = %pkgname, error = %e, "Failed to get env from Lua config");
435                HashMap::new()
436            }
437        };
438
439        // If we have save_wrkdir_patterns, tell the script not to clean so we can save files
440        let patterns = self.config.save_wrkdir_patterns();
441        if !patterns.is_empty() {
442            envs.push(("SKIP_CLEAN".to_string(), "1".to_string()));
443        }
444
445        let Some(pkg_build_script) = self.config.script("pkg-build") else {
446            error!(pkgname = %pkgname, "No pkg-build script defined");
447            bail!("No pkg-build script defined");
448        };
449
450        // Format ScanIndex as pbulk-index output for stdin
451        let stdin_data = format_scan_index(&self.pkginfo);
452
453        debug!(pkgname = %pkgname,
454            env_count = envs.len(),
455            "Executing build scripts"
456        );
457        trace!(pkgname = %pkgname,
458            envs = ?envs,
459            stdin = %stdin_data,
460            "Build environment variables"
461        );
462
463        // Run pre-build script if defined (always runs)
464        if let Some(pre_build) = self.config.script("pre-build") {
465            debug!(pkgname = %pkgname, "Running pre-build script");
466            let child = self.sandbox.execute(
467                self.id,
468                pre_build,
469                envs.clone(),
470                None,
471                None,
472            )?;
473            let output = child
474                .wait_with_output()
475                .context("Failed to wait for pre-build")?;
476            if !output.status.success() {
477                warn!(pkgname = %pkgname, exit_code = ?output.status.code(), "pre-build script failed");
478            }
479        }
480
481        // Run pkg-build with status and output channels
482        let (mut status_reader, status_writer) =
483            status::channel().context("Failed to create status channel")?;
484        let status_fd = status_writer.fd();
485
486        let (mut output_reader, output_writer) = status::output_channel()
487            .context("Failed to create output channel")?;
488        let output_fd = output_writer.fd();
489
490        // Pass the output fd to the script
491        envs.push(("bob_output_fd".to_string(), output_fd.to_string()));
492
493        let mut child = self.sandbox.execute(
494            self.id,
495            pkg_build_script,
496            envs.clone(),
497            Some(&stdin_data),
498            Some(status_fd),
499        )?;
500
501        // Close write ends in parent so we get EOF when child exits
502        status_writer.close();
503        output_writer.close();
504
505        // Track if we received a "skipped" message
506        let mut was_skipped = false;
507
508        // Poll status/output channels and child process
509        loop {
510            // Read any available status messages
511            for msg in status_reader.read_all() {
512                match msg {
513                    StatusMessage::Stage(stage) => {
514                        let _ = status_tx.send(ChannelCommand::StageUpdate(
515                            self.id,
516                            Some(stage),
517                        ));
518                    }
519                    StatusMessage::Skipped => {
520                        was_skipped = true;
521                    }
522                }
523            }
524
525            // Read any available output lines
526            let output_lines = output_reader.read_all_lines();
527            if !output_lines.is_empty() {
528                let _ = status_tx
529                    .send(ChannelCommand::OutputLines(self.id, output_lines));
530            }
531
532            // Check if child has exited
533            match child.try_wait() {
534                Ok(Some(_status)) => break,
535                Ok(None) => {
536                    // Still running, sleep briefly
537                    std::thread::sleep(Duration::from_millis(10));
538                }
539                Err(e) => {
540                    return Err(e).context("Failed to wait for pkg-build");
541                }
542            }
543        }
544
545        // Read any remaining output after child exits
546        let remaining = output_reader.read_all_lines();
547        if !remaining.is_empty() {
548            let _ =
549                status_tx.send(ChannelCommand::OutputLines(self.id, remaining));
550        }
551
552        // Clear stage display
553        let _ = status_tx.send(ChannelCommand::StageUpdate(self.id, None));
554
555        // Get final exit status
556        let status =
557            child.wait().context("Failed to get pkg-build exit status")?;
558
559        let result = if was_skipped {
560            info!(pkgname = %pkgname,
561                "pkg-build skipped (up-to-date)"
562            );
563            PackageBuildResult::Skipped
564        } else {
565            match status.code() {
566                Some(0) => {
567                    info!(pkgname = %pkgname,
568                        "pkg-build completed successfully"
569                    );
570                    PackageBuildResult::Success
571                }
572                Some(code) => {
573                    error!(pkgname = %pkgname,
574                        exit_code = code,
575                        "pkg-build failed"
576                    );
577
578                    // Save wrkdir files matching configured patterns, then clean up
579                    if !patterns.is_empty() {
580                        self.save_wrkdir_files(
581                            pkgname, pkgpath, logdir, patterns, &pkg_env,
582                        );
583                        self.run_clean(pkgpath);
584                    }
585                    PackageBuildResult::Failed
586                }
587                None => {
588                    // Process was terminated by signal (e.g., Ctrl+C)
589                    warn!(pkgname = %pkgname,
590                        "pkg-build terminated by signal"
591                    );
592                    PackageBuildResult::Failed
593                }
594            }
595        };
596
597        // Run post-build script if defined (always runs regardless of pkg-build result)
598        if let Some(post_build) = self.config.script("post-build") {
599            debug!(pkgname = %pkgname, "Running post-build script");
600            if let Ok(child) =
601                self.sandbox.execute(self.id, post_build, envs, None, None)
602            {
603                match child.wait_with_output() {
604                    Ok(output) if !output.status.success() => {
605                        warn!(pkgname = %pkgname, exit_code = ?output.status.code(), "post-build script failed");
606                    }
607                    Err(e) => {
608                        warn!(pkgname = %pkgname, error = %e, "Failed to wait for post-build");
609                    }
610                    _ => {}
611                }
612            }
613        }
614
615        Ok(result)
616    }
617
618    /// Save files matching patterns from WRKDIR to logdir on build failure.
619    fn save_wrkdir_files(
620        &self,
621        pkgname: &str,
622        pkgpath: &PkgPath,
623        logdir: &Path,
624        patterns: &[String],
625        pkg_env: &HashMap<String, String>,
626    ) {
627        let make = MakeQuery::new(
628            &self.config,
629            &self.sandbox,
630            self.id,
631            pkgpath,
632            pkg_env,
633        );
634
635        // Get WRKDIR
636        let wrkdir = match make.wrkdir() {
637            Some(w) => w,
638            None => {
639                debug!(pkgname = %pkgname, "Could not determine WRKDIR, skipping file save");
640                return;
641            }
642        };
643
644        // Resolve to actual filesystem path
645        let wrkdir_path = make.resolve_path(&wrkdir);
646
647        if !wrkdir_path.exists() {
648            debug!(pkgname = %pkgname,
649                wrkdir = %wrkdir_path.display(),
650                "WRKDIR does not exist, skipping file save"
651            );
652            return;
653        }
654
655        let save_dir = logdir.join(pkgname).join("wrkdir-files");
656        if let Err(e) = fs::create_dir_all(&save_dir) {
657            warn!(pkgname = %pkgname,
658                error = %e,
659                "Failed to create wrkdir-files directory"
660            );
661            return;
662        }
663
664        // Compile glob patterns
665        let compiled_patterns: Vec<Pattern> = patterns
666            .iter()
667            .filter_map(|p| {
668                Pattern::new(p).ok().or_else(|| {
669                    warn!(pattern = %p, "Invalid glob pattern");
670                    None
671                })
672            })
673            .collect();
674
675        if compiled_patterns.is_empty() {
676            return;
677        }
678
679        // Walk the wrkdir and find matching files
680        let mut saved_count = 0;
681        if let Err(e) = walk_and_save(
682            &wrkdir_path,
683            &wrkdir_path,
684            &save_dir,
685            &compiled_patterns,
686            &mut saved_count,
687        ) {
688            warn!(pkgname = %pkgname,
689                error = %e,
690                "Error while saving wrkdir files"
691            );
692        }
693
694        if saved_count > 0 {
695            info!(pkgname = %pkgname,
696                count = saved_count,
697                dest = %save_dir.display(),
698                "Saved wrkdir files"
699            );
700        }
701    }
702
703    /// Run bmake clean for a package.
704    fn run_clean(&self, pkgpath: &PkgPath) {
705        let pkgdir = self.config.pkgsrc().join(pkgpath.as_path());
706
707        let result = if self.sandbox.enabled() {
708            Command::new("/usr/sbin/chroot")
709                .arg(self.sandbox.path(self.id))
710                .arg(self.config.make())
711                .arg("-C")
712                .arg(&pkgdir)
713                .arg("clean")
714                .stdout(std::process::Stdio::null())
715                .stderr(std::process::Stdio::null())
716                .status()
717        } else {
718            Command::new(self.config.make())
719                .arg("-C")
720                .arg(&pkgdir)
721                .arg("clean")
722                .stdout(std::process::Stdio::null())
723                .stderr(std::process::Stdio::null())
724                .status()
725        };
726
727        if let Err(e) = result {
728            debug!(error = %e, "Failed to run bmake clean");
729        }
730    }
731}
732
733/// Recursively walk a directory and save files matching patterns.
734fn walk_and_save(
735    base: &Path,
736    current: &Path,
737    save_dir: &Path,
738    patterns: &[Pattern],
739    saved_count: &mut usize,
740) -> std::io::Result<()> {
741    if !current.is_dir() {
742        return Ok(());
743    }
744
745    for entry in fs::read_dir(current)? {
746        let entry = entry?;
747        let path = entry.path();
748
749        if path.is_dir() {
750            walk_and_save(base, &path, save_dir, patterns, saved_count)?;
751        } else if path.is_file() {
752            // Get relative path from base
753            let rel_path = path.strip_prefix(base).unwrap_or(&path);
754            let rel_str = rel_path.to_string_lossy();
755
756            // Check if any pattern matches
757            for pattern in patterns {
758                if pattern.matches(&rel_str)
759                    || pattern.matches(
760                        path.file_name()
761                            .unwrap_or_default()
762                            .to_string_lossy()
763                            .as_ref(),
764                    )
765                {
766                    // Create destination directory
767                    let dest_path = save_dir.join(rel_path);
768                    if let Some(parent) = dest_path.parent() {
769                        fs::create_dir_all(parent)?;
770                    }
771
772                    // Copy the file
773                    if let Err(e) = fs::copy(&path, &dest_path) {
774                        warn!(src = %path.display(),
775                            dest = %dest_path.display(),
776                            error = %e,
777                            "Failed to copy file"
778                        );
779                    } else {
780                        debug!(src = %path.display(),
781                            dest = %dest_path.display(),
782                            "Saved wrkdir file"
783                        );
784                        *saved_count += 1;
785                    }
786                    break; // Don't copy same file multiple times
787                }
788            }
789        }
790    }
791
792    Ok(())
793}
794
795/**
796 * Commands sent between the manager and clients.
797 */
798#[derive(Debug)]
799enum ChannelCommand {
800    /**
801     * Client (with specified identifier) indicating they are ready for work.
802     */
803    ClientReady(usize),
804    /**
805     * Manager has no work available at the moment, try again later.
806     */
807    ComeBackLater,
808    /**
809     * Manager directing a client to build a specific package.
810     */
811    JobData(Box<PackageBuild>),
812    /**
813     * Client returning a successful package build with duration.
814     */
815    JobSuccess(PkgName, Duration),
816    /**
817     * Client returning a failed package build with duration.
818     */
819    JobFailed(PkgName, Duration),
820    /**
821     * Client returning a skipped package (up-to-date).
822     */
823    JobSkipped(PkgName),
824    /**
825     * Client returning an error during the package build.
826     */
827    JobError((PkgName, Duration, anyhow::Error)),
828    /**
829     * Manager directing a client to quit.
830     */
831    Quit,
832    /**
833     * Shutdown signal - workers should stop immediately.
834     */
835    Shutdown,
836    /**
837     * Client reporting a stage update for a build.
838     */
839    StageUpdate(usize, Option<String>),
840    /**
841     * Client reporting output lines from a build.
842     */
843    OutputLines(usize, Vec<String>),
844}
845
846/**
847 * Return the current build job status.
848 */
849#[derive(Debug)]
850enum BuildStatus {
851    /**
852     * The next package ordered by priority is available for building.
853     */
854    Available(PkgName),
855    /**
856     * No packages are currently available for building, i.e. all remaining
857     * packages have at least one dependency that is still unavailable.
858     */
859    NoneAvailable,
860    /**
861     * All package builds have been completed.
862     */
863    Done,
864}
865
866#[derive(Clone, Debug)]
867struct BuildJobs {
868    scanpkgs: IndexMap<PkgName, ResolvedIndex>,
869    incoming: HashMap<PkgName, HashSet<PkgName>>,
870    running: HashSet<PkgName>,
871    done: HashSet<PkgName>,
872    failed: HashSet<PkgName>,
873    results: Vec<BuildResult>,
874    logdir: PathBuf,
875    /// Number of packages loaded from cache.
876    #[allow(dead_code)]
877    cached_count: usize,
878}
879
880impl BuildJobs {
881    /**
882     * Mark a package as successful and remove it from pending dependencies.
883     */
884    fn mark_success(&mut self, pkgname: &PkgName, duration: Duration) {
885        self.mark_done(pkgname, BuildOutcome::Success, duration);
886    }
887
888    /**
889     * Mark a package as up-to-date and remove it from pending dependencies.
890     */
891    fn mark_up_to_date(&mut self, pkgname: &PkgName) {
892        self.mark_done(pkgname, BuildOutcome::UpToDate, Duration::ZERO);
893    }
894
895    fn mark_done(
896        &mut self,
897        pkgname: &PkgName,
898        outcome: BuildOutcome,
899        duration: Duration,
900    ) {
901        /*
902         * Remove the package from the list of dependencies in all
903         * packages it is listed in.  Once a package has no outstanding
904         * dependencies remaining it is ready for building.
905         */
906        for dep in self.incoming.values_mut() {
907            if dep.contains(pkgname) {
908                dep.remove(pkgname);
909            }
910        }
911        /*
912         * The package was already removed from "incoming" when it started
913         * building, so we only need to add it to "done".
914         */
915        self.done.insert(pkgname.clone());
916
917        // Record the result
918        let scanpkg = self.scanpkgs.get(pkgname);
919        let log_dir = Some(self.logdir.join(pkgname.pkgname()));
920        self.results.push(BuildResult {
921            pkgname: pkgname.clone(),
922            pkgpath: scanpkg.and_then(|s| s.pkg_location.clone()),
923            outcome,
924            duration,
925            log_dir,
926        });
927    }
928
929    /**
930     * Recursively mark a package and its dependents as failed.
931     */
932    fn mark_failure(&mut self, pkgname: &PkgName, duration: Duration) {
933        let mut broken: HashSet<PkgName> = HashSet::new();
934        let mut to_check: Vec<PkgName> = vec![];
935        to_check.push(pkgname.clone());
936        /*
937         * Starting with the original failed package, recursively loop through
938         * adding any packages that depend on it, adding them to broken.
939         */
940        loop {
941            /* No packages left to check, we're done. */
942            let Some(badpkg) = to_check.pop() else {
943                break;
944            };
945            /* Already checked this package. */
946            if broken.contains(&badpkg) {
947                continue;
948            }
949            for (pkg, deps) in &self.incoming {
950                if deps.contains(&badpkg) {
951                    to_check.push(pkg.clone());
952                }
953            }
954            broken.insert(badpkg);
955        }
956        /*
957         * We now have a full HashSet of affected packages.  Remove them from
958         * incoming and move to failed.  The original failed package will
959         * already be removed from incoming, we rely on .remove() accepting
960         * this.
961         */
962        let is_original = |p: &PkgName| p == pkgname;
963        for pkg in broken {
964            self.incoming.remove(&pkg);
965            self.failed.insert(pkg.clone());
966
967            // Record the result
968            let scanpkg = self.scanpkgs.get(&pkg);
969            let log_dir = Some(self.logdir.join(pkg.pkgname()));
970            let (outcome, dur) = if is_original(&pkg) {
971                (BuildOutcome::Failed("Build failed".to_string()), duration)
972            } else {
973                (
974                    BuildOutcome::IndirectFailed(pkgname.pkgname().to_string()),
975                    Duration::ZERO,
976                )
977            };
978            self.results.push(BuildResult {
979                pkgname: pkg,
980                pkgpath: scanpkg.and_then(|s| s.pkg_location.clone()),
981                outcome,
982                duration: dur,
983                log_dir,
984            });
985        }
986    }
987
988    /**
989     * Recursively mark a package as pre-failed and its dependents as
990     * indirect-pre-failed.
991     */
992    #[allow(dead_code)]
993    fn mark_prefailed(&mut self, pkgname: &PkgName, reason: String) {
994        let mut broken: HashSet<PkgName> = HashSet::new();
995        let mut to_check: Vec<PkgName> = vec![];
996        to_check.push(pkgname.clone());
997
998        loop {
999            let Some(badpkg) = to_check.pop() else {
1000                break;
1001            };
1002            if broken.contains(&badpkg) {
1003                continue;
1004            }
1005            for (pkg, deps) in &self.incoming {
1006                if deps.contains(&badpkg) {
1007                    to_check.push(pkg.clone());
1008                }
1009            }
1010            broken.insert(badpkg);
1011        }
1012
1013        let is_original = |p: &PkgName| p == pkgname;
1014        for pkg in broken {
1015            self.incoming.remove(&pkg);
1016            self.failed.insert(pkg.clone());
1017
1018            let scanpkg = self.scanpkgs.get(&pkg);
1019            let log_dir = Some(self.logdir.join(pkg.pkgname()));
1020            let outcome = if is_original(&pkg) {
1021                BuildOutcome::PreFailed(reason.clone())
1022            } else {
1023                BuildOutcome::IndirectPreFailed(pkgname.pkgname().to_string())
1024            };
1025            self.results.push(BuildResult {
1026                pkgname: pkg,
1027                pkgpath: scanpkg.and_then(|s| s.pkg_location.clone()),
1028                outcome,
1029                duration: Duration::ZERO,
1030                log_dir,
1031            });
1032        }
1033    }
1034
1035    /**
1036     * Get next package status.
1037     */
1038    fn get_next_build(&self) -> BuildStatus {
1039        /*
1040         * If incoming is empty then we're done.
1041         */
1042        if self.incoming.is_empty() {
1043            return BuildStatus::Done;
1044        }
1045
1046        /*
1047         * Get all packages in incoming that are cleared for building, ordered
1048         * by weighting.
1049         *
1050         * TODO: weighting should be the sum of all transitive dependencies.
1051         */
1052        let mut pkgs: Vec<(PkgName, usize)> = self
1053            .incoming
1054            .iter()
1055            .filter(|(_, v)| v.is_empty())
1056            .map(|(k, _)| {
1057                (
1058                    k.clone(),
1059                    self.scanpkgs
1060                        .get(k)
1061                        .unwrap()
1062                        .pbulk_weight
1063                        .clone()
1064                        .unwrap_or("100".to_string())
1065                        .parse()
1066                        .unwrap_or(100),
1067                )
1068            })
1069            .collect();
1070
1071        /*
1072         * If no packages are returned then we're still waiting for
1073         * dependencies to finish.  Clients should keep retrying until this
1074         * changes.
1075         */
1076        if pkgs.is_empty() {
1077            return BuildStatus::NoneAvailable;
1078        }
1079
1080        /*
1081         * Order packages by build weight and return the highest.
1082         */
1083        pkgs.sort_by_key(|&(_, weight)| std::cmp::Reverse(weight));
1084        BuildStatus::Available(pkgs[0].0.clone())
1085    }
1086}
1087
1088impl Build {
1089    pub fn new(
1090        config: &Config,
1091        scanpkgs: IndexMap<PkgName, ResolvedIndex>,
1092    ) -> Build {
1093        let sandbox = Sandbox::new(config);
1094        info!(
1095            package_count = scanpkgs.len(),
1096            sandbox_enabled = sandbox.enabled(),
1097            build_threads = config.build_threads(),
1098            "Creating new Build instance"
1099        );
1100        for (pkgname, index) in &scanpkgs {
1101            debug!(pkgname = %pkgname.pkgname(),
1102                pkgpath = ?index.pkg_location,
1103                depends_count = index.depends.len(),
1104                depends = ?index.depends.iter().map(|d| d.pkgname()).collect::<Vec<_>>(),
1105                "Package in build queue"
1106            );
1107        }
1108        Build {
1109            config: config.clone(),
1110            sandbox,
1111            scanpkgs,
1112            cached: IndexMap::new(),
1113        }
1114    }
1115
1116    /// Load cached build results.
1117    ///
1118    /// Returns the number of packages loaded from cache. Packages that have
1119    /// not finished processing (Success, Failed, UpToDate, PreFailed, etc.)
1120    /// are not loaded.
1121    pub fn load_cached(
1122        &mut self,
1123        cached: IndexMap<PkgName, BuildResult>,
1124    ) -> usize {
1125        let mut count = 0;
1126        for (pkgname, result) in cached {
1127            // Only cache packages that are in our build queue
1128            if self.scanpkgs.contains_key(&pkgname) {
1129                self.cached.insert(pkgname, result);
1130                count += 1;
1131            }
1132        }
1133        info!(cached_count = count, "Loaded cached build results");
1134        count
1135    }
1136
1137    /// Access completed build results.
1138    pub fn cached(&self) -> &IndexMap<PkgName, BuildResult> {
1139        &self.cached
1140    }
1141
1142    pub fn start(&mut self, ctx: &RunContext) -> anyhow::Result<BuildSummary> {
1143        let started = Instant::now();
1144
1145        info!(package_count = self.scanpkgs.len(), "Build::start() called");
1146
1147        let shutdown_flag = Arc::clone(&ctx.shutdown);
1148        let stats = ctx.stats.clone();
1149
1150        /*
1151         * Populate BuildJobs.
1152         */
1153        debug!("Populating BuildJobs from scanpkgs");
1154        let mut incoming: HashMap<PkgName, HashSet<PkgName>> = HashMap::new();
1155        for (pkgname, index) in &self.scanpkgs {
1156            let mut deps: HashSet<PkgName> = HashSet::new();
1157            for dep in &index.depends {
1158                deps.insert(dep.clone());
1159            }
1160            trace!(pkgname = %pkgname.pkgname(),
1161                deps_count = deps.len(),
1162                deps = ?deps.iter().map(|d| d.pkgname()).collect::<Vec<_>>(),
1163                "Adding package to incoming build queue"
1164            );
1165            incoming.insert(pkgname.clone(), deps);
1166        }
1167
1168        /*
1169         * Process cached build results.
1170         */
1171        let mut done: HashSet<PkgName> = HashSet::new();
1172        let mut failed: HashSet<PkgName> = HashSet::new();
1173        let mut results: Vec<BuildResult> = Vec::new();
1174        let mut cached_count = 0usize;
1175
1176        for (pkgname, result) in &self.cached {
1177            match result.outcome {
1178                BuildOutcome::Success | BuildOutcome::UpToDate => {
1179                    // Completed package - remove from incoming, add to done
1180                    incoming.remove(pkgname);
1181                    done.insert(pkgname.clone());
1182                    // Remove from deps of other packages
1183                    for deps in incoming.values_mut() {
1184                        deps.remove(pkgname);
1185                    }
1186                    results.push(result.clone());
1187                    cached_count += 1;
1188                }
1189                BuildOutcome::Failed(_)
1190                | BuildOutcome::PreFailed(_)
1191                | BuildOutcome::IndirectFailed(_)
1192                | BuildOutcome::IndirectPreFailed(_) => {
1193                    // Failed package - remove from incoming, add to failed
1194                    incoming.remove(pkgname);
1195                    failed.insert(pkgname.clone());
1196                    results.push(result.clone());
1197                    cached_count += 1;
1198                }
1199            }
1200        }
1201
1202        if cached_count > 0 {
1203            println!("Loaded {} cached build results", cached_count);
1204        }
1205
1206        info!(
1207            incoming_count = incoming.len(),
1208            scanpkgs_count = self.scanpkgs.len(),
1209            cached_count = cached_count,
1210            "BuildJobs populated"
1211        );
1212
1213        let running: HashSet<PkgName> = HashSet::new();
1214        let logdir = self.config.logdir().clone();
1215        let jobs = BuildJobs {
1216            scanpkgs: self.scanpkgs.clone(),
1217            incoming,
1218            running,
1219            done,
1220            failed,
1221            results,
1222            logdir,
1223            cached_count,
1224        };
1225
1226        // Create sandboxes before starting progress display
1227        if self.sandbox.enabled() {
1228            println!("Creating sandboxes...");
1229            for i in 0..self.config.build_threads() {
1230                if let Err(e) = self.sandbox.create(i) {
1231                    // Rollback: destroy sandboxes including the failed one (may be partial)
1232                    for j in (0..=i).rev() {
1233                        if let Err(destroy_err) = self.sandbox.destroy(j) {
1234                            eprintln!(
1235                                "Warning: failed to destroy sandbox {}: {}",
1236                                j, destroy_err
1237                            );
1238                        }
1239                    }
1240                    return Err(e);
1241                }
1242            }
1243        }
1244
1245        println!("Building packages...");
1246
1247        // Set up multi-line progress display using ratatui inline viewport
1248        let progress = Arc::new(Mutex::new(
1249            MultiProgress::new(
1250                "Building",
1251                "Built",
1252                self.scanpkgs.len(),
1253                self.config.build_threads(),
1254            )
1255            .expect("Failed to initialize progress display"),
1256        ));
1257
1258        // Mark cached packages in progress display
1259        if cached_count > 0 {
1260            if let Ok(mut p) = progress.lock() {
1261                p.state_mut().cached = cached_count;
1262            }
1263        }
1264
1265        // Flag to stop the refresh thread
1266        let stop_refresh = Arc::new(AtomicBool::new(false));
1267
1268        // Spawn a thread to periodically refresh the display (for timer updates)
1269        let progress_refresh = Arc::clone(&progress);
1270        let stop_flag = Arc::clone(&stop_refresh);
1271        let shutdown_for_refresh = Arc::clone(&shutdown_flag);
1272        let refresh_thread = std::thread::spawn(move || {
1273            while !stop_flag.load(Ordering::Relaxed)
1274                && !shutdown_for_refresh.load(Ordering::SeqCst)
1275            {
1276                if let Ok(mut p) = progress_refresh.lock() {
1277                    // Check for keyboard events (like 'v' for view toggle)
1278                    let _ = p.poll_events();
1279                    let _ = p.render_throttled();
1280                }
1281                std::thread::sleep(Duration::from_millis(50));
1282            }
1283        });
1284
1285        /*
1286         * Configure a mananger channel.  This is used for clients to indicate
1287         * to the manager that they are ready for work.
1288         */
1289        let (manager_tx, manager_rx) = mpsc::channel::<ChannelCommand>();
1290
1291        /*
1292         * Client threads.  Each client has its own channel to the manager,
1293         * with the client sending ready status on the manager channel, and
1294         * receiving instructions on its private channel.
1295         */
1296        let mut threads = vec![];
1297        let mut clients: HashMap<usize, Sender<ChannelCommand>> =
1298            HashMap::new();
1299        for i in 0..self.config.build_threads() {
1300            let (client_tx, client_rx) = mpsc::channel::<ChannelCommand>();
1301            clients.insert(i, client_tx);
1302            let manager_tx = manager_tx.clone();
1303            let thread = std::thread::spawn(move || {
1304                loop {
1305                    // Use send() which can fail if receiver is dropped (manager shutdown)
1306                    if manager_tx.send(ChannelCommand::ClientReady(i)).is_err()
1307                    {
1308                        break;
1309                    }
1310
1311                    let Ok(msg) = client_rx.recv() else {
1312                        break;
1313                    };
1314
1315                    match msg {
1316                        ChannelCommand::ComeBackLater => {
1317                            std::thread::sleep(Duration::from_millis(100));
1318                            continue;
1319                        }
1320                        ChannelCommand::JobData(pkg) => {
1321                            let pkgname = pkg.pkginfo.pkgname.clone();
1322                            let build_start = Instant::now();
1323                            match pkg.build(&manager_tx) {
1324                                Ok(PackageBuildResult::Success) => {
1325                                    let duration = build_start.elapsed();
1326                                    let _ = manager_tx.send(
1327                                        ChannelCommand::JobSuccess(
1328                                            pkgname, duration,
1329                                        ),
1330                                    );
1331                                }
1332                                Ok(PackageBuildResult::Skipped) => {
1333                                    let _ = manager_tx.send(
1334                                        ChannelCommand::JobSkipped(pkgname),
1335                                    );
1336                                }
1337                                Ok(PackageBuildResult::Failed) => {
1338                                    let duration = build_start.elapsed();
1339                                    let _ = manager_tx.send(
1340                                        ChannelCommand::JobFailed(
1341                                            pkgname, duration,
1342                                        ),
1343                                    );
1344                                }
1345                                Err(e) => {
1346                                    let duration = build_start.elapsed();
1347                                    let _ = manager_tx.send(
1348                                        ChannelCommand::JobError((
1349                                            pkgname, duration, e,
1350                                        )),
1351                                    );
1352                                }
1353                            }
1354                            continue;
1355                        }
1356                        ChannelCommand::Quit | ChannelCommand::Shutdown => {
1357                            break;
1358                        }
1359                        _ => todo!(),
1360                    }
1361                }
1362            });
1363            threads.push(thread);
1364        }
1365
1366        /*
1367         * Manager thread.  Read incoming commands from clients and reply
1368         * accordingly.  Returns the build results via a channel.
1369         */
1370        let config = self.config.clone();
1371        let sandbox = self.sandbox.clone();
1372        let progress_clone = Arc::clone(&progress);
1373        let shutdown_for_manager = Arc::clone(&shutdown_flag);
1374        let stats_for_manager = stats.clone();
1375        let (results_tx, results_rx) = mpsc::channel::<Vec<BuildResult>>();
1376        let (interrupted_tx, interrupted_rx) = mpsc::channel::<bool>();
1377        let manager = std::thread::spawn(move || {
1378            let mut clients = clients.clone();
1379            let config = config.clone();
1380            let sandbox = sandbox.clone();
1381            let mut jobs = jobs.clone();
1382            let mut was_interrupted = false;
1383            let stats = stats_for_manager;
1384
1385            // Track which thread is building which package
1386            let mut thread_packages: HashMap<usize, PkgName> = HashMap::new();
1387
1388            loop {
1389                // Check shutdown flag periodically
1390                if shutdown_for_manager.load(Ordering::SeqCst) {
1391                    // Suppress all further output
1392                    if let Ok(mut p) = progress_clone.lock() {
1393                        p.state_mut().suppress();
1394                    }
1395                    // Send shutdown to all remaining clients
1396                    for (_, client) in clients.drain() {
1397                        let _ = client.send(ChannelCommand::Shutdown);
1398                    }
1399                    was_interrupted = true;
1400                    break;
1401                }
1402
1403                // Use recv_timeout to check shutdown flag periodically
1404                let command =
1405                    match manager_rx.recv_timeout(Duration::from_millis(50)) {
1406                        Ok(cmd) => cmd,
1407                        Err(mpsc::RecvTimeoutError::Timeout) => continue,
1408                        Err(mpsc::RecvTimeoutError::Disconnected) => break,
1409                    };
1410
1411                match command {
1412                    ChannelCommand::ClientReady(c) => {
1413                        let client = clients.get(&c).unwrap();
1414                        match jobs.get_next_build() {
1415                            BuildStatus::Available(pkg) => {
1416                                let pkginfo = jobs.scanpkgs.get(&pkg).unwrap();
1417                                jobs.incoming.remove(&pkg);
1418                                jobs.running.insert(pkg.clone());
1419
1420                                // Update thread progress
1421                                thread_packages.insert(c, pkg.clone());
1422                                if let Ok(mut p) = progress_clone.lock() {
1423                                    p.clear_output_buffer(c);
1424                                    p.state_mut()
1425                                        .set_worker_active(c, pkg.pkgname());
1426                                    let _ = p.render_throttled();
1427                                }
1428
1429                                let _ = client.send(ChannelCommand::JobData(
1430                                    Box::new(PackageBuild {
1431                                        id: c,
1432                                        config: config.clone(),
1433                                        pkginfo: pkginfo.clone(),
1434                                        sandbox: sandbox.clone(),
1435                                    }),
1436                                ));
1437                            }
1438                            BuildStatus::NoneAvailable => {
1439                                if let Ok(mut p) = progress_clone.lock() {
1440                                    p.clear_output_buffer(c);
1441                                    p.state_mut().set_worker_idle(c);
1442                                    let _ = p.render_throttled();
1443                                }
1444                                let _ =
1445                                    client.send(ChannelCommand::ComeBackLater);
1446                            }
1447                            BuildStatus::Done => {
1448                                if let Ok(mut p) = progress_clone.lock() {
1449                                    p.clear_output_buffer(c);
1450                                    p.state_mut().set_worker_idle(c);
1451                                    let _ = p.render_throttled();
1452                                }
1453                                let _ = client.send(ChannelCommand::Quit);
1454                                clients.remove(&c);
1455                                if clients.is_empty() {
1456                                    break;
1457                                }
1458                            }
1459                        };
1460                    }
1461                    ChannelCommand::JobSuccess(pkgname, duration) => {
1462                        // Don't report if we're shutting down
1463                        if shutdown_for_manager.load(Ordering::SeqCst) {
1464                            continue;
1465                        }
1466
1467                        // Record stats
1468                        if let Some(ref s) = stats {
1469                            let pkgpath = jobs
1470                                .scanpkgs
1471                                .get(&pkgname)
1472                                .and_then(|idx| idx.pkg_location.as_ref())
1473                                .map(|p| {
1474                                    p.as_path().to_string_lossy().to_string()
1475                                });
1476                            s.build(
1477                                pkgname.pkgname(),
1478                                pkgpath.as_deref(),
1479                                duration,
1480                                "success",
1481                            );
1482                        }
1483
1484                        jobs.mark_success(&pkgname, duration);
1485                        jobs.running.remove(&pkgname);
1486
1487                        // Find which thread completed and mark idle
1488                        if let Ok(mut p) = progress_clone.lock() {
1489                            let _ = p.print_status(&format!(
1490                                "       Built {} ({})",
1491                                pkgname.pkgname(),
1492                                format_duration(duration)
1493                            ));
1494                            p.state_mut().increment_completed();
1495                            for (tid, pkg) in &thread_packages {
1496                                if pkg == &pkgname {
1497                                    p.clear_output_buffer(*tid);
1498                                    p.state_mut().set_worker_idle(*tid);
1499                                    break;
1500                                }
1501                            }
1502                            let _ = p.render_throttled();
1503                        }
1504                    }
1505                    ChannelCommand::JobSkipped(pkgname) => {
1506                        // Don't report if we're shutting down
1507                        if shutdown_for_manager.load(Ordering::SeqCst) {
1508                            continue;
1509                        }
1510
1511                        // Record stats
1512                        if let Some(ref s) = stats {
1513                            let pkgpath = jobs
1514                                .scanpkgs
1515                                .get(&pkgname)
1516                                .and_then(|idx| idx.pkg_location.as_ref())
1517                                .map(|p| {
1518                                    p.as_path().to_string_lossy().to_string()
1519                                });
1520                            s.build(
1521                                pkgname.pkgname(),
1522                                pkgpath.as_deref(),
1523                                Duration::ZERO,
1524                                "skipped",
1525                            );
1526                        }
1527
1528                        jobs.mark_up_to_date(&pkgname);
1529                        jobs.running.remove(&pkgname);
1530
1531                        // Find which thread completed and mark idle
1532                        if let Ok(mut p) = progress_clone.lock() {
1533                            let _ = p.print_status(&format!(
1534                                "     Skipped {} (up-to-date)",
1535                                pkgname.pkgname()
1536                            ));
1537                            p.state_mut().increment_skipped();
1538                            for (tid, pkg) in &thread_packages {
1539                                if pkg == &pkgname {
1540                                    p.clear_output_buffer(*tid);
1541                                    p.state_mut().set_worker_idle(*tid);
1542                                    break;
1543                                }
1544                            }
1545                            let _ = p.render_throttled();
1546                        }
1547                    }
1548                    ChannelCommand::JobFailed(pkgname, duration) => {
1549                        // Don't report if we're shutting down
1550                        if shutdown_for_manager.load(Ordering::SeqCst) {
1551                            continue;
1552                        }
1553
1554                        // Record stats
1555                        if let Some(ref s) = stats {
1556                            let pkgpath = jobs
1557                                .scanpkgs
1558                                .get(&pkgname)
1559                                .and_then(|idx| idx.pkg_location.as_ref())
1560                                .map(|p| {
1561                                    p.as_path().to_string_lossy().to_string()
1562                                });
1563                            s.build(
1564                                pkgname.pkgname(),
1565                                pkgpath.as_deref(),
1566                                duration,
1567                                "failed",
1568                            );
1569                        }
1570
1571                        jobs.mark_failure(&pkgname, duration);
1572                        jobs.running.remove(&pkgname);
1573
1574                        // Find which thread failed and mark idle
1575                        if let Ok(mut p) = progress_clone.lock() {
1576                            let _ = p.print_status(&format!(
1577                                "      Failed {} ({})",
1578                                pkgname.pkgname(),
1579                                format_duration(duration)
1580                            ));
1581                            p.state_mut().increment_failed();
1582                            for (tid, pkg) in &thread_packages {
1583                                if pkg == &pkgname {
1584                                    p.clear_output_buffer(*tid);
1585                                    p.state_mut().set_worker_idle(*tid);
1586                                    break;
1587                                }
1588                            }
1589                            let _ = p.render_throttled();
1590                        }
1591                    }
1592                    ChannelCommand::JobError((pkgname, duration, e)) => {
1593                        // Don't report if we're shutting down
1594                        if shutdown_for_manager.load(Ordering::SeqCst) {
1595                            continue;
1596                        }
1597
1598                        // Record stats
1599                        if let Some(ref s) = stats {
1600                            let pkgpath = jobs
1601                                .scanpkgs
1602                                .get(&pkgname)
1603                                .and_then(|idx| idx.pkg_location.as_ref())
1604                                .map(|p| {
1605                                    p.as_path().to_string_lossy().to_string()
1606                                });
1607                            s.build(
1608                                pkgname.pkgname(),
1609                                pkgpath.as_deref(),
1610                                duration,
1611                                "error",
1612                            );
1613                        }
1614
1615                        jobs.mark_failure(&pkgname, duration);
1616                        jobs.running.remove(&pkgname);
1617
1618                        // Find which thread errored and mark idle
1619                        if let Ok(mut p) = progress_clone.lock() {
1620                            let _ = p.print_status(&format!(
1621                                "      Failed {} ({})",
1622                                pkgname.pkgname(),
1623                                format_duration(duration)
1624                            ));
1625                            p.state_mut().increment_failed();
1626                            for (tid, pkg) in &thread_packages {
1627                                if pkg == &pkgname {
1628                                    p.clear_output_buffer(*tid);
1629                                    p.state_mut().set_worker_idle(*tid);
1630                                    break;
1631                                }
1632                            }
1633                            let _ = p.render_throttled();
1634                        }
1635                        tracing::error!(error = %e, pkgname = %pkgname.pkgname(), "Build error");
1636                    }
1637                    ChannelCommand::StageUpdate(tid, stage) => {
1638                        if let Ok(mut p) = progress_clone.lock() {
1639                            p.state_mut()
1640                                .set_worker_stage(tid, stage.as_deref());
1641                            let _ = p.render_throttled();
1642                        }
1643                    }
1644                    ChannelCommand::OutputLines(tid, lines) => {
1645                        if let Ok(mut p) = progress_clone.lock() {
1646                            if let Some(buf) = p.output_buffer_mut(tid) {
1647                                for line in lines {
1648                                    buf.push(line);
1649                                }
1650                            }
1651                        }
1652                    }
1653                    _ => {}
1654                }
1655            }
1656
1657            // Send results and interrupted status back
1658            let _ = results_tx.send(jobs.results);
1659            let _ = interrupted_tx.send(was_interrupted);
1660        });
1661
1662        threads.push(manager);
1663        for thread in threads {
1664            thread.join().expect("thread panicked");
1665        }
1666
1667        // Stop the refresh thread
1668        stop_refresh.store(true, Ordering::Relaxed);
1669        let _ = refresh_thread.join();
1670
1671        // Check if we were interrupted
1672        let was_interrupted = interrupted_rx.recv().unwrap_or(false);
1673
1674        // Print appropriate summary
1675        if let Ok(mut p) = progress.lock() {
1676            if was_interrupted {
1677                let _ = p.finish_interrupted();
1678            } else {
1679                let _ = p.finish();
1680            }
1681        }
1682
1683        // Collect results from manager
1684        let results = results_rx.recv().unwrap_or_default();
1685        let summary = BuildSummary {
1686            duration: started.elapsed(),
1687            results,
1688            scan_failed: Vec::new(),
1689        };
1690
1691        if self.sandbox.enabled() {
1692            self.sandbox.destroy_all(self.config.build_threads())?;
1693        }
1694
1695        Ok(summary)
1696    }
1697}