bob 0.99.0

/*!
 * Integration tests for scan and build using a fake pkgsrc tree.
 *
 * Creates a temporary pkgsrc tree with real Makefiles that respond to the
 * targets bob invokes (pbulk-index, build stages, show-var, etc.).  Tests
 * call Scan::start() and Build::start() directly with sandboxes disabled,
 * exercising the full orchestration: thread pools, DAG scheduling, DB
 * persistence, failure propagation, and resume.
 */

use anyhow::{Context, Result};
use bob::PackageStateKind::*;
use bob::{
    Build, Config, Database, PackageState, RunState, Sandbox, Scan, ScanSummary, config::PkgsrcEnv,
    sandbox::SandboxScope,
};
use std::collections::HashMap;
use std::fs;
use std::os::unix::fs::PermissionsExt;
use std::path::PathBuf;
use std::process::Command;
use tempfile::TempDir;

struct PkgDef<'a> {
    pkgpath: &'a str,
    pkgname: &'a str,
    all_depends: &'a str,
    skip_reason: &'a str,
    fail_reason: &'a str,
    bootstrap_pkg: &'a str,
    usergroup_phase: &'a str,
    multi_version: &'a str,
    fail_target: Option<(&'a str, &'a str)>,
}

impl<'a> PkgDef<'a> {
    fn new(pkgpath: &'a str, pkgname: &'a str) -> Self {
        Self {
            pkgpath,
            pkgname,
            all_depends: "",
            skip_reason: "",
            fail_reason: "",
            bootstrap_pkg: "",
            usergroup_phase: "",
            multi_version: "",
            fail_target: None,
        }
    }

    fn depends(mut self, d: &'a str) -> Self {
        self.all_depends = d;
        self
    }

    fn skip(mut self, s: &'a str) -> Self {
        self.skip_reason = s;
        self
    }

    fn fail(mut self, f: &'a str) -> Self {
        self.fail_reason = f;
        self
    }

    fn bootstrap(mut self) -> Self {
        self.bootstrap_pkg = "yes";
        self
    }

    fn usergroup(mut self, u: &'a str) -> Self {
        self.usergroup_phase = u;
        self
    }

    fn multi(mut self, m: &'a str) -> Self {
        self.multi_version = m;
        self
    }

    fn fail_at(mut self, target: &'a str, msg: &'a str) -> Self {
        self.fail_target = Some((target, msg));
        self
    }
}

/// Detect a usable bmake binary.
fn find_make() -> Option<PathBuf> {
    for candidate in &["/opt/pkg/bin/bmake", "/usr/pkg/bin/bmake", "/usr/bin/make"] {
        let p = PathBuf::from(candidate);
        if p.exists() {
            return Some(p);
        }
    }
    None
}

/// A fake pkgsrc tree for integration testing.
struct TestHarness {
    _tmpdir: TempDir,
    root: PathBuf,
    make: PathBuf,
}

impl TestHarness {
    fn new() -> Result<Self> {
        let make = match find_make() {
            Some(m) => m,
            None => anyhow::bail!("bmake not found, skipping integration test"),
        };

        let tmpdir = TempDir::new().context("Failed to create temp dir")?;
        let root = tmpdir.path().to_path_buf();

        let harness = Self {
            _tmpdir: tmpdir,
            root,
            make,
        };
        harness.create_tree()?;
        Ok(harness)
    }

    fn pkgsrc(&self) -> PathBuf {
        self.root.join("pkgsrc")
    }

    fn dbdir(&self) -> PathBuf {
        self.root.join("db")
    }

    fn logdir(&self) -> PathBuf {
        self.dbdir().join("logs")
    }

    fn config_path(&self) -> PathBuf {
        self.root.join("config.lua")
    }

    fn packages_dir(&self) -> PathBuf {
        self.root.join("packages")
    }

    /// Create the full fake pkgsrc tree.
    fn create_tree(&self) -> Result<()> {
        // Create directory structure
        let dirs = [
            "pkgsrc/test/base",
            "pkgsrc/test/mid",
            "pkgsrc/test/also-base",
            "pkgsrc/test/top",
            "pkgsrc/test/multi",
            "pkgsrc/test/dual",
            "pkgsrc/test/skip-me",
            "pkgsrc/test/dep-skip",
            "pkgsrc/test/fail-me",
            "pkgsrc/test/dep-fail",
            "pkgsrc/test/bad-dep",
            "pkgsrc/test/build-fail",
            "pkgsrc/test/dep-bfail",
            "pkgsrc/test/fail-checksum",
            "pkgsrc/test/fail-at-build",
            "pkgsrc/test/fail-install",
            "pkgsrc/test/fail-package",
            "pkgsrc/test/chain-a",
            "pkgsrc/test/chain-b",
            "pkgsrc/test/chain-c",
            "pkgsrc/test/chain-d",
            "pkgsrc/pkgtools/pkg_install",
            "packages/All",
            "pkgtools",
            "pkg_dbdir",
            "pkg_refcount_dbdir",
            "prefix",
            "logs",
        ];
        for d in &dirs {
            fs::create_dir_all(self.root.join(d))?;
        }

        self.write_pkgsrc_root_makefile()?;
        self.write_category_makefile()?;
        self.write_pkg_install_makefile()?;
        self.write_mock_pkg_tools()?;
        self.write_package_makefiles()?;
        self.write_config_lua()?;

        Ok(())
    }

    /// pkgsrc root Makefile: show-subdir-var outputs category list.
    fn write_pkgsrc_root_makefile(&self) -> Result<()> {
        let content = "\
show-subdir-var:
\t@echo \"test\"
";
        fs::write(self.pkgsrc().join("Makefile"), content)?;
        Ok(())
    }

    /// Category Makefile: show-subdir-var outputs package list.
    fn write_category_makefile(&self) -> Result<()> {
        let content = "\
show-subdir-var:
\t@echo \"base mid also-base top multi dual skip-me dep-skip fail-me dep-fail bad-dep build-fail dep-bfail fail-checksum fail-at-build fail-install fail-package chain-a chain-b chain-c chain-d\"
";
        fs::write(self.pkgsrc().join("test/Makefile"), content)?;
        Ok(())
    }

    /// pkgtools/pkg_install Makefile: show-vars outputs environment values.
    fn write_pkg_install_makefile(&self) -> Result<()> {
        let content = format!(
            "\
show-vars:
\t@for v in ${{VARNAMES}}; do \\\n\
\t  case \"$$v\" in \\\n\
\t    PACKAGES) echo \"{packages}\" ;; \\\n\
\t    PKG_DBDIR) echo \"{pkg_dbdir}\" ;; \\\n\
\t    PKG_REFCOUNT_DBDIR) echo \"{pkg_refcount_dbdir}\" ;; \\\n\
\t    PKG_TOOLS_BIN) echo \"{pkgtools}\" ;; \\\n\
\t    PREFIX) echo \"{prefix}\" ;; \\\n\
\t    *) echo ;; \\\n\
\t  esac; \\\n\
\tdone
",
            packages = self.packages_dir().display(),
            pkg_dbdir = self.root.join("pkg_dbdir").display(),
            pkg_refcount_dbdir = self.root.join("pkg_refcount_dbdir").display(),
            pkgtools = self.root.join("pkgtools").display(),
            prefix = self.root.join("prefix").display(),
        );
        fs::write(self.pkgsrc().join("pkgtools/pkg_install/Makefile"), content)?;
        Ok(())
    }

    /// Mock pkg_add and pkg_delete that always succeed.
    fn write_mock_pkg_tools(&self) -> Result<()> {
        let script = "#!/bin/sh\nexit 0\n";
        for tool in &["pkg_add", "pkg_delete"] {
            let path = self.root.join("pkgtools").join(tool);
            fs::write(&path, script)?;
            fs::set_permissions(&path, fs::Permissions::from_mode(0o755))?;
        }
        Ok(())
    }

    /// Write all package Makefiles.
    fn write_package_makefiles(&self) -> Result<()> {
        let packages = [
            PkgDef::new("test/base", "base-1.0").bootstrap(),
            PkgDef::new("test/mid", "mid-1.0")
                .depends("base>=1.0:test/base")
                .usergroup("configure"),
            PkgDef::new("test/also-base", "also-base-1.0").depends("base>=1.0:test/base"),
            PkgDef::new("test/top", "top-1.0").depends(
                "mid-[0-9]*:test/mid \
                 also-base-[0-9]*:test/also-base",
            ),
            PkgDef::new("test/multi", "py313-multi-1.0").multi("PYTHON_VERSION_REQD=313"),
            PkgDef::new("test/skip-me", "skip-me-1.0").skip("not supported"),
            PkgDef::new("test/dep-skip", "dep-skip-1.0").depends("skip-me-[0-9]*:test/skip-me"),
            PkgDef::new("test/fail-me", "fail-me-1.0").fail("known broken"),
            PkgDef::new("test/dep-fail", "dep-fail-1.0").depends("fail-me-[0-9]*:test/fail-me"),
            PkgDef::new("test/bad-dep", "bad-dep-1.0")
                .depends("nonexistent-[0-9]*:test/nonexistent"),
            PkgDef::new("test/build-fail", "build-fail-1.0")
                .fail_at("configure", "configure failed"),
            PkgDef::new("test/dep-bfail", "dep-bfail-1.0")
                .depends("build-fail-[0-9]*:test/build-fail"),
            PkgDef::new("test/fail-checksum", "fail-checksum-1.0")
                .fail_at("checksum", "checksum mismatch"),
            PkgDef::new("test/fail-at-build", "fail-at-build-1.0")
                .fail_at("all", "compilation error"),
            PkgDef::new("test/fail-install", "fail-install-1.0")
                .fail_at("stage-install", "install failed: permission denied"),
            PkgDef::new("test/fail-package", "fail-package-1.0")
                .fail_at("stage-package-create", "pkg_create: error writing archive"),
            PkgDef::new("test/chain-d", "chain-d-1.0")
                .fail_at("configure", "chain-d configure failed"),
            PkgDef::new("test/chain-c", "chain-c-1.0").depends("chain-d-[0-9]*:test/chain-d"),
            PkgDef::new("test/chain-b", "chain-b-1.0").depends("chain-c-[0-9]*:test/chain-c"),
            PkgDef::new("test/chain-a", "chain-a-1.0").depends("chain-b-[0-9]*:test/chain-b"),
        ];
        for pkg in &packages {
            self.write_pkg_makefile(pkg)?;
        }
        self.write_dual_variant_makefile()?;
        Ok(())
    }

    fn write_dual_variant_makefile(&self) -> Result<()> {
        let content = "\
PKGNAME=py314-dual-1.0

pbulk-index:
\t@printf 'PKGNAME=py27-dual-1.0\\nALL_DEPENDS=\\nPKG_SKIP_REASON=\\nPKG_FAIL_REASON=\\nNO_BIN_ON_FTP=\\nRESTRICTED=\\nCATEGORIES=test\\nMAINTAINER=test@example.com\\nUSE_DESTDIR=yes\\nBOOTSTRAP_PKG=\\nUSERGROUP_PHASE=\\nSCAN_DEPENDS=\\nMULTI_VERSION=PYTHON_VERSION_REQD=27\\n'
\t@printf 'PKGNAME=py314-dual-1.0\\nALL_DEPENDS=\\nPKG_SKIP_REASON=\\nPKG_FAIL_REASON=\\nNO_BIN_ON_FTP=\\nRESTRICTED=\\nCATEGORIES=test\\nMAINTAINER=test@example.com\\nUSE_DESTDIR=yes\\nBOOTSTRAP_PKG=\\nUSERGROUP_PHASE=\\nSCAN_DEPENDS=\\nMULTI_VERSION=PYTHON_VERSION_REQD=314\\n'

clean checksum configure all stage-install create-usergroup:
\t@true

stage-package-create:
\t@mkdir -p ${.CURDIR}/pkg
\t@case \"${PYTHON_VERSION_REQD}\" in \\\n\
\t27) pkgname=py27-dual-1.0 ;; \\\n\
\t*) pkgname=py314-dual-1.0 ;; \\\n\
\tesac; \\\n\
\td=$$(mktemp -d) && \\\n\
\tprintf '@name %s\\n' \"$$pkgname\" > \"$$d/+CONTENTS\" && \\\n\
\tprintf 'Test package\\n' > \"$$d/+COMMENT\" && \\\n\
\tprintf 'Test package description\\n' > \"$$d/+DESC\" && \\\n\
\tprintf '0\\n' > \"$$d/+SIZE_PKG\" && \\\n\
\tprintf 'BUILD_DATE=%s\\nCATEGORIES=test\\nMACHINE_ARCH=x86_64\\nOPSYS=Test\\nOS_VERSION=1.0\\nPKGPATH=test/dual\\nPKGTOOLS_VERSION=20210710\\n' \\\n\
\t\"$$(date '+%Y-%m-%d %H:%M:%S %z')\" > \"$$d/+BUILD_INFO\" && \\\n\
\t(cd \"$$d\" && COPYFILE_DISABLE=1 tar czf \"${.CURDIR}/pkg/$$pkgname.tgz\" \\\n\
\t+CONTENTS +COMMENT +DESC +SIZE_PKG +BUILD_INFO) && \\\n\
\trm -rf \"$$d\"

show-var:
\t@case \"${VARNAME}:${PYTHON_VERSION_REQD}\" in \\\n\
\tSTAGE_PKGFILE:27) echo \"${.CURDIR}/pkg/py27-dual-1.0.tgz\" ;; \\\n\
\tSTAGE_PKGFILE:*) echo \"${.CURDIR}/pkg/py314-dual-1.0.tgz\" ;; \\\n\
\tesac

show-vars:
\t@for v in ${VARNAMES}; do \\\n\
\t  case \"$$v\" in \\\n\
\t    WRKDIR) echo \"${.CURDIR}/work\" ;; \\\n\
\t    *) echo ;; \\\n\
\t  esac; \\\n\
\tdone
";
        fs::write(self.pkgsrc().join("test/dual/Makefile"), content)?;
        Ok(())
    }

    fn write_pkg_makefile(&self, pkg: &PkgDef) -> Result<()> {
        let mut fields = vec![
            format!("PKGNAME={}\\n", pkg.pkgname),
            format!("PKG_LOCATION={}\\n", pkg.pkgpath),
            format!("ALL_DEPENDS={}\\n", pkg.all_depends),
            format!("PKG_SKIP_REASON={}\\n", pkg.skip_reason),
            format!("PKG_FAIL_REASON={}\\n", pkg.fail_reason),
            "NO_BIN_ON_FTP=\\n".to_string(),
            "RESTRICTED=\\n".to_string(),
            "CATEGORIES=test\\n".to_string(),
            "MAINTAINER=test@example.com\\n".to_string(),
            "USE_DESTDIR=yes\\n".to_string(),
            format!("BOOTSTRAP_PKG={}\\n", pkg.bootstrap_pkg),
            format!("USERGROUP_PHASE={}\\n", pkg.usergroup_phase),
            "SCAN_DEPENDS=\\n".to_string(),
        ];
        if !pkg.multi_version.is_empty() {
            fields.push(format!("MULTI_VERSION={}\\n", pkg.multi_version));
        }
        let index_body = fields.join("");

        let buildable = pkg.skip_reason.is_empty() && pkg.fail_reason.is_empty();

        let build_targets = if buildable {
            let all_targets = [
                "clean",
                "checksum",
                "configure",
                "all",
                "stage-install",
                "stage-package-create",
                "create-usergroup",
            ];

            let mut sections = Vec::new();

            match pkg.fail_target {
                Some((fail_tgt, msg)) => {
                    let pass: Vec<&str> = all_targets
                        .iter()
                        .filter(|t| **t != fail_tgt)
                        .copied()
                        .collect();
                    if !pass.is_empty() {
                        sections.push(format!("\n{targets}:\n\t@true", targets = pass.join(" "),));
                    }
                    sections.push(format!(
                        "\n{fail_tgt}:\n\
                         \t@echo '{msg}' >&2; exit 1",
                    ));
                }
                None => {
                    let pass: Vec<&str> = all_targets
                        .iter()
                        .filter(|t| **t != "stage-package-create")
                        .copied()
                        .collect();
                    sections.push(format!("\n{targets}:\n\t@true", targets = pass.join(" "),));
                    sections.push(format!(
                        "\nstage-package-create:\n\
                         \t@mkdir -p ${{.CURDIR}}/pkg\n\
                         \t@d=$$(mktemp -d) && \\\n\
                         \tprintf '@name {pkgname}\\n' > \"$$d/+CONTENTS\" && \\\n\
                         \tprintf 'Test package\\n' > \"$$d/+COMMENT\" && \\\n\
                         \tprintf 'Test package description\\n' > \"$$d/+DESC\" && \\\n\
                         \tprintf '0\\n' > \"$$d/+SIZE_PKG\" && \\\n\
                         \tprintf 'BUILD_DATE=%s\\nCATEGORIES=test\\nMACHINE_ARCH=x86_64\\nOPSYS=Test\\nOS_VERSION=1.0\\nPKGPATH={pkgpath}\\nPKGTOOLS_VERSION=20210710\\n' \
                         \"$$(date '+%Y-%m-%d %H:%M:%S %z')\" > \"$$d/+BUILD_INFO\" && \\\n\
                         \t(cd \"$$d\" && COPYFILE_DISABLE=1 tar czf \"${{.CURDIR}}/pkg/{pkgname}.tgz\" \
                         +CONTENTS +COMMENT +DESC +SIZE_PKG +BUILD_INFO) && \\\n\
                         \trm -rf \"$$d\"",
                        pkgname = pkg.pkgname,
                        pkgpath = pkg.pkgpath,
                    ));
                }
            }

            sections.push(format!(
                "\nshow-var:\n\
                 \t@case \"${{VARNAME}}\" in \
                 STAGE_PKGFILE) echo \
                 \"${{.CURDIR}}/pkg/{pkgname}.tgz\" ;; esac",
                pkgname = pkg.pkgname,
            ));

            sections.push(
                "\nshow-vars:\n\
                 \t@for v in ${VARNAMES}; do \\\n\
                 \t  case \"$$v\" in \\\n\
                 \t    WRKDIR) echo \"${.CURDIR}/work\" ;; \\\n\
                 \t    *) echo ;; \\\n\
                 \t  esac; \\\n\
                 \tdone"
                    .to_string(),
            );

            sections.join("\n")
        } else {
            String::new()
        };

        let content = format!(
            "PKGNAME={pkgname}\n\
             \n\
             pbulk-index:\n\
             \t@printf '{index_body}'\n\
             {build_targets}\n",
            pkgname = pkg.pkgname,
        );
        let pkgdir = self.pkgsrc().join(pkg.pkgpath);
        fs::write(pkgdir.join("Makefile"), content)?;
        Ok(())
    }

    /// Write the Lua config file.
    fn write_config_lua(&self) -> Result<()> {
        let content = format!(
            "\
options = {{
    build_threads = 2,
    dbdir = \"{dbdir}\",
    scan_threads = 2,
    tui = false,
}}
pkgsrc = {{
    basedir = \"{pkgsrc}\",
    make = \"{make}\",
}}
",
            dbdir = self.dbdir().display(),
            pkgsrc = self.pkgsrc().display(),
            make = self.make.display(),
        );
        fs::write(self.config_path(), content)?;
        Ok(())
    }

    /// Load the config.
    fn load_config(&self) -> Result<Config> {
        let path = self.config_path();
        Config::load(Some(&path))
    }

    /// Open the database.
    fn open_db(&self) -> Result<Database> {
        Database::open(&self.dbdir())
    }

    /// Create a RunState with no shutdown.
    fn run_state(&self) -> RunState {
        RunState::new()
    }

    /// Create a PkgsrcEnv from our known paths.
    fn pkgsrc_env(&self) -> PkgsrcEnv {
        PkgsrcEnv {
            packages: self.packages_dir(),
            pkgtools: self.root.join("pkgtools"),
            prefix: self.root.join("prefix"),
            pkg_dbdir: self.root.join("pkg_dbdir"),
            pkg_refcount_dbdir: self.root.join("pkg_refcount_dbdir"),
            metadata: std::collections::HashMap::new(),
            cachevars: std::collections::HashMap::new(),
        }
    }

    /// Run a full scan and return the summary.
    fn run_scan(&self) -> Result<ScanSummary> {
        let config = self.load_config()?;
        let db = self.open_db()?;
        let sandbox = Sandbox::new(&config);
        let mut scope = SandboxScope::new(sandbox, self.run_state());

        let mut scan = Scan::new(&config);
        scan.init_from_db(&db)?;
        scan.start(&db, &mut scope)?;
        scan.resolve_with_report(&db, false)
    }
}

#[test]
fn test_full_tree_scan() -> Result<()> {
    let h = TestHarness::new()?;
    let result = h.run_scan()?;

    let c = result.counts();

    // 17 buildable: base, mid, also-base, top, multi, py27-dual, py314-dual,
    //   build-fail, dep-bfail,
    //   fail-checksum, fail-at-build, fail-install, fail-package,
    //   chain-a, chain-b, chain-c, chain-d
    assert_eq!(
        c.buildable,
        17,
        "expected 17 buildable, got {} (total packages: {})",
        c.buildable,
        result.packages.len()
    );

    // 1 pkg_skip: skip-me
    assert_eq!(c.states[PreSkipped], 1, "expected 1 pkg_skip");

    // 1 pkg_fail: fail-me
    assert_eq!(c.states[PreFailed], 1, "expected 1 pkg_fail");

    // 1 indirect_preskip: dep-skip (depends on skip-me)
    assert_eq!(
        c.states[IndirectPreSkipped], 1,
        "expected 1 indirect_preskip"
    );

    // 1 indirect_prefail: dep-fail (depends on fail-me)
    assert_eq!(
        c.states[IndirectPreFailed], 1,
        "expected 1 indirect_prefail"
    );

    // 1 unresolved: bad-dep (depends on nonexistent)
    assert_eq!(c.states[Unresolved], 1, "expected 1 unresolved");

    // Total should be 22
    assert_eq!(result.packages.len(), 22, "expected 22 total packages");

    // Verify specific skip reasons
    for pkg in &result.packages {
        match pkg.pkgpath().as_path().to_string_lossy().as_ref() {
            "test/skip-me" => {
                if let bob::ScanResult::Skipped { state, .. } = pkg {
                    assert!(
                        matches!(state, PackageState::PreSkipped(_)),
                        "skip-me should be PreSkipped, got {:?}",
                        state
                    );
                } else {
                    panic!("skip-me should be Skipped");
                }
            }
            "test/dep-skip" => {
                if let bob::ScanResult::Skipped { state, .. } = pkg {
                    assert!(
                        matches!(state, PackageState::IndirectPreSkipped(_)),
                        "dep-skip should be IndirectPreSkipped, got {:?}",
                        state
                    );
                } else {
                    panic!("dep-skip should be Skipped");
                }
            }
            "test/fail-me" => {
                if let bob::ScanResult::Skipped { state, .. } = pkg {
                    assert!(
                        matches!(state, PackageState::PreFailed(_)),
                        "fail-me should be PreFailed, got {:?}",
                        state
                    );
                } else {
                    panic!("fail-me should be Skipped");
                }
            }
            "test/dep-fail" => {
                if let bob::ScanResult::Skipped { state, .. } = pkg {
                    assert!(
                        matches!(state, PackageState::IndirectPreFailed(_)),
                        "dep-fail should be IndirectPreFailed, got {:?}",
                        state
                    );
                } else {
                    panic!("dep-fail should be Skipped");
                }
            }
            "test/bad-dep" => {
                if let bob::ScanResult::Skipped { state, .. } = pkg {
                    assert!(
                        matches!(state, PackageState::Unresolved(_)),
                        "bad-dep should be Unresolved, got {:?}",
                        state
                    );
                } else {
                    panic!("bad-dep should be Skipped");
                }
            }
            _ => {}
        }
    }

    Ok(())
}

#[test]
fn test_limited_scan() -> Result<()> {
    let h = TestHarness::new()?;
    let config = h.load_config()?;
    let db = h.open_db()?;
    let state = h.run_state();
    let sandbox = Sandbox::new(&config);
    let mut scope = SandboxScope::new(sandbox, state);

    let mut scan = Scan::new(&config);
    let top = pkgsrc::PkgPath::new("test/top")?;
    scan.add(&top);
    scan.init_from_db(&db)?;
    scan.start(&db, &mut scope)?;
    let result = scan.resolve_with_report(&db, false)?;

    // Limited scan from test/top should discover:
    // test/top -> test/mid, test/also-base -> test/base (transitively)
    // That's 4 packages total, all buildable.
    assert_eq!(
        result.packages.len(),
        4,
        "expected 4 packages from limited scan of test/top, got {}",
        result.packages.len()
    );

    let c = result.counts();
    assert_eq!(c.buildable, 4, "all 4 packages should be buildable");
    assert_eq!(c.states[PreSkipped], 0);
    assert_eq!(c.states[PreFailed], 0);
    assert_eq!(c.states[Unresolved], 0);

    // Verify the expected packages are present
    let pkgpaths: Vec<String> = result
        .packages
        .iter()
        .map(|p| p.pkgpath().as_path().to_string_lossy().to_string())
        .collect();
    assert!(pkgpaths.contains(&"test/top".to_string()));
    assert!(pkgpaths.contains(&"test/mid".to_string()));
    assert!(pkgpaths.contains(&"test/also-base".to_string()));
    assert!(pkgpaths.contains(&"test/base".to_string()));

    Ok(())
}

#[test]
fn test_full_build() -> Result<()> {
    let h = TestHarness::new()?;
    let config = h.load_config()?;
    let db = h.open_db()?;
    let state = h.run_state();

    // Run scan first
    let sandbox = Sandbox::new(&config);
    let mut scan_scope = SandboxScope::new(sandbox, state.clone());
    let mut scan = Scan::new(&config);
    scan.init_from_db(&db)?;
    scan.start(&db, &mut scan_scope)?;
    let scan_result = scan.resolve_with_report(&db, false)?;

    // Collect buildable packages for the build
    let scanpkgs = scan_result
        .buildable()
        .map(|p| (p.pkgname().clone(), p.clone()))
        .collect();

    // Fetch pkgsrc env (or use our known paths)
    let pkgsrc_env = h.pkgsrc_env();

    // Run build
    let build_sandbox = Sandbox::new(&config);
    let build_scope = SandboxScope::new(build_sandbox, state.clone());
    let build = Build::new(&config, pkgsrc_env, build_scope, scanpkgs);
    let build_result = build.start(&state, &db)?;

    let bc = build_result.counts();

    // 7 success: base, mid, also-base, top, multi, py27-dual, py314-dual
    assert_eq!(
        bc.states[Success], 7,
        "expected 7 successful builds, got {}",
        bc.states[Success]
    );

    // 6 failed: build-fail, fail-checksum, fail-at-build, fail-install,
    //   fail-package, chain-d
    assert_eq!(
        bc.states[Failed], 6,
        "expected 6 failed builds, got {}",
        bc.states[Failed]
    );

    // 4 skipped: dep-bfail, chain-c, chain-b, chain-a
    let indirect_failed = bc.states[IndirectFailed];
    assert_eq!(
        indirect_failed, 4,
        "expected 4 indirect-failed skips, got {}",
        indirect_failed
    );

    // Verify specific outcomes
    for r in &build_result.results {
        let name = r.pkgname.pkgname();
        match name {
            "base-1.0" | "mid-1.0" | "also-base-1.0" | "top-1.0" | "py313-multi-1.0" => {
                assert!(
                    matches!(r.state, PackageState::Success),
                    "{} should be Success, got {:?}",
                    name,
                    r.state
                );
            }
            "build-fail-1.0" | "fail-checksum-1.0" | "fail-at-build-1.0" | "fail-install-1.0"
            | "fail-package-1.0" | "chain-d-1.0" => {
                assert!(
                    matches!(r.state, PackageState::Failed(_)),
                    "{} should be Failed, got {:?}",
                    name,
                    r.state
                );
            }
            "dep-bfail-1.0" | "chain-c-1.0" | "chain-b-1.0" | "chain-a-1.0" => {
                assert!(
                    matches!(r.state, PackageState::IndirectFailed(_)),
                    "{} should be Skipped(IndirectFailed), got {:?}",
                    name,
                    r.state
                );
            }
            _ => {}
        }
    }

    // Verify .tgz files exist for successful packages
    let packages_all = h.packages_dir().join("All");
    for name in &[
        "base-1.0",
        "mid-1.0",
        "also-base-1.0",
        "top-1.0",
        "py313-multi-1.0",
    ] {
        let tgz = packages_all.join(format!("{}.tgz", name));
        assert!(tgz.exists(), "Package file {} should exist", tgz.display());
    }

    // Verify no .tgz files for failed packages
    for name in &[
        "build-fail-1.0",
        "fail-checksum-1.0",
        "fail-at-build-1.0",
        "fail-install-1.0",
        "fail-package-1.0",
        "chain-d-1.0",
    ] {
        let tgz = packages_all.join(format!("{}.tgz", name));
        assert!(
            !tgz.exists(),
            "Failed package {} should not exist",
            tgz.display()
        );
    }

    // Verify log directory exists for failed builds
    for name in &["build-fail-1.0", "fail-checksum-1.0", "fail-at-build-1.0"] {
        let fail_logdir = h.logdir().join(name);
        assert!(
            fail_logdir.exists(),
            "Log dir for {} should exist at {}",
            name,
            fail_logdir.display()
        );
    }

    Ok(())
}

#[test]
fn test_scan_database_caching() -> Result<()> {
    let h = TestHarness::new()?;

    // Run first scan
    let result1 = h.run_scan()?;
    assert_eq!(result1.packages.len(), 22);

    // Create a new scan and check database caching
    let config = h.load_config()?;
    let db = h.open_db()?;
    let mut scan2 = Scan::new(&config);
    let (cached_count, _pending) = scan2.init_from_db(&db)?;

    assert_eq!(
        cached_count, 21,
        "second scan should find 21 cached package paths, got {}",
        cached_count
    );

    // With full_scan_complete set, start() should skip re-scanning
    scan2.set_full_scan_complete();
    let state = h.run_state();
    let sandbox = Sandbox::new(&config);
    let mut scope = SandboxScope::new(sandbox, state);
    scan2.start(&db, &mut scope)?;

    // Resolve should still work with cached data
    let result2 = scan2.resolve_with_report(&db, false)?;
    assert_eq!(
        result2.packages.len(),
        22,
        "cached resolve should produce same result"
    );

    Ok(())
}

#[test]
fn test_build_bootstrap_skips_deinstall() -> Result<()> {
    let h = TestHarness::new()?;
    let config = h.load_config()?;
    let db = h.open_db()?;
    let state = h.run_state();

    // Run scan
    let sandbox = Sandbox::new(&config);
    let mut scan_scope = SandboxScope::new(sandbox, state.clone());
    let mut scan = Scan::new(&config);
    scan.init_from_db(&db)?;
    scan.start(&db, &mut scan_scope)?;
    let scan_result = scan.resolve_with_report(&db, false)?;

    // Only build test/base (bootstrap package)
    let scanpkgs = scan_result
        .buildable()
        .filter(|p| p.pkgpath.as_path().to_string_lossy() == "test/base")
        .map(|p| (p.pkgname().clone(), p.clone()))
        .collect();

    let pkgsrc_env = h.pkgsrc_env();
    let build_sandbox = Sandbox::new(&config);
    let build_scope = SandboxScope::new(build_sandbox, state.clone());
    let build = Build::new(&config, pkgsrc_env, build_scope, scanpkgs);
    let build_result = build.start(&state, &db)?;

    // base should succeed
    assert_eq!(build_result.counts().states[Success], 1);

    // Verify no deinstall log exists (bootstrap skips deinstall)
    let base_logdir = h.logdir().join("base-1.0");
    let deinstall_log = base_logdir.join("deinstall.log");
    assert!(
        !deinstall_log.exists(),
        "Bootstrap package should not have deinstall.log (log dir removed on success)"
    );

    Ok(())
}

#[test]
fn test_scan_resume() -> Result<()> {
    let h = TestHarness::new()?;
    let config = h.load_config()?;
    let db = h.open_db()?;

    // Run first scan with early shutdown
    let state = bob::RunState::new();
    let sandbox = Sandbox::new(&config);
    let mut scope = SandboxScope::new(sandbox, state.clone());

    let mut scan = Scan::new(&config);
    scan.init_from_db(&db)?;

    // Set shutdown flag after a short delay to interrupt the scan.
    // We use a thread to set it while the scan is running.
    let state_clone = state.clone();
    let _trigger = std::thread::spawn(move || {
        std::thread::sleep(std::time::Duration::from_millis(200));
        state_clone.shutdown();
    });

    // Start scan - may be interrupted
    let _ = scan.start(&db, &mut scope);

    // Check that we have some partial results in the DB
    let scanned = db.get_scanned_pkgpaths()?;
    // We should have at least some packages scanned (may be all if fast enough)
    // The key test is that resuming works correctly.

    // Now resume: create a new scan with a fresh context (no shutdown)
    let state2 = h.run_state();
    let sandbox2 = Sandbox::new(&config);
    let mut scope2 = SandboxScope::new(sandbox2, state2);

    let mut scan2 = Scan::new(&config);
    let (cached, _) = scan2.init_from_db(&db)?;

    // cached should match what was scanned in the first run
    assert_eq!(
        cached,
        scanned.len(),
        "cached count should match scanned pkgpaths"
    );

    // Complete the scan
    scan2.start(&db, &mut scope2)?;
    let result = scan2.resolve_with_report(&db, false)?;

    // Final result should have at least 22 packages.  The scanner may
    // also discover test/nonexistent (from bad-dep's dependency) and
    // record it as a ScanFail, giving 23.
    assert!(
        result.packages.len() >= 22,
        "resumed scan should produce at least 22 packages, got {}",
        result.packages.len()
    );

    let c = result.counts();
    assert_eq!(c.buildable, 17, "expected 17 buildable after resume");

    Ok(())
}

/// Run a full scan + build using the test harness, returning the DB, scan
/// summary, and build summary for further assertions.
fn run_scan_and_build(h: &TestHarness) -> Result<(Database, ScanSummary, bob::BuildSummary)> {
    let config = h.load_config()?;
    let db = h.open_db()?;
    let state = h.run_state();

    let sandbox = Sandbox::new(&config);
    let mut scan_scope = SandboxScope::new(sandbox, state.clone());
    let mut scan = Scan::new(&config);
    scan.init_from_db(&db)?;
    scan.start(&db, &mut scan_scope)?;
    let scan_result = scan.resolve_with_report(&db, false)?;

    let scanpkgs = scan_result
        .buildable()
        .map(|p| (p.pkgname().clone(), p.clone()))
        .collect();
    let pkgsrc_env = h.pkgsrc_env();
    let build_sandbox = Sandbox::new(&config);
    let build_scope = SandboxScope::new(build_sandbox, state.clone());
    let build = Build::new(&config, pkgsrc_env, build_scope, scanpkgs);
    let build_result = build.start(&state, &db)?;

    Ok((db, scan_result, build_result))
}

#[test]
fn test_cached_build_resume() -> Result<()> {
    let h = TestHarness::new()?;

    // First scan + build
    let (_, scan_result, _) = run_scan_and_build(&h)?;

    // Second build using the same DB (results are cached)
    let config = h.load_config()?;
    let db = h.open_db()?;
    let state = h.run_state();

    let scanpkgs = scan_result
        .buildable()
        .map(|p| (p.pkgname().clone(), p.clone()))
        .collect();
    let pkgsrc_env = h.pkgsrc_env();
    let build_sandbox = Sandbox::new(&config);
    let build_scope = SandboxScope::new(build_sandbox, state.clone());
    let mut build2 = Build::new(&config, pkgsrc_env, build_scope, scanpkgs);

    let cached = build2.load_cached_from_db(&db)?;
    assert_eq!(cached, 17, "expected 17 cached results, got {}", cached);

    let result2 = build2.start(&state, &db)?;
    assert!(
        result2.results.is_empty(),
        "second build should produce no new results (all cached), got {}",
        result2.results.len()
    );

    Ok(())
}

#[test]
fn test_build_results_in_db() -> Result<()> {
    let h = TestHarness::new()?;
    let (db, _, _) = run_scan_and_build(&h)?;

    // Verify successful package
    let base = db
        .get_package_by_name("base-1.0")?
        .expect("base-1.0 should exist");
    let base_result = db
        .get_build_result(base.id)?
        .expect("base-1.0 should have a build result");
    assert!(
        matches!(base_result.state, PackageState::Success),
        "base-1.0 should be Success, got {:?}",
        base_result.state
    );

    // Verify failed package
    let bf = db
        .get_package_by_name("build-fail-1.0")?
        .expect("build-fail-1.0 should exist");
    let bf_result = db
        .get_build_result(bf.id)?
        .expect("build-fail-1.0 should have a build result");
    assert!(
        matches!(bf_result.state, PackageState::Failed(_)),
        "build-fail-1.0 should be Failed, got {:?}",
        bf_result.state
    );

    // get_all_build_results
    let all = db.get_all_build_results()?;
    assert!(
        all.len() >= 14,
        "expected at least 14 build results, got {}",
        all.len()
    );

    // get_failed_packages
    let failed = db.get_failed_packages()?;
    assert!(
        failed.contains(&"build-fail-1.0".to_string()),
        "failed list should contain build-fail-1.0"
    );

    // delete_build_by_name
    assert!(db.delete_build_by_name("base-1.0")?);
    assert!(!db.delete_build_by_name("base-1.0")?);

    // delete_build_by_pkgpath
    let del_count = db.delete_build_by_pkgpath("test/build-fail")?;
    assert_eq!(del_count, 1, "expected 1 deleted, got {}", del_count);

    // clear_builds
    let cleared = db.clear_builds()?;
    assert!(cleared >= 1, "expected at least 1 cleared, got {}", cleared);

    // After clearing, no results should remain
    let remaining = db.get_all_build_results()?;
    assert!(remaining.is_empty(), "expected 0 after clear_builds");

    Ok(())
}

#[test]
fn test_strict_scan() -> Result<()> {
    let h = TestHarness::new()?;
    let config = h.load_config()?;
    let db = h.open_db()?;
    let state = h.run_state();
    let sandbox = Sandbox::new(&config);
    let mut scope = SandboxScope::new(sandbox, state);

    let mut scan = Scan::new(&config);
    scan.init_from_db(&db)?;
    scan.start(&db, &mut scope)?;

    let result = scan.resolve_with_report(&db, true);
    assert!(
        result.is_err(),
        "strict scan should fail with unresolved deps"
    );
    let err_msg = format!("{}", result.expect_err("expected error"));
    assert!(
        err_msg.contains("strict_scan"),
        "error should mention strict_scan, got: {}",
        err_msg
    );

    Ok(())
}

#[test]
fn test_pkgsrc_env_persistence() -> Result<()> {
    let h = TestHarness::new()?;
    let db = h.open_db()?;

    // full_scan_complete lifecycle
    assert!(!db.full_scan_complete());
    db.set_full_scan_complete()?;
    assert!(db.full_scan_complete());
    db.clear_full_scan_complete()?;
    assert!(!db.full_scan_complete());

    // Store and load PkgsrcEnv
    let env = h.pkgsrc_env();
    db.store_pkgsrc_env(&env)?;
    let loaded = db.load_pkgsrc_env()?;
    assert_eq!(loaded.packages, env.packages);
    assert_eq!(loaded.pkgtools, env.pkgtools);
    assert_eq!(loaded.prefix, env.prefix);
    assert_eq!(loaded.pkg_dbdir, env.pkg_dbdir);
    assert_eq!(loaded.pkg_refcount_dbdir, env.pkg_refcount_dbdir);

    // Storing again should fail (INSERT, not REPLACE)
    let dup = db.store_pkgsrc_env(&env);
    assert!(dup.is_err(), "duplicate store_pkgsrc_env should fail");

    Ok(())
}

#[test]
fn test_config_options_and_environment() -> Result<()> {
    let h = TestHarness::new()?;

    let content = format!(
        "\
options = {{
    build_threads = 4,
    dbdir = \"{dbdir}\",
    scan_threads = 3,
    strict_scan = true,
}}
pkgsrc = {{
    basedir = \"{pkgsrc}\",
    make = \"{make}\",
    cachevars = {{ \"NATIVE_OPSYS\" }},
    save_wrkdir_patterns = {{ \"**/config.log\" }},
}}
sandboxes = {{
    basedir = \"{dbdir}/sandboxes\",
    environment = {{
        build = {{
            clear = true,
            inherit = {{ \"TERM\", \"HOME\" }},
            vars = {{ PATH = \"/sbin:/bin\", LC_ALL = \"C\" }},
        }},
        dev = {{
            clear = true,
            inherit = {{ \"TERM\", \"HOME\" }},
            vars = {{ BINPKG_SITES = \"$bob_packages\" }},
        }},
    }},
}}
",
        dbdir = h.dbdir().display(),
        pkgsrc = h.pkgsrc().display(),
        make = h.make.display(),
    );
    fs::write(h.config_path(), content)?;

    let config = h.load_config()?;
    assert_eq!(config.build_threads(), 4);
    assert_eq!(config.scan_threads(), 3);
    assert!(config.strict_scan());
    assert_eq!(config.cachevars(), &["NATIVE_OPSYS"]);
    assert_eq!(config.save_wrkdir_patterns(), &["**/config.log"]);

    let env = config
        .environment()
        .expect("environment section should exist");
    let build = env.build.as_ref().expect("environment.build should exist");
    assert!(build.clear);
    assert_eq!(build.inherit, vec!["TERM", "HOME"]);
    let mut expected_build_vars = HashMap::new();
    expected_build_vars.insert("PATH".to_string(), "/sbin:/bin".to_string());
    expected_build_vars.insert("LC_ALL".to_string(), "C".to_string());
    assert_eq!(build.vars, expected_build_vars);
    let dev = env.dev.as_ref().expect("environment.dev should exist");
    assert!(dev.clear);
    assert_eq!(dev.inherit, vec!["TERM", "HOME"]);
    let mut expected_dev_vars = HashMap::new();
    expected_dev_vars.insert("BINPKG_SITES".to_string(), "$bob_packages".to_string());
    assert_eq!(dev.vars, expected_dev_vars);

    Ok(())
}

#[test]
fn test_config_validation() -> Result<()> {
    let h = TestHarness::new()?;

    // Invalid make path
    let content = format!(
        "\
pkgsrc = {{
    basedir = \"{pkgsrc}\",
    make = \"/nonexistent/bmake\",
}}
",
        pkgsrc = h.pkgsrc().display(),
    );
    fs::write(h.config_path(), &content)?;
    let config = h.load_config()?;
    let result = config.validate();
    assert!(result.is_err(), "validate should fail with bad make path");
    let errors = result.expect_err("expected validation errors");
    assert!(
        errors.iter().any(|e| e.contains("make")),
        "errors should mention make: {:?}",
        errors
    );

    // Invalid dbdir parent
    let content = format!(
        "\
options = {{
    dbdir = \"/nonexistent/parent/db\",
}}
pkgsrc = {{
    basedir = \"{pkgsrc}\",
    make = \"{make}\",
}}
",
        pkgsrc = h.pkgsrc().display(),
        make = h.make.display(),
    );
    fs::write(h.config_path(), &content)?;
    let config = h.load_config()?;
    let result = config.validate();
    assert!(result.is_err(), "validate should fail with bad dbdir");
    let errors = result.expect_err("expected validation errors");
    assert!(
        errors.iter().any(|e| e.contains("dbdir")),
        "errors should mention dbdir: {:?}",
        errors
    );

    Ok(())
}

#[test]
fn test_scan_failure_handling() -> Result<()> {
    let h = TestHarness::new()?;

    // Create a package whose pbulk-index target fails
    let scan_fail_dir = h.pkgsrc().join("test/scan-fail");
    fs::create_dir_all(&scan_fail_dir)?;
    let makefile = "\
PKGNAME=scan-fail-1.0

pbulk-index:
\t@echo 'fatal error' >&2; exit 1
";
    fs::write(scan_fail_dir.join("Makefile"), makefile)?;

    // Update category Makefile to include scan-fail
    let cat_content = "\
show-subdir-var:
\t@echo \"base mid also-base top multi dual skip-me dep-skip fail-me dep-fail bad-dep build-fail dep-bfail fail-checksum fail-at-build fail-install fail-package chain-a chain-b chain-c chain-d scan-fail\"
";
    fs::write(h.pkgsrc().join("test/Makefile"), cat_content)?;

    let result = h.run_scan()?;
    let c = result.counts();

    assert_eq!(c.scanfail, 1, "expected 1 scanfail, got {}", c.scanfail);
    assert_eq!(c.buildable, 17, "buildable count should remain 17");

    let scan_fail_found = result.packages.iter().any(|p| {
        matches!(p, bob::ScanResult::ScanFail { .. })
            && p.pkgpath().as_path().to_string_lossy() == "test/scan-fail"
    });
    assert!(scan_fail_found, "scan-fail should appear as ScanFail");

    Ok(())
}

/// Verify that each build phase can independently fail and produce the
/// correct PackageState::Failed result.
#[test]
fn test_build_failure_at_each_phase() -> Result<()> {
    let h = TestHarness::new()?;
    let (_, _, build_result) = run_scan_and_build(&h)?;

    let outcomes: HashMap<&str, &PackageState> = build_result
        .results
        .iter()
        .map(|r| (r.pkgname.pkgname(), &r.state))
        .collect();

    // Each fail-* package should have Failed outcome
    let expected_failures = [
        "fail-checksum-1.0",
        "fail-at-build-1.0",
        "fail-install-1.0",
        "fail-package-1.0",
    ];
    for name in &expected_failures {
        let outcome = outcomes
            .get(name)
            .unwrap_or_else(|| panic!("{} should have a build result", name));
        assert!(
            matches!(outcome, PackageState::Failed(_)),
            "{} should be Failed, got {:?}",
            name,
            outcome
        );
    }

    Ok(())
}

/// Verify build log files: successful builds clean up their log dirs,
/// failed builds leave logs for the phases that ran.
#[test]
fn test_build_logs() -> Result<()> {
    let h = TestHarness::new()?;
    run_scan_and_build(&h)?;

    // Successful packages should not have log directories
    for name in &["base-1.0", "mid-1.0", "top-1.0"] {
        let logdir = h.logdir().join(name);
        assert!(
            !logdir.exists(),
            "Successful build {} should not have log dir at {}",
            name,
            logdir.display()
        );
    }

    // build-fail fails at configure: should have pre-clean.log, configure.log
    let bf_log = h.logdir().join("build-fail-1.0");
    assert!(bf_log.exists(), "build-fail log dir should exist");
    assert!(
        bf_log.join("pre-clean.log").exists(),
        "build-fail should have pre-clean.log"
    );
    assert!(
        bf_log.join("configure.log").exists(),
        "build-fail should have configure.log"
    );
    // build.log should not exist (never reached)
    assert!(
        !bf_log.join("build.log").exists(),
        "build-fail should not have build.log (configure failed first)"
    );

    // fail-checksum fails at checksum: should have pre-clean.log, checksum.log
    let fc_log = h.logdir().join("fail-checksum-1.0");
    assert!(fc_log.exists(), "fail-checksum log dir should exist");
    assert!(
        fc_log.join("pre-clean.log").exists(),
        "fail-checksum should have pre-clean.log"
    );
    assert!(
        fc_log.join("checksum.log").exists(),
        "fail-checksum should have checksum.log"
    );
    assert!(
        !fc_log.join("configure.log").exists(),
        "fail-checksum should not have configure.log"
    );

    // fail-at-build fails at build (all): should have configure.log + build.log
    let fab_log = h.logdir().join("fail-at-build-1.0");
    assert!(fab_log.exists(), "fail-at-build log dir should exist");
    assert!(
        fab_log.join("configure.log").exists(),
        "fail-at-build should have configure.log"
    );
    assert!(
        fab_log.join("build.log").exists(),
        "fail-at-build should have build.log"
    );
    assert!(
        !fab_log.join("install.log").exists(),
        "fail-at-build should not have install.log"
    );

    // fail-install: should have build.log + install.log
    let fi_log = h.logdir().join("fail-install-1.0");
    assert!(fi_log.exists(), "fail-install log dir should exist");
    assert!(
        fi_log.join("build.log").exists(),
        "fail-install should have build.log"
    );
    assert!(
        fi_log.join("install.log").exists(),
        "fail-install should have install.log"
    );

    // fail-package: should have install.log + package.log
    let fp_log = h.logdir().join("fail-package-1.0");
    assert!(fp_log.exists(), "fail-package log dir should exist");
    assert!(
        fp_log.join("install.log").exists(),
        "fail-package should have install.log"
    );
    assert!(
        fp_log.join("package.log").exists(),
        "fail-package should have package.log"
    );

    Ok(())
}

/// Verify that the 4-level dependency chain cascades failure correctly:
/// chain-d fails → chain-c, chain-b, chain-a all IndirectFailed.
#[test]
fn test_cascading_failure_chain() -> Result<()> {
    let h = TestHarness::new()?;
    let (_, _, build_result) = run_scan_and_build(&h)?;

    let outcomes: HashMap<&str, &PackageState> = build_result
        .results
        .iter()
        .map(|r| (r.pkgname.pkgname(), &r.state))
        .collect();

    // chain-d: direct failure
    let chain_d = outcomes.get("chain-d-1.0").expect("chain-d should exist");
    assert!(
        matches!(chain_d, PackageState::Failed(_)),
        "chain-d should be Failed, got {:?}",
        chain_d
    );

    // chain-c, chain-b, chain-a: indirect failure from chain-d
    for name in &["chain-c-1.0", "chain-b-1.0", "chain-a-1.0"] {
        let outcome = outcomes
            .get(name)
            .unwrap_or_else(|| panic!("{} should exist", name));
        assert!(
            matches!(outcome, PackageState::IndirectFailed(_)),
            "{} should be IndirectFailed, got {:?}",
            name,
            outcome
        );
        // Verify the reason mentions chain-d
        if let PackageState::IndirectFailed(msg) = outcome {
            assert!(
                msg.contains("chain-d"),
                "{} IndirectFailed reason should mention chain-d, got: {}",
                name,
                msg
            );
        }
    }

    Ok(())
}

/// Verify DAG scheduling: dependencies always build before dependents.
#[test]
fn test_build_order() -> Result<()> {
    let h = TestHarness::new()?;
    let (_, _, build_result) = run_scan_and_build(&h)?;

    // Build position map: earlier index = completed earlier
    let positions: HashMap<&str, usize> = build_result
        .results
        .iter()
        .enumerate()
        .map(|(i, r)| (r.pkgname.pkgname(), i))
        .collect();

    // base must complete before mid and also-base
    if let (Some(&base), Some(&mid)) = (positions.get("base-1.0"), positions.get("mid-1.0")) {
        assert!(
            base < mid,
            "base ({}) should complete before mid ({})",
            base,
            mid
        );
    }
    if let (Some(&base), Some(&also)) = (positions.get("base-1.0"), positions.get("also-base-1.0"))
    {
        assert!(
            base < also,
            "base ({}) should complete before also-base ({})",
            base,
            also
        );
    }

    // mid and also-base must complete before top
    if let (Some(&mid), Some(&top)) = (positions.get("mid-1.0"), positions.get("top-1.0")) {
        assert!(
            mid < top,
            "mid ({}) should complete before top ({})",
            mid,
            top
        );
    }
    if let (Some(&also), Some(&top)) = (positions.get("also-base-1.0"), positions.get("top-1.0")) {
        assert!(
            also < top,
            "also-base ({}) should complete before top ({})",
            also,
            top
        );
    }

    Ok(())
}

/// Verify building a limited subset: scan only test/top and its transitive
/// deps, then build only those packages.
#[test]
fn test_limited_build() -> Result<()> {
    let h = TestHarness::new()?;
    let config = h.load_config()?;
    let db = h.open_db()?;
    let state = h.run_state();

    // Limited scan from test/top
    let sandbox = Sandbox::new(&config);
    let mut scan_scope = SandboxScope::new(sandbox, state.clone());
    let mut scan = Scan::new(&config);
    let top = pkgsrc::PkgPath::new("test/top")?;
    scan.add(&top);
    scan.init_from_db(&db)?;
    scan.start(&db, &mut scan_scope)?;
    let scan_result = scan.resolve_with_report(&db, false)?;

    // Should have 4 packages: top, mid, also-base, base
    assert_eq!(scan_result.packages.len(), 4);
    let c = scan_result.counts();
    assert_eq!(c.buildable, 4);

    // Build them
    let scanpkgs = scan_result
        .buildable()
        .map(|p| (p.pkgname().clone(), p.clone()))
        .collect();
    let pkgsrc_env = h.pkgsrc_env();
    let build_sandbox = Sandbox::new(&config);
    let build_scope = SandboxScope::new(build_sandbox, state.clone());
    let build = Build::new(&config, pkgsrc_env, build_scope, scanpkgs);
    let build_result = build.start(&state, &db)?;

    // All 4 should succeed
    let bc = build_result.counts();
    assert_eq!(
        bc.states[Success], 4,
        "expected 4 successful builds, got {}",
        bc.states[Success]
    );
    assert_eq!(
        bc.states[Failed], 0,
        "expected 0 failures, got {}",
        bc.states[Failed]
    );

    // Verify the expected packages built
    let built: Vec<&str> = build_result
        .results
        .iter()
        .map(|r| r.pkgname.pkgname())
        .collect();
    assert!(built.contains(&"base-1.0"));
    assert!(built.contains(&"mid-1.0"));
    assert!(built.contains(&"also-base-1.0"));
    assert!(built.contains(&"top-1.0"));

    // No other packages should have been attempted
    assert_eq!(
        build_result.results.len(),
        4,
        "only 4 packages should have build results"
    );

    Ok(())
}

/// Verify pkg_up_to_date correctly identifies missing packages.
#[test]
fn test_pkg_up_to_date_not_found() -> Result<()> {
    let h = TestHarness::new()?;

    let result = bob::pkg_up_to_date(
        "nonexistent-1.0",
        &[],
        &h.packages_dir().join("All"),
        &h.pkgsrc(),
    )?;
    assert!(
        matches!(result, Some(bob::BuildReason::PackageNotFound)),
        "nonexistent package should be PackageNotFound, got {:?}",
        result
    );

    Ok(())
}

/// Verify that after a successful build, re-running the build with cached
/// results produces no new work, and all results show UpToDate or are
/// loaded from cache.
#[test]
fn test_build_resume_no_new_work() -> Result<()> {
    let h = TestHarness::new()?;

    // First full scan + build
    let (_, scan_result, first_build) = run_scan_and_build(&h)?;
    assert_eq!(first_build.counts().states[Success], 7);

    // Second build with same config - should produce no new results
    let config = h.load_config()?;
    let db = h.open_db()?;
    let state = h.run_state();
    let scanpkgs = scan_result
        .buildable()
        .map(|p| (p.pkgname().clone(), p.clone()))
        .collect();
    let pkgsrc_env = h.pkgsrc_env();
    let build_sandbox = Sandbox::new(&config);
    let build_scope = SandboxScope::new(build_sandbox, state.clone());
    let mut build = Build::new(&config, pkgsrc_env, build_scope, scanpkgs);

    let cached = build.load_cached_from_db(&db)?;
    assert_eq!(cached, 17, "all 17 buildable should be cached");

    let result = build.start(&state, &db)?;
    assert!(
        result.results.is_empty(),
        "second build should produce no new results, got {}",
        result.results.len()
    );

    Ok(())
}

/// Verify that build_threads=1 produces identical results to parallel
/// builds - a single-threaded build eliminates scheduling races.
#[test]
fn test_single_threaded_build() -> Result<()> {
    let h = TestHarness::new()?;

    // Override config with 1 build thread
    let content = format!(
        "\
options = {{
    build_threads = 1,
    dbdir = \"{dbdir}\",
    scan_threads = 1,
    tui = false,
}}
pkgsrc = {{
    basedir = \"{pkgsrc}\",
    make = \"{make}\",
}}
",
        dbdir = h.dbdir().display(),
        pkgsrc = h.pkgsrc().display(),
        make = h.make.display(),
    );
    fs::write(h.config_path(), content)?;

    let config = h.load_config()?;
    let db = h.open_db()?;
    let state = h.run_state();

    let sandbox = Sandbox::new(&config);
    let mut scan_scope = SandboxScope::new(sandbox, state.clone());
    let mut scan = Scan::new(&config);
    scan.init_from_db(&db)?;
    scan.start(&db, &mut scan_scope)?;
    let scan_result = scan.resolve_with_report(&db, false)?;

    let scanpkgs = scan_result
        .buildable()
        .map(|p| (p.pkgname().clone(), p.clone()))
        .collect();
    let pkgsrc_env = h.pkgsrc_env();
    let build_sandbox = Sandbox::new(&config);
    let build_scope = SandboxScope::new(build_sandbox, state.clone());
    let build = Build::new(&config, pkgsrc_env, build_scope, scanpkgs);
    let build_result = build.start(&state, &db)?;

    let bc = build_result.counts();
    assert_eq!(bc.states[Success], 7, "expected 7 successful builds");
    assert_eq!(bc.states[Failed], 6, "expected 6 failed builds");
    assert_eq!(bc.states[IndirectFailed], 4, "expected 4 indirect-failed");

    Ok(())
}

/// Verify that clearing build results and rebuilding works correctly.
#[test]
fn test_rebuild_after_clear() -> Result<()> {
    let h = TestHarness::new()?;

    // First build
    let (db, scan_result, first) = run_scan_and_build(&h)?;
    assert_eq!(first.counts().states[Success], 7);

    // Clear all build results
    let cleared = db.clear_builds()?;
    assert!(cleared > 0, "should clear some builds");

    // Rebuild - should produce new results
    let config = h.load_config()?;
    let state = h.run_state();
    let scanpkgs = scan_result
        .buildable()
        .map(|p| (p.pkgname().clone(), p.clone()))
        .collect();
    let pkgsrc_env = h.pkgsrc_env();
    let build_sandbox = Sandbox::new(&config);
    let build_scope = SandboxScope::new(build_sandbox, state.clone());
    let mut build = Build::new(&config, pkgsrc_env, build_scope, scanpkgs);

    let cached = build.load_cached_from_db(&db)?;
    assert_eq!(cached, 0, "no cached results after clear");

    let rebuild_result = build.start(&state, &db)?;
    assert_eq!(
        rebuild_result.counts().states[Success],
        7,
        "rebuild should succeed for same 7 packages"
    );

    Ok(())
}

/// Verify that building only failed packages after fixing them works.
/// Simulates a "rebuild failed" workflow by deleting a specific build
/// result and re-running.
#[test]
fn test_selective_rebuild_after_failure() -> Result<()> {
    let h = TestHarness::new()?;
    let (db, scan_result, first) = run_scan_and_build(&h)?;

    // Verify build-fail is in the failed list
    let failed = db.get_failed_packages()?;
    assert!(
        failed.contains(&"build-fail-1.0".to_string()),
        "build-fail should be in failed list"
    );

    // Delete build result for build-fail and dep-bfail
    assert!(db.delete_build_by_name("build-fail-1.0")?);
    assert!(db.delete_build_by_name("dep-bfail-1.0")?);

    // "Fix" the package by replacing its Makefile with a working one
    h.write_pkg_makefile(&PkgDef::new("test/build-fail", "build-fail-1.0"))?;

    // Rebuild with cached results (most will be cached)
    let config = h.load_config()?;
    let state = h.run_state();
    let scanpkgs = scan_result
        .buildable()
        .map(|p| (p.pkgname().clone(), p.clone()))
        .collect();
    let pkgsrc_env = h.pkgsrc_env();
    let build_sandbox = Sandbox::new(&config);
    let build_scope = SandboxScope::new(build_sandbox, state.clone());
    let mut build = Build::new(&config, pkgsrc_env, build_scope, scanpkgs);

    let cached = build.load_cached_from_db(&db)?;
    // Most should be cached, but build-fail and dep-bfail were deleted
    assert!(
        cached < first.results.len(),
        "should have fewer cached than total: cached={}, total={}",
        cached,
        first.results.len()
    );

    let result = build.start(&state, &db)?;

    // build-fail should now succeed
    let bf_result = result
        .results
        .iter()
        .find(|r| r.pkgname.pkgname() == "build-fail-1.0");
    assert!(
        bf_result.is_some(),
        "build-fail should have a new build result"
    );
    assert!(
        matches!(bf_result.map(|r| &r.state), Some(PackageState::Success)),
        "fixed build-fail should succeed, got {:?}",
        bf_result.map(|r| &r.state)
    );

    Ok(())
}

/// Verify that the multi_version package builds correctly and that its
/// PYTHON_VERSION_REQD flag is effectively passed through.
#[test]
fn test_multi_version_package() -> Result<()> {
    let h = TestHarness::new()?;
    let (_, _, build_result) = run_scan_and_build(&h)?;

    let multi = build_result
        .results
        .iter()
        .find(|r| r.pkgname.pkgname() == "py313-multi-1.0");
    assert!(multi.is_some(), "py313-multi should have a build result");
    assert!(
        matches!(multi.map(|r| &r.state), Some(PackageState::Success)),
        "py313-multi should succeed"
    );

    // Verify the package file exists
    let tgz = h.packages_dir().join("All").join("py313-multi-1.0.tgz");
    assert!(tgz.exists(), "py313-multi package should exist");

    Ok(())
}

/// Verify that a single pkgpath emitting multiple scan records is fully
/// resolved and that every variant reaches a persisted build outcome.
#[test]
fn test_multi_version_multiple_records_build_all_variants() -> Result<()> {
    let h = TestHarness::new()?;
    let config = h.load_config()?;
    let db = h.open_db()?;
    let state = h.run_state();
    let sandbox = Sandbox::new(&config);
    let mut scope = SandboxScope::new(sandbox, state.clone());

    let mut scan = Scan::new(&config);
    scan.init_from_db(&db)?;
    scan.start(&db, &mut scope)?;
    let scan_result = scan.resolve_with_report(&db, false)?;

    let dual_pkgs: Vec<_> = scan_result
        .buildable()
        .filter(|p| p.pkgpath.as_path().to_string_lossy() == "test/dual")
        .map(|p| p.pkgname().pkgname().to_string())
        .collect();
    assert_eq!(
        dual_pkgs,
        vec!["py27-dual-1.0".to_string(), "py314-dual-1.0".to_string()],
        "expected both dual variants in resolved buildable set"
    );

    let scanpkgs = scan_result
        .buildable()
        .map(|p| (p.pkgname().clone(), p.clone()))
        .collect();
    let pkgsrc_env = h.pkgsrc_env();
    let build_sandbox = Sandbox::new(&config);
    let build_scope = SandboxScope::new(build_sandbox, state.clone());
    let build = Build::new(&config, pkgsrc_env, build_scope, scanpkgs);
    let build_result = build.start(&state, &db)?;

    for pkgname in ["py27-dual-1.0", "py314-dual-1.0"] {
        let built = build_result
            .results
            .iter()
            .find(|r| r.pkgname.pkgname() == pkgname);
        assert!(built.is_some(), "{pkgname} should be in build results");
        assert!(
            matches!(built.map(|r| &r.state), Some(PackageState::Success)),
            "{pkgname} should build successfully"
        );
        assert!(
            h.packages_dir()
                .join("All")
                .join(format!("{pkgname}.tgz"))
                .exists(),
            "{pkgname} package file should exist"
        );
    }

    let all_status = db.get_all_package_status(true)?;
    let lingering_pending: Vec<_> = all_status
        .into_iter()
        .filter(|p| {
            p.pkgpath == "test/dual" && p.build_outcome.is_none() && p.build_reason.is_none()
        })
        .map(|p| p.pkgname)
        .collect();
    assert!(
        lingering_pending.is_empty(),
        "dual variants should not remain blank pending: {:?}",
        lingering_pending
    );

    Ok(())
}

/// Verify that a fresh scan against already-built multi-version package files
/// records them as up-to-date instead of leaving them as blank pending rows.
#[test]
fn test_cli_scan_marks_multi_version_variants_up_to_date() -> Result<()> {
    let h = TestHarness::new()?;
    run_scan_and_build(&h)?;

    let db_path = h.dbdir().join("bob.db");
    if db_path.exists() {
        fs::remove_file(&db_path)?;
    }

    let bob = env!("CARGO_BIN_EXE_bob");
    let scan = Command::new(bob)
        .arg("-c")
        .arg(h.config_path())
        .arg("scan")
        .output()?;
    assert!(
        scan.status.success(),
        "bob scan failed:\nstdout:\n{}\nstderr:\n{}",
        String::from_utf8_lossy(&scan.stdout),
        String::from_utf8_lossy(&scan.stderr)
    );

    let status = Command::new(bob)
        .arg("-c")
        .arg(h.config_path())
        .arg("status")
        .output()?;
    assert!(
        status.status.success(),
        "bob status failed:\nstdout:\n{}\nstderr:\n{}",
        String::from_utf8_lossy(&status.stdout),
        String::from_utf8_lossy(&status.stderr)
    );

    let db = h.open_db()?;
    for pkgname in ["py27-dual-1.0", "py314-dual-1.0"] {
        let pkg = db
            .get_package_by_name(pkgname)?
            .unwrap_or_else(|| panic!("{pkgname} missing from database"));
        let result = db
            .get_build_result(pkg.id)?
            .unwrap_or_else(|| panic!("{pkgname} missing build result"));
        assert!(
            matches!(result.state, PackageState::UpToDate),
            "{pkgname} should be up-to-date after cli scan, got {:?}",
            result.state
        );
    }

    let status_stdout = String::from_utf8_lossy(&status.stdout);
    assert!(
        !status_stdout.contains("py27-dual-1.0"),
        "py27-dual should not remain visible in status output:\n{status_stdout}"
    );
    assert!(
        !status_stdout.contains("py314-dual-1.0"),
        "py314-dual should not remain visible in status output:\n{status_stdout}"
    );

    Ok(())
}

/// Verify that cached scan rows outside the latest resolved package set do not
/// leak into status, scheduling, or rebuild queries.
#[test]
fn test_unselected_packages_are_hidden_from_status_queries() -> Result<()> {
    let h = TestHarness::new()?;
    let (db, _, _) = run_scan_and_build(&h)?;

    let extra: pkgsrc::ScanIndex = "\
PKGNAME=extra-1.0
PKG_LOCATION=test/extra
ALL_DEPENDS=
PKG_SKIP_REASON=
PKG_FAIL_REASON=
NO_BIN_ON_FTP=
RESTRICTED=
CATEGORIES=test
MAINTAINER=test@example.com
USE_DESTDIR=yes
BOOTSTRAP_PKG=
USERGROUP_PHASE=
SCAN_DEPENDS=
"
    .parse()?;
    db.store_package("test/extra", &extra)?;

    let status_rows = db.get_all_package_status(true)?;
    assert!(
        !status_rows.iter().any(|p| p.pkgname == "extra-1.0"),
        "unselected package should not appear in status rows"
    );

    let buildable = db.get_buildable_packages()?;
    assert!(
        !buildable.iter().any(|p| p.pkgname == "extra-1.0"),
        "unselected package should not appear in buildable rows"
    );

    let loaded = db.load_resolved_packages()?;
    assert!(
        !loaded.iter().any(|p| p.pkgname().pkgname() == "extra-1.0"),
        "unselected package should not appear in resolved rebuild set"
    );

    Ok(())
}

/// Verify the usergroup_phase package: mid has USERGROUP_PHASE=configure,
/// meaning create-usergroup should run before configure. Since our mock
/// create-usergroup succeeds, the build should complete normally.
#[test]
fn test_usergroup_phase_package() -> Result<()> {
    let h = TestHarness::new()?;
    let (_, scan_result, _) = run_scan_and_build(&h)?;

    // Verify mid's scan data includes USERGROUP_PHASE
    let mid = scan_result
        .buildable()
        .find(|p| p.pkgpath.as_path().to_string_lossy() == "test/mid");
    assert!(mid.is_some(), "mid should be in scan results");
    let mid = mid.expect("mid exists");
    assert_eq!(
        mid.usergroup_phase(),
        Some("configure"),
        "mid should have USERGROUP_PHASE=configure"
    );

    Ok(())
}

/// Verify that load_resolved_packages reconstructs the same buildable set
/// and dependencies as the original scan resolution.  This is the code path
/// used by `bob rebuild` — it must not re-resolve dependency patterns.
#[test]
fn test_load_resolved_packages_matches_scan() -> Result<()> {
    let h = TestHarness::new()?;
    let scan_result = h.run_scan()?;
    let db = h.open_db()?;

    let to_depset = |name: &str, deps: &[pkgsrc::PkgName]| -> (String, Vec<String>) {
        let mut v: Vec<String> = deps.iter().map(|d| d.pkgname().to_string()).collect();
        v.sort();
        (name.to_string(), v)
    };

    let expected: HashMap<String, Vec<String>> = scan_result
        .buildable()
        .map(|p| to_depset(p.pkgname().pkgname(), p.depends()))
        .collect();

    let loaded = db.load_resolved_packages()?;
    let actual: HashMap<String, Vec<String>> = loaded
        .iter()
        .map(|p| to_depset(p.pkgname().pkgname(), p.depends()))
        .collect();

    assert_eq!(expected, actual);

    Ok(())
}

/// Verify that after building, build durations are recorded and non-zero
/// for packages that actually built (not indirect-failed).
#[test]
fn test_build_durations() -> Result<()> {
    let h = TestHarness::new()?;
    let (_, _, build_result) = run_scan_and_build(&h)?;

    for r in &build_result.results {
        match &r.state {
            PackageState::Success | PackageState::Failed(_) => {
                // Direct builds should have non-zero duration
                // (though very fast builds might be sub-millisecond)
                // Just verify it's a valid Duration
                assert!(
                    r.build_stats.duration.as_nanos() > 0,
                    "{} should have non-zero duration",
                    r.pkgname.pkgname()
                );
            }
            PackageState::IndirectFailed(_) => {
                // Indirect failures have zero duration (never attempted)
                assert_eq!(
                    r.build_stats.duration,
                    std::time::Duration::ZERO,
                    "{} indirect failure should have zero duration",
                    r.pkgname.pkgname()
                );
            }
            _ => {}
        }
    }

    Ok(())
}

// ====== pkg_summary generation tests ======

/// Get mtime of pkg_summary.gz, if it exists.
fn pkg_summary_gz_mtime(h: &TestHarness) -> Option<std::time::SystemTime> {
    let path = h.packages_dir().join("All").join("pkg_summary.gz");
    fs::metadata(path).ok().and_then(|m| m.modified().ok())
}

/// Read and decode pkg_summary.gz, returning its content as a String.
fn read_pkg_summary_gz(h: &TestHarness) -> Result<String> {
    use flate2::read::GzDecoder;
    use std::io::Read;
    let path = h.packages_dir().join("All").join("pkg_summary.gz");
    let file =
        fs::File::open(&path).with_context(|| format!("Failed to open {}", path.display()))?;
    let mut decoder = GzDecoder::new(file);
    let mut content = String::new();
    decoder.read_to_string(&mut content)?;
    Ok(content)
}

/// Count the number of package entries in pkg_summary content.
/// Entries are separated by blank lines; each starts with PKGNAME=.
fn count_pkg_summary_entries(content: &str) -> usize {
    content
        .split("\n\n")
        .filter(|block| block.starts_with("PKGNAME=") || block.contains("\nPKGNAME="))
        .count()
}

/// Replicate the maybe_generate_pkg_summary condition from main.rs.
/// Returns true if pkg_summary was regenerated.
fn maybe_generate(db: &Database, prior: &[String], summary: &bob::BuildSummary) -> Result<bool> {
    let current = db.get_successful_packages()?;
    if prior != current || summary.counts().states[Success] > 0 {
        bob::generate_pkg_summary(db, 2)?;
        Ok(true)
    } else {
        Ok(false)
    }
}

/// Run a build loading cached results, simulating a rebuild or re-run.
fn run_cached_build(
    h: &TestHarness,
    db: &Database,
    scan_result: &ScanSummary,
) -> Result<bob::BuildSummary> {
    let config = h.load_config()?;
    let state = h.run_state();
    let scanpkgs = scan_result
        .buildable()
        .map(|p| (p.pkgname().clone(), p.clone()))
        .collect();
    let pkgsrc_env = h.pkgsrc_env();
    let sandbox = Sandbox::new(&config);
    let scope = SandboxScope::new(sandbox, state.clone());
    let mut build = Build::new(&config, pkgsrc_env, scope, scanpkgs);
    build.load_cached_from_db(db)?;
    build.start(&state, db)
}

/// After a first build with successful packages, pkg_summary should be
/// generated (prior was empty, current has packages).
#[test]
fn test_pkg_summary_generated_after_first_build() -> Result<()> {
    let h = TestHarness::new()?;
    let (db, _, summary) = run_scan_and_build(&h)?;

    let prior: Vec<String> = vec![];
    let generated = maybe_generate(&db, &prior, &summary)?;

    assert!(generated, "should generate after first build");
    assert!(h.packages_dir().join("All/pkg_summary.gz").exists());
    assert!(h.packages_dir().join("All/pkg_summary.zst").exists());

    let content = read_pkg_summary_gz(&h)?;
    assert!(!content.is_empty(), "pkg_summary.gz should have content");

    let successful = db.get_successful_packages()?;
    let entry_count = count_pkg_summary_entries(&content);
    assert_eq!(
        entry_count,
        successful.len(),
        "pkg_summary should have one entry per successful package"
    );

    for pkgname in &successful {
        assert!(
            content.contains(&format!("PKGNAME={}", pkgname)),
            "pkg_summary should contain entry for {}",
            pkgname
        );
    }

    Ok(())
}

/// When all packages are up-to-date (second run, all cached), pkg_summary
/// should NOT be regenerated.
#[test]
fn test_pkg_summary_skipped_when_up_to_date() -> Result<()> {
    let h = TestHarness::new()?;
    let (db, scan_result, _) = run_scan_and_build(&h)?;

    bob::generate_pkg_summary(&db, 2)?;
    let mtime_before = pkg_summary_gz_mtime(&h).expect("pkg_summary.gz should exist");
    std::thread::sleep(std::time::Duration::from_millis(50));

    let prior = db.get_successful_packages()?;
    let summary = run_cached_build(&h, &db, &scan_result)?;
    let generated = maybe_generate(&db, &prior, &summary)?;

    assert!(!generated, "should not regenerate when all up-to-date");
    assert_eq!(
        pkg_summary_gz_mtime(&h).expect("should exist"),
        mtime_before,
        "pkg_summary.gz mtime should be unchanged"
    );

    Ok(())
}

/// Rebuilding a failed package that fails again should NOT regenerate
/// pkg_summary -- the set of successful packages is unchanged.
#[test]
fn test_pkg_summary_skipped_when_rebuild_fails_again() -> Result<()> {
    let h = TestHarness::new()?;
    let (db, scan_result, _) = run_scan_and_build(&h)?;

    bob::generate_pkg_summary(&db, 2)?;
    let mtime_before = pkg_summary_gz_mtime(&h).expect("pkg_summary.gz should exist");
    std::thread::sleep(std::time::Duration::from_millis(50));

    let prior = db.get_successful_packages()?;
    db.delete_build_by_name("build-fail-1.0")?;

    let summary = run_cached_build(&h, &db, &scan_result)?;
    let generated = maybe_generate(&db, &prior, &summary)?;

    assert!(
        !generated,
        "should not regenerate when failed rebuild fails again"
    );
    assert_eq!(
        pkg_summary_gz_mtime(&h).expect("should exist"),
        mtime_before,
    );

    Ok(())
}

/// Rebuilding a failed package that now succeeds should regenerate
/// pkg_summary -- a new package joined the successful set.
#[test]
fn test_pkg_summary_regenerated_when_rebuild_succeeds() -> Result<()> {
    let h = TestHarness::new()?;
    let (db, scan_result, _) = run_scan_and_build(&h)?;

    bob::generate_pkg_summary(&db, 2)?;
    let mtime_before = pkg_summary_gz_mtime(&h).expect("pkg_summary.gz should exist");
    std::thread::sleep(std::time::Duration::from_millis(50));

    // Fix build-fail by replacing its Makefile with a working version.
    h.write_pkg_makefile(&PkgDef::new("test/build-fail", "build-fail-1.0"))?;

    let prior = db.get_successful_packages()?;
    assert!(
        !prior.contains(&"build-fail-1.0".to_string()),
        "build-fail should not be in successful set before fix"
    );
    db.delete_build_by_name("build-fail-1.0")?;
    db.delete_build_by_name("dep-bfail-1.0")?;

    let summary = run_cached_build(&h, &db, &scan_result)?;
    let generated = maybe_generate(&db, &prior, &summary)?;

    assert!(
        generated,
        "should regenerate when failed package now succeeds"
    );

    let current = db.get_successful_packages()?;
    assert!(current.contains(&"build-fail-1.0".to_string()));

    let mtime_after = pkg_summary_gz_mtime(&h).expect("should exist");
    assert!(
        mtime_after > mtime_before,
        "pkg_summary.gz should have newer mtime than before rebuild"
    );

    let content = read_pkg_summary_gz(&h)?;
    assert!(
        content.contains("PKGNAME=build-fail-1.0"),
        "pkg_summary should now include the fixed package"
    );
    let entry_count = count_pkg_summary_entries(&content);
    assert_eq!(
        entry_count,
        current.len(),
        "pkg_summary should have one entry per successful package"
    );

    // pkg_summary should be at least as recent as unchanged package files
    let unchanged_tgz = h.packages_dir().join("All").join("base-1.0.tgz");
    if unchanged_tgz.exists() {
        let tgz_mtime = fs::metadata(&unchanged_tgz)?.modified()?;
        assert!(
            mtime_after >= tgz_mtime,
            "pkg_summary.gz should be at least as recent as unchanged .tgz files"
        );
    }

    Ok(())
}

/// Rebuilding a successful package that now fails should regenerate
/// pkg_summary -- the package left the successful set.
#[test]
fn test_pkg_summary_regenerated_when_successful_pkg_fails() -> Result<()> {
    let h = TestHarness::new()?;
    let (db, scan_result, _) = run_scan_and_build(&h)?;

    bob::generate_pkg_summary(&db, 2)?;
    let mtime_before = pkg_summary_gz_mtime(&h).expect("pkg_summary.gz should exist");
    std::thread::sleep(std::time::Duration::from_millis(50));

    // Break top-1.0 (nothing depends on it, so no cascade concerns).
    h.write_pkg_makefile(
        &PkgDef::new("test/top", "top-1.0")
            .depends("mid-[0-9]*:test/mid also-base-[0-9]*:test/also-base")
            .fail_at("configure", "broken"),
    )?;

    let prior = db.get_successful_packages()?;
    assert!(prior.contains(&"top-1.0".to_string()));
    db.delete_build_by_name("top-1.0")?;

    let summary = run_cached_build(&h, &db, &scan_result)?;

    let current = db.get_successful_packages()?;
    assert!(
        !current.contains(&"top-1.0".to_string()),
        "top-1.0 should no longer be in successful set"
    );

    let generated = maybe_generate(&db, &prior, &summary)?;

    assert!(
        generated,
        "should regenerate when successful package now fails"
    );
    assert!(
        pkg_summary_gz_mtime(&h).expect("should exist") > mtime_before,
        "pkg_summary.gz should have newer mtime"
    );

    let content = read_pkg_summary_gz(&h)?;
    assert!(
        !content.contains("PKGNAME=top-1.0"),
        "pkg_summary should no longer include the now-failed package"
    );
    assert_eq!(
        count_pkg_summary_entries(&content),
        current.len(),
        "pkg_summary entry count should match successful package count"
    );

    Ok(())
}

/// Rebuilding a successful package that succeeds again (same pkgname, new
/// binary) should regenerate pkg_summary -- metadata may have changed.
#[test]
fn test_pkg_summary_regenerated_on_same_name_rebuild() -> Result<()> {
    let h = TestHarness::new()?;
    let (db, scan_result, _) = run_scan_and_build(&h)?;

    bob::generate_pkg_summary(&db, 2)?;
    let mtime_before = pkg_summary_gz_mtime(&h).expect("pkg_summary.gz should exist");
    std::thread::sleep(std::time::Duration::from_millis(50));

    let prior = db.get_successful_packages()?;
    assert!(prior.contains(&"top-1.0".to_string()));
    db.delete_build_by_name("top-1.0")?;

    let summary = run_cached_build(&h, &db, &scan_result)?;

    let top = summary
        .results
        .iter()
        .find(|r| r.pkgname.pkgname() == "top-1.0");
    assert!(
        matches!(top.map(|r| &r.state), Some(PackageState::Success)),
        "top-1.0 should have been rebuilt successfully"
    );

    let current = db.get_successful_packages()?;
    assert_eq!(prior, current, "same set of successful package names");
    assert!(
        summary.counts().states[Success] > 0,
        "but a new build succeeded (metadata may differ)"
    );

    let generated = maybe_generate(&db, &prior, &summary)?;
    assert!(
        generated,
        "should regenerate when package rebuilt with same name"
    );
    assert!(
        pkg_summary_gz_mtime(&h).expect("should exist") > mtime_before,
        "pkg_summary.gz should have newer mtime"
    );

    // The rebuilt .tgz should not be newer than the regenerated pkg_summary
    let rebuilt_tgz = h.packages_dir().join("All").join("top-1.0.tgz");
    if rebuilt_tgz.exists() {
        let tgz_mtime = fs::metadata(&rebuilt_tgz)?.modified()?;
        let summary_mtime = pkg_summary_gz_mtime(&h).expect("should exist");
        assert!(
            summary_mtime >= tgz_mtime,
            "pkg_summary.gz should be at least as recent as rebuilt .tgz"
        );
    }

    Ok(())
}

/// When ALL packages fail on a first build (prior empty, current empty),
/// pkg_summary should not be generated.
#[test]
fn test_pkg_summary_skipped_when_all_fail() -> Result<()> {
    let h = TestHarness::new()?;
    let config = h.load_config()?;
    let db = h.open_db()?;
    let state = h.run_state();

    // Scan only build-fail (no deps, will fail at configure)
    let sandbox = Sandbox::new(&config);
    let mut scope = SandboxScope::new(sandbox, state.clone());
    let mut scan = Scan::new(&config);
    scan.add(&pkgsrc::PkgPath::new("test/build-fail")?);
    scan.init_from_db(&db)?;
    scan.start(&db, &mut scope)?;
    let scan_result = scan.resolve_with_report(&db, false)?;

    let scanpkgs = scan_result
        .buildable()
        .map(|p| (p.pkgname().clone(), p.clone()))
        .collect();
    let pkgsrc_env = h.pkgsrc_env();
    let build_sandbox = Sandbox::new(&config);
    let build_scope = SandboxScope::new(build_sandbox, state.clone());
    let build = Build::new(&config, pkgsrc_env, build_scope, scanpkgs);
    let summary = build.start(&state, &db)?;

    assert_eq!(summary.counts().states[Success], 0);
    assert!(summary.counts().states[Failed] > 0);

    let prior: Vec<String> = vec![];
    let generated = maybe_generate(&db, &prior, &summary)?;

    assert!(!generated, "should not generate when all packages fail");
    assert!(
        pkg_summary_gz_mtime(&h).is_none(),
        "pkg_summary.gz should not exist"
    );

    Ok(())
}

/// Adding new packages to an existing build should regenerate pkg_summary
/// when the new packages succeed (successful set grows).
#[test]
fn test_pkg_summary_regenerated_when_new_packages_added() -> Result<()> {
    let h = TestHarness::new()?;
    let config = h.load_config()?;
    let db = h.open_db()?;
    let state = h.run_state();

    // First build: only test/base (1 package)
    let sandbox = Sandbox::new(&config);
    let mut scope = SandboxScope::new(sandbox, state.clone());
    let mut scan = Scan::new(&config);
    scan.add(&pkgsrc::PkgPath::new("test/base")?);
    scan.init_from_db(&db)?;
    scan.start(&db, &mut scope)?;
    let scan_result = scan.resolve_with_report(&db, false)?;

    let scanpkgs = scan_result
        .buildable()
        .map(|p| (p.pkgname().clone(), p.clone()))
        .collect();
    let pkgsrc_env = h.pkgsrc_env();
    let build_sandbox = Sandbox::new(&config);
    let build_scope = SandboxScope::new(build_sandbox, state.clone());
    let build = Build::new(&config, pkgsrc_env, build_scope, scanpkgs);
    let first_summary = build.start(&state, &db)?;

    assert_eq!(first_summary.counts().states[Success], 1);
    let prior_first: Vec<String> = vec![];
    let generated = maybe_generate(&db, &prior_first, &first_summary)?;
    assert!(generated);

    let mtime_first = pkg_summary_gz_mtime(&h).expect("should exist after first build");
    std::thread::sleep(std::time::Duration::from_millis(50));

    // Second build: test/top (pulls in base, mid, also-base, top)
    let state2 = h.run_state();
    let sandbox2 = Sandbox::new(&config);
    let mut scope2 = SandboxScope::new(sandbox2, state2.clone());
    let mut scan2 = Scan::new(&config);
    scan2.add(&pkgsrc::PkgPath::new("test/top")?);
    scan2.init_from_db(&db)?;
    scan2.start(&db, &mut scope2)?;
    let scan_result2 = scan2.resolve_with_report(&db, false)?;

    let prior = db.get_successful_packages()?;
    assert_eq!(
        prior,
        vec!["base-1.0"],
        "only base should be successful so far"
    );

    let scanpkgs2 = scan_result2
        .buildable()
        .map(|p| (p.pkgname().clone(), p.clone()))
        .collect();
    let pkgsrc_env2 = h.pkgsrc_env();
    let build_sandbox2 = Sandbox::new(&config);
    let build_scope2 = SandboxScope::new(build_sandbox2, state2.clone());
    let mut build2 = Build::new(&config, pkgsrc_env2, build_scope2, scanpkgs2);
    build2.load_cached_from_db(&db)?;
    let second_summary = build2.start(&state2, &db)?;

    assert!(
        second_summary.counts().states[Success] >= 3,
        "mid, also-base, top should succeed (base cached)"
    );

    let generated = maybe_generate(&db, &prior, &second_summary)?;
    assert!(generated, "should regenerate when new packages are added");
    assert!(
        pkg_summary_gz_mtime(&h).expect("should exist") > mtime_first,
        "pkg_summary.gz should be newer after adding packages"
    );

    let current = db.get_successful_packages()?;
    assert!(
        current.len() > prior.len(),
        "successful set should have grown"
    );

    Ok(())
}

/// Adding new packages that all fail should NOT regenerate pkg_summary --
/// the successful set is unchanged.
#[test]
fn test_pkg_summary_skipped_when_new_packages_fail() -> Result<()> {
    let h = TestHarness::new()?;
    let (db, _, _) = run_scan_and_build(&h)?;

    bob::generate_pkg_summary(&db, 2)?;
    let mtime_before = pkg_summary_gz_mtime(&h).expect("should exist");
    std::thread::sleep(std::time::Duration::from_millis(50));

    // Clear only the failed packages and rebuild them (they'll fail again).
    // This simulates "adding new (failing) packages" from the perspective
    // of maybe_generate: prior == current, success == 0.
    let prior = db.get_successful_packages()?;
    db.delete_build_by_name("build-fail-1.0")?;
    db.delete_build_by_name("fail-checksum-1.0")?;
    db.delete_build_by_name("fail-at-build-1.0")?;
    db.delete_build_by_name("fail-install-1.0")?;
    db.delete_build_by_name("fail-package-1.0")?;

    let config = h.load_config()?;
    let state = h.run_state();

    // Full scan to get all packages
    let sandbox = Sandbox::new(&config);
    let mut scope = SandboxScope::new(sandbox, state.clone());
    let mut scan = Scan::new(&config);
    scan.init_from_db(&db)?;
    scan.start(&db, &mut scope)?;
    let scan_result = scan.resolve_with_report(&db, false)?;

    let summary = run_cached_build(&h, &db, &scan_result)?;
    let generated = maybe_generate(&db, &prior, &summary)?;

    assert!(
        !generated,
        "should not regenerate when only new failures occur"
    );
    assert_eq!(
        pkg_summary_gz_mtime(&h).expect("should exist"),
        mtime_before,
    );

    Ok(())
}

/// Interrupted build should NOT generate pkg_summary, even if the DB state
/// changed (packages completed before the interrupt).
#[test]
fn test_pkg_summary_skipped_when_interrupted() -> Result<()> {
    let h = TestHarness::new()?;
    let (db, scan_result, _) = run_scan_and_build(&h)?;

    bob::generate_pkg_summary(&db, 2)?;
    let mtime_before = pkg_summary_gz_mtime(&h).expect("should exist");
    std::thread::sleep(std::time::Duration::from_millis(50));

    // Clear a successful package to create work that would change the DB
    let prior = db.get_successful_packages()?;
    db.delete_build_by_name("top-1.0")?;

    // Run build with pre-set shutdown (build exits immediately)
    let config = h.load_config()?;
    let state = h.run_state();
    state.shutdown();

    let scanpkgs = scan_result
        .buildable()
        .map(|p| (p.pkgname().clone(), p.clone()))
        .collect();
    let pkgsrc_env = h.pkgsrc_env();
    let sandbox = Sandbox::new(&config);
    let scope = SandboxScope::new(sandbox, state.clone());
    let mut build = Build::new(&config, pkgsrc_env, scope, scanpkgs);
    build.load_cached_from_db(&db)?;
    let summary = build.start(&state, &db)?;

    // Replicate run_build + caller logic: interrupted -> skip pkg_summary
    assert!(state.interrupted(), "state should be interrupted");
    // Don't call maybe_generate -- this is the Err(Interrupted) path

    assert_eq!(
        pkg_summary_gz_mtime(&h).expect("should exist"),
        mtime_before,
        "pkg_summary.gz mtime should be unchanged after interrupt"
    );

    // Verify the DB state actually changed (top-1.0 was cleared and not rebuilt)
    let current = db.get_successful_packages()?;
    assert!(
        prior != current,
        "DB state should differ (top-1.0 cleared but not rebuilt)"
    );

    // Without the interrupt, this WOULD have triggered regeneration
    assert!(
        prior != current || summary.counts().states[Success] > 0,
        "condition should be true (would regenerate if not interrupted)"
    );

    Ok(())
}

/// After an interrupted build that changed the DB, the next non-interrupted
/// build should regenerate pkg_summary.
#[test]
fn test_pkg_summary_recovered_after_interrupt() -> Result<()> {
    let h = TestHarness::new()?;
    let (db, scan_result, _) = run_scan_and_build(&h)?;

    bob::generate_pkg_summary(&db, 2)?;
    let mtime_before = pkg_summary_gz_mtime(&h).expect("should exist");
    std::thread::sleep(std::time::Duration::from_millis(50));

    // Simulate interrupted rebuild: clear a package, don't regenerate
    db.delete_build_by_name("top-1.0")?;

    // Run build with pre-set shutdown (interrupted, pkg_summary NOT generated)
    let config = h.load_config()?;
    let state = h.run_state();
    state.shutdown();

    let scanpkgs = scan_result
        .buildable()
        .map(|p| (p.pkgname().clone(), p.clone()))
        .collect();
    let pkgsrc_env = h.pkgsrc_env();
    let sandbox = Sandbox::new(&config);
    let scope = SandboxScope::new(sandbox, state.clone());
    let mut build = Build::new(&config, pkgsrc_env, scope, scanpkgs);
    build.load_cached_from_db(&db)?;
    let _interrupted_summary = build.start(&state, &db)?;

    // pkg_summary is stale -- top-1.0 was cleared but not rebuilt
    assert_eq!(
        pkg_summary_gz_mtime(&h).expect("should exist"),
        mtime_before,
        "pkg_summary should be unchanged after interrupted build"
    );

    // Recovery: non-interrupted build rebuilds top-1.0
    let prior = db.get_successful_packages()?;
    assert!(!prior.contains(&"top-1.0".to_string()));

    let summary = run_cached_build(&h, &db, &scan_result)?;
    let generated = maybe_generate(&db, &prior, &summary)?;

    assert!(generated, "should regenerate on recovery after interrupt");
    assert!(
        pkg_summary_gz_mtime(&h).expect("should exist") > mtime_before,
        "pkg_summary.gz should be newer after recovery"
    );

    let current = db.get_successful_packages()?;
    assert!(
        current.contains(&"top-1.0".to_string()),
        "top-1.0 should be back in successful set"
    );

    Ok(())
}

/**
 * Build a package whose `all` target backgrounds a process that holds
 * stdout open.  Without the orphan-cleanup fix in run_command_logged_with_env,
 * bob hangs forever on tee_handle.join() waiting for pipe EOF.
 */
#[test]
fn test_build_orphan_stdout() -> Result<()> {
    let h = TestHarness::new()?;

    // Add the orphan-stdout package to the existing tree.
    let pkg_dir = h.pkgsrc().join("test/orphan-stdout");
    fs::create_dir_all(&pkg_dir)?;

    // Overwrite category Makefile to include the new package.
    let cat_makefile = h.pkgsrc().join("test/Makefile");
    let existing = fs::read_to_string(&cat_makefile)?;
    fs::write(
        &cat_makefile,
        existing.replace(
            "chain-a chain-b chain-c chain-d",
            "chain-a chain-b chain-c chain-d orphan-stdout",
        ),
    )?;

    // Makefile whose "all" target backgrounds a process holding stdout.
    let makefile = "\
PKGNAME=orphan-stdout-1.0

pbulk-index:
\t@printf 'PKGNAME=orphan-stdout-1.0\\nALL_DEPENDS=\\nPKG_SKIP_REASON=\\nPKG_FAIL_REASON=\\nNO_BIN_ON_FTP=\\nRESTRICTED=\\nCATEGORIES=test\\nMAINTAINER=test@example.com\\nUSE_DESTDIR=yes\\nBOOTSTRAP_PKG=yes\\nUSERGROUP_PHASE=\\nSCAN_DEPENDS=\\n'

clean checksum configure stage-install create-usergroup:
\t@true

all:
\t@sleep 3600 &

stage-package-create:
\t@mkdir -p ${.CURDIR}/pkg
\t@d=$$(mktemp -d) && \\\n\
\tprintf '@name orphan-stdout-1.0\\n' > \"$$d/+CONTENTS\" && \\\n\
\tprintf 'Test package\\n' > \"$$d/+COMMENT\" && \\\n\
\tprintf 'Test package description\\n' > \"$$d/+DESC\" && \\\n\
\tprintf '0\\n' > \"$$d/+SIZE_PKG\" && \\\n\
\tprintf 'BUILD_DATE=%s\\nCATEGORIES=test\\nMACHINE_ARCH=x86_64\\nOPSYS=Test\\nOS_VERSION=1.0\\nPKGPATH=test/orphan-stdout\\nPKGTOOLS_VERSION=20210710\\n' \
\"$$(date '+%Y-%m-%d %H:%M:%S %z')\" > \"$$d/+BUILD_INFO\" && \\\n\
\t(cd \"$$d\" && COPYFILE_DISABLE=1 tar czf \"${.CURDIR}/pkg/orphan-stdout-1.0.tgz\" \
+CONTENTS +COMMENT +DESC +SIZE_PKG +BUILD_INFO) && \\\n\
\trm -rf \"$$d\"

show-var:
\t@case \"${VARNAME}\" in STAGE_PKGFILE) echo \"${.CURDIR}/pkg/orphan-stdout-1.0.tgz\" ;; esac

show-vars:
\t@for v in ${VARNAMES}; do \\\n\
\t  case \"$$v\" in \\\n\
\t    WRKDIR) echo \"${.CURDIR}/work\" ;; \\\n\
\t    *) echo ;; \\\n\
\t  esac; \\\n\
\tdone
";
    fs::write(pkg_dir.join("Makefile"), makefile)?;

    let config = h.load_config()?;
    let db = h.open_db()?;
    let state = h.run_state();

    // Force shutdown after 30s so the test doesn't hang forever if broken.
    let state_clone = state.clone();
    std::thread::spawn(move || {
        std::thread::sleep(std::time::Duration::from_secs(30));
        state_clone.shutdown();
    });

    // Scan
    let sandbox = Sandbox::new(&config);
    let mut scan_scope = SandboxScope::new(sandbox, state.clone());
    let mut scan = Scan::new(&config);
    scan.init_from_db(&db)?;
    scan.start(&db, &mut scan_scope)?;
    let scan_result = scan.resolve_with_report(&db, false)?;

    // Build only the orphan-stdout package
    let scanpkgs = scan_result
        .buildable()
        .filter(|p| p.pkgpath.as_path().to_string_lossy() == "test/orphan-stdout")
        .map(|p| (p.pkgname().clone(), p.clone()))
        .collect();

    let pkgsrc_env = h.pkgsrc_env();
    let build_sandbox = Sandbox::new(&config);
    let build_scope = SandboxScope::new(build_sandbox, state.clone());
    let build = Build::new(&config, pkgsrc_env, build_scope, scanpkgs);
    let build_result = build.start(&state, &db)?;

    assert_eq!(
        build_result.counts().states[Success],
        1,
        "orphan-stdout package should build successfully"
    );

    Ok(())
}