yapi_core/packages/

mod.rs

//! Packages: npm, git and local sources that bring extensions, skills,
//! prompt templates and themes. Port of pi's package manager
//! (`core/package-manager.ts` in pi `v1.0.0`), with a built-in npm client.
//!
//! Packages live under the scope's directory: `<agent>/npm` and
//! `<agent>/git/<host>/<path>` for the user, `<cwd>/.yapi/...` for a project.
//! Settings list them in `packages`.

pub mod npm;
pub mod resolve;
pub mod source;

use std::ops::ControlFlow;
use std::path::{Path, PathBuf};

use aws_lc_rs::digest::{SHA256, digest};
use futures_util::StreamExt as _;
use futures_util::future::Either;
use serde_json::Value;
use yapi_types::settings::FilteredPackage;

use crate::config::PROJECT_DIR;
use crate::settings::{Scope, SettingsError, SettingsManager};
pub use source::{Source, parse};

/// Why a package operation failed, with pi's message where pi has one.
#[derive(Debug, thiserror::Error)]
pub enum PackageError {
    /// A failure with pi's message.
    #[error("{0}")]
    Message(String),
    /// Project settings cannot change while the project is untrusted.
    #[error("Project is not trusted. Use --approve to modify local package config.")]
    Untrusted,
    /// The npm client failed.
    #[error(transparent)]
    Npm(#[from] npm::NpmError),
    /// Settings could not be written.
    #[error(transparent)]
    Settings(#[from] SettingsError),
}

/// A configured package, as `list` shows it.
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct Configured {
    /// The source as written in settings.
    pub source: String,
    /// Whose settings list it.
    pub scope: Scope,
    /// Whether the entry filters the package's resources.
    pub filtered: bool,
    /// Where it is installed, when it is.
    pub installed_path: Option<PathBuf>,
    /// Its extension entry files after the entry's filters. Empty when it
    /// is not installed.
    pub extensions: Vec<PathBuf>,
}

/// Where `source`, written in settings whose directory is `base`, is or
/// would be installed.
pub fn install_location(source: &Source, base: &Path) -> PathBuf {
    match source {
        Source::Npm { name, .. } => base.join("npm").join("node_modules").join(name),
        Source::Git { host, path, .. } => base.join("git").join(host).join(path),
        Source::Local { path } => source::local_path(path, base),
    }
}

/// pi's `DefaultPackageManager.resolve` for the settings of the project in
/// `project_root` and of `agent_dir`: every resource of installed packages,
/// settings entries and the discovered directories, each enabled or not,
/// with `builtins` as the built-in extensions, and the packages and placed
/// skills that `chest` installed. Packages that are not installed are left
/// out, and so is each copy of a package that another source installs too,
/// as [`LeftOut`](crate::chest::startup::LeftOut) decides.
pub fn resolve_resources(
    project_root: &Path,
    agent_dir: &Path,
    settings: &SettingsManager,
    builtins: &[&str],
    chest: &crate::chest::startup::Installed,
) -> resolve::ResolvedPaths {
    let project_dir = project_root.join(PROJECT_DIR);
    let base = |scope: &str| {
        if scope == "project" {
            project_dir.clone()
        } else {
            agent_dir.to_path_buf()
        }
    };
    let project_only = crate::config::project_only()
        .then(|| resolve::ProjectOnly::from_settings(settings.document(Scope::Global)));
    let left_out =
        crate::chest::startup::LeftOut::new(chest, settings, project_only.as_ref(), agent_dir);
    let kept = |scope: Scope| {
        let mut packages = settings_packages(settings, scope);
        packages.retain(|entry| {
            entry_source(entry).is_none_or(|source| !left_out.settings(scope, source))
        });
        packages
    };
    let mut packages = resolve::package_inputs(
        &kept(Scope::Project),
        &kept(Scope::Global),
        base,
        |source, scope| install_location(source, &base(scope)),
    );
    packages.extend(
        chest
            .packages
            .iter()
            .filter(|input| !left_out.chest(input))
            .cloned(),
    );
    let user = resolve::settings_lists(settings.document(Scope::Global));
    let project = resolve::settings_lists(settings.document(Scope::Project));
    resolve::resolve(&resolve::ResolveInput {
        cwd: project_root.to_path_buf(),
        agent_dir: agent_dir.to_path_buf(),
        project_dir: project_dir.clone(),
        home: crate::tools::path::home_dir(),
        project_trusted: settings.project_trusted(),
        packages,
        skills: chest.skills.clone(),
        user: [&user[0], &user[1], &user[2], &user[3]],
        project: [&project[0], &project[1], &project[2], &project[3]],
        builtins,
        project_only,
    })
}

/// The `packages` entries of `scope`'s settings.
pub fn settings_packages(settings: &SettingsManager, scope: Scope) -> Vec<Value> {
    settings
        .document(scope)
        .get("packages")
        .and_then(Value::as_array)
        .cloned()
        .unwrap_or_default()
}

/// The directory a scope's packages and relative settings paths use: the
/// agent directory for the user, the project's config directory otherwise.
pub fn scope_dir(scope: Scope, cwd: &Path, agent_dir: &Path) -> PathBuf {
    match scope {
        Scope::Global => agent_dir.to_path_buf(),
        Scope::Project => cwd.join(PROJECT_DIR),
    }
}

/// A settings entry: a source string or a filtered package.
pub fn entry_source(entry: &Value) -> Option<&str> {
    entry.as_str().or_else(|| entry["source"].as_str())
}

/// Installs, removes and resolves packages for a working directory.
pub struct PackageManager {
    cwd: PathBuf,
    agent_dir: PathBuf,
    settings: SettingsManager,
    npm: npm::Npm,
    progress: Box<dyn Fn(&str) + Send + Sync>,
    /// In a project-only run, what it keeps of the user's packages.
    project_only: Option<resolve::ProjectOnly>,
}

impl std::fmt::Debug for PackageManager {
    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
        f.debug_struct("PackageManager")
            .field("cwd", &self.cwd)
            .finish_non_exhaustive()
    }
}

/// The standard output of `git` with `args`, run in `dir` (else the process
/// directory) without a terminal prompt.
pub(crate) async fn run_git(args: &[&str], dir: Option<&Path>) -> Result<String, PackageError> {
    run_git_with(args, dir, &[]).await
}

/// [`run_git`] with `env` set on top of what git normally gets.
pub(crate) async fn run_git_with(
    args: &[&str],
    dir: Option<&Path>,
    env: &[(&str, &str)],
) -> Result<String, PackageError> {
    let mut command = tokio::process::Command::new("git");
    command
        .args(args)
        .env("GIT_TERMINAL_PROMPT", "0")
        .envs(crate::config::child_env())
        .envs(env.iter().copied())
        .stdin(std::process::Stdio::null())
        .kill_on_drop(true);
    if let Some(dir) = dir {
        command.current_dir(dir);
    }
    let output = command
        .output()
        .await
        .map_err(|err| PackageError::Message(format!("git {}: {err}", args[0])))?;
    if output.status.success() {
        return Ok(String::from_utf8_lossy(&output.stdout).into_owned());
    }
    Err(PackageError::Message(format!(
        "git {} failed with code {}: {}",
        args[0],
        output.status.code().unwrap_or(-1),
        String::from_utf8_lossy(&output.stderr).trim()
    )))
}

/// The scope that [`resolve::dedupe_packages`] and
/// [`resolve::locate_packages`] name `project` or `user`.
fn named_scope(name: &str) -> Scope {
    if name == "project" {
        Scope::Project
    } else {
        Scope::Global
    }
}

/// pi's `NETWORK_TIMEOUT_MS`, for registry lookups and git remotes.
const NETWORK_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(10);

/// pi's `UPDATE_CHECK_CONCURRENCY`: how many update checks run at once.
const UPDATE_CHECK_CONCURRENCY: usize = 4;

/// The standard output of `command`, given up after pi's network timeout.
/// `None` when it fails.
async fn command_output(mut command: tokio::process::Command) -> Option<String> {
    command
        .envs(crate::config::child_env())
        .stdin(std::process::Stdio::null())
        .kill_on_drop(true);
    let output = tokio::time::timeout(NETWORK_TIMEOUT, command.output())
        .await
        .ok()?
        .ok()?;
    output
        .status
        .success()
        .then(|| String::from_utf8_lossy(&output.stdout).into_owned())
}

/// The output of `git` with `args` in `dir`, given up after pi's network
/// timeout. `None` when it fails.
async fn git_output(args: &[&str], dir: &Path) -> Option<String> {
    tokio::time::timeout(NETWORK_TIMEOUT, run_git(args, Some(dir)))
        .await
        .ok()?
        .ok()
}

/// pi's `getLatestNpmVersion` for npm package `name` from `version`: what
/// `npm view` reports through the `npmCommand` setting when it is set, else
/// the built-in registry client's answer, given up after pi's network
/// timeout.
async fn latest_npm_version(
    npm: &mut npm::Npm,
    command: Option<&[String]>,
    cwd: &Path,
    name: &str,
    version: Option<&str>,
) -> Option<String> {
    let Some((program, args)) = command.and_then(<[String]>::split_first) else {
        let lookup = npm.latest(name, version.unwrap_or("latest"));
        return tokio::time::timeout(NETWORK_TIMEOUT, lookup)
            .await
            .ok()?
            .ok();
    };
    let spec = version.map_or_else(|| name.to_owned(), |version| format!("{name}@{version}"));
    let mut view = tokio::process::Command::new(program);
    view.args(args)
        .args(["view", &spec, "version", "--json"])
        .current_dir(cwd);
    let output = command_output(view).await?;
    npm::viewed_version(&output, version.filter(|version| npm::is_range(version)))
}

/// pi's `gitHasAvailableUpdate`: whether the remote commit of the checkout
/// in `dir`'s upstream branch, else of the remote's `HEAD`, differs from the
/// checkout's. `false` when git cannot tell.
async fn git_newer(dir: &Path) -> bool {
    let Some(local) = git_output(&["rev-parse", "HEAD"], dir).await else {
        return false;
    };
    let branch = git_output(&["rev-parse", "--abbrev-ref", "@{upstream}"], dir)
        .await
        .and_then(|upstream| {
            let branch = upstream.trim().strip_prefix("origin/")?;
            (!branch.is_empty()).then(|| format!("refs/heads/{branch}"))
        });
    let commit = |listing: &str, reference: Option<&str>| {
        listing.lines().find_map(|line| {
            let (commit, name) = line.split_once(char::is_whitespace)?;
            let named = reference.is_none_or(|reference| name.trim() == reference);
            (commit.len() == 40 && commit.bytes().all(|byte| byte.is_ascii_hexdigit()) && named)
                .then(|| commit.to_owned())
        })
    };
    let mut remote = None;
    if let Some(branch) = branch
        && let Some(listing) = git_output(&["ls-remote", "origin", &branch], dir).await
    {
        remote = commit(&listing, None);
    }
    if remote.is_none()
        && let Some(listing) = git_output(&["ls-remote", "origin", "HEAD"], dir).await
    {
        remote = commit(&listing, Some("HEAD"));
    }
    remote.is_some_and(|remote| remote != local.trim())
}

/// pi's `resolveManagedPath`: `parts` resolved against `root`, which it
/// must not leave.
fn managed_path(root: &Path, parts: &Path) -> Result<PathBuf, PackageError> {
    let path = crate::tools::path::resolve_lexically(root, parts);
    if path.starts_with(root) {
        Ok(path)
    } else {
        Err(PackageError::Message(format!(
            "Refusing to use path outside package install root: {}",
            path.display()
        )))
    }
}

impl PackageManager {
    /// A manager for `cwd`, reading and writing `settings`, with npm's
    /// configuration `npm_config`, as [`npm::config`] reads it.
    pub fn new(
        cwd: PathBuf,
        agent_dir: PathBuf,
        settings: SettingsManager,
        npm_config: std::collections::HashMap<String, String>,
    ) -> PackageManager {
        PackageManager {
            cwd,
            agent_dir,
            settings,
            npm: npm::Npm::new(npm_config),
            progress: Box::new(|_| {}),
            project_only: None,
        }
    }

    /// Makes [`PackageManager::install_missing`] install only the user's
    /// packages that `keep`, a project-only run's keep list, could need.
    pub fn project_only(&mut self, keep: resolve::ProjectOnly) {
        self.project_only = Some(keep);
    }

    /// Reports the start of each install, removal and update.
    pub fn on_progress(&mut self, progress: impl Fn(&str) + Send + Sync + 'static) {
        self.progress = Box::new(progress);
    }

    /// The settings it reads and writes.
    pub fn settings(&self) -> &SettingsManager {
        &self.settings
    }

    fn base(&self, scope: Scope) -> PathBuf {
        scope_dir(scope, &self.cwd, &self.agent_dir)
    }

    fn npm_root(&self, scope: Scope) -> PathBuf {
        self.base(scope).join("npm")
    }

    /// Where `source`, as written in `scope`'s settings, is or would be
    /// installed.
    fn install_path(&self, source: &Source, scope: Scope) -> PathBuf {
        install_location(source, &self.base(scope))
    }

    /// Where `source`, as written in `scope`'s settings, is installed.
    pub fn installed_path(&self, source: &str, scope: Scope) -> Option<PathBuf> {
        let path = self.install_path(&parse(source), scope);
        path.exists().then_some(path)
    }

    fn assert_writable(&self, scope: Scope) -> Result<(), PackageError> {
        if scope == Scope::Project && !self.settings.project_trusted() {
            return Err(PackageError::Untrusted);
        }
        Ok(())
    }

    fn scope(local: bool) -> Scope {
        if local { Scope::Project } else { Scope::Global }
    }

    /// pi's `installAndPersist`: installs `source` into the user's (or with
    /// `local`, the project's) packages and lists it in their settings.
    pub async fn install(&mut self, source: &str, local: bool) -> Result<(), PackageError> {
        let scope = Self::scope(local);
        self.assert_writable(scope)?;
        (self.progress)(&format!("Installing {source}..."));
        let parsed = parse(source);
        match &parsed {
            Source::Npm { name, version, .. } => {
                self.install_npm(&self.npm_root(scope), name, version.as_deref(), false)
                    .await?;
            }
            Source::Git {
                repo, reference, ..
            } => {
                let dir = self.install_path(&parsed, scope);
                let git_root = self.base(scope).join("git");
                self.install_git(repo, reference.as_deref(), &dir, Some(&git_root))
                    .await?;
            }
            Source::Local { path } => {
                let resolved = source::local_path(path, &self.cwd);
                if !resolved.exists() {
                    return Err(PackageError::Message(format!(
                        "Path does not exist: {}",
                        resolved.display()
                    )));
                }
            }
        }
        self.add_to_settings(source, scope)
    }

    /// Installs `name` (at `version`) into the npm root `root`, holding the
    /// root's lock. With `stale_only`, a package that another process
    /// installed meanwhile, so that it is no longer [`npm_stale`], is kept.
    async fn install_npm(
        &mut self,
        root: &Path,
        name: &str,
        version: Option<&str>,
        stale_only: bool,
    ) -> Result<(), PackageError> {
        let _lock = npm::lock_root(root).await?;
        if stale_only && !npm_stale(&root.join("node_modules").join(name), version) {
            return Ok(());
        }
        if let Some(command) = self.settings.settings().npm_command.clone() {
            // pi's `npmCommand` runs a real package manager, scripts and all.
            let (program, args) = command.split_first().ok_or_else(|| {
                PackageError::Message(
                    "Invalid npmCommand: first array entry must be a non-empty command".into(),
                )
            })?;
            let spec =
                version.map_or_else(|| name.to_owned(), |version| format!("{name}@{version}"));
            // As pi when stdout carries JSON or RPC: npm's output goes to
            // stderr, and npm never reads the client's input.
            let status = tokio::process::Command::new(program)
                .envs(crate::config::child_env())
                .args(args)
                .args(["install", &spec, "--prefix"])
                .arg(root)
                .arg("--legacy-peer-deps")
                .stdin(std::process::Stdio::null())
                .stdout(std::io::stderr())
                .status()
                .await
                .map_err(|err| PackageError::Message(format!("{program}: {err}")))?;
            if !status.success() {
                return Err(PackageError::Message(format!(
                    "{program} install {spec} failed with code {}",
                    status.code().unwrap_or(-1)
                )));
            }
            return Ok(());
        }
        npm::install(&mut self.npm, root, name, version).await?;
        Ok(())
    }

    /// pi's `installGit`: clones `repo` into `dir` at `reference` and installs
    /// its dependencies, removing the clone when any step fails. A new clone
    /// first gets a `.gitignore` in `git_root`, the scope's `git` folder. An
    /// existing checkout is updated in place instead and kept when that fails.
    /// Unlike pi, one install of `dir` runs at a time across processes, and
    /// a clone whose install never finished, as when yapi was stopped, is
    /// cloned again.
    async fn install_git(
        &mut self,
        repo: &str,
        reference: Option<&str>,
        dir: &Path,
        git_root: Option<&Path>,
    ) -> Result<(), PackageError> {
        if let Some(parent) = dir.parent() {
            std::fs::create_dir_all(parent)
                .map_err(|err| PackageError::Message(err.to_string()))?;
        }
        // The lock is a folder beside the checkout, so it never enters it.
        let lock = npm::lock(dir).await?;
        let incomplete = clone_marker(dir);
        if dir.exists() && !incomplete.exists() {
            return self.update_git(dir, reference).await;
        }
        if dir.exists() {
            std::fs::remove_dir_all(dir).map_err(|err| PackageError::Message(err.to_string()))?;
        }
        if let Some(git_root) = git_root {
            npm::ensure_git_ignore(git_root)?;
        }
        let target = dir.to_string_lossy().into_owned();
        // Before the clone starts, so a clone cut off or still running never
        // counts as installed.
        std::fs::write(&incomplete, "").map_err(|err| PackageError::Message(err.to_string()))?;
        let installed: Result<(), PackageError> = async {
            run_git(&["clone", "--", repo, &target], None).await?;
            if let Some(reference) = reference {
                run_git(&["checkout", reference], Some(dir)).await?;
            }
            npm::install_dependencies(&mut self.npm, dir).await?;
            std::fs::remove_file(&incomplete)
                .map_err(|err| PackageError::Message(err.to_string()))?;
            Ok(())
        }
        .await;
        if installed.is_err() {
            let _ = std::fs::remove_dir_all(dir);
            let _ = std::fs::remove_file(&incomplete);
            // Without the lock's folder, the empty folders above go too.
            drop(lock);
            self.remove_dir(dir);
        }
        installed
    }

    /// Moves the checkout in `dir` to the latest commit of `reference`, else
    /// of its upstream branch, and installs its dependencies. Fails unless
    /// `dir` is the top of its own checkout, so git never acts on an enclosing
    /// repository.
    async fn update_git(
        &mut self,
        dir: &Path,
        reference: Option<&str>,
    ) -> Result<(), PackageError> {
        let top = run_git(&["rev-parse", "--show-toplevel"], Some(dir)).await?;
        let top = Path::new(top.trim()).canonicalize().ok();
        if top.is_none() || top != dir.canonicalize().ok() {
            return Err(PackageError::Message(format!(
                "Refusing to update {}: it is not a git checkout of its own",
                dir.display()
            )));
        }
        match reference {
            Some(reference) => {
                run_git(&["fetch", "origin", reference], Some(dir)).await?;
                run_git(&["checkout", "FETCH_HEAD"], Some(dir)).await?;
            }
            None => {
                run_git(&["fetch", "--prune", "origin"], Some(dir)).await?;
                run_git(&["reset", "--hard", "@{upstream}"], Some(dir)).await?;
            }
        }
        npm::install_dependencies(&mut self.npm, dir).await?;
        Ok(())
    }

    /// Removes `dir` and the empty directories above it, up to the git root
    /// or the temporary packages folder.
    fn remove_dir(&self, dir: &Path) {
        let _ = std::fs::remove_dir_all(dir);
        let temporary = self.agent_dir.join("tmp").join("extensions");
        let mut current = dir.parent();
        while let Some(parent) = current {
            if parent.file_name().is_some_and(|name| name == "git")
                || parent == temporary
                || std::fs::remove_dir(parent).is_err()
            {
                break;
            }
            current = parent.parent();
        }
    }

    /// pi's `removeAndPersist`: uninstalls `source` from the scope and drops
    /// its settings entries. Returns `false` when settings did not list it.
    pub async fn remove(&mut self, source: &str, local: bool) -> Result<bool, PackageError> {
        let scope = Self::scope(local);
        self.assert_writable(scope)?;
        (self.progress)(&format!("Removing {source}..."));
        let parsed = parse(source);
        match &parsed {
            Source::Npm { name, .. } => npm::uninstall(&self.npm_root(scope), name).await?,
            Source::Git { .. } => {
                let dir = self.install_path(&parsed, scope);
                if dir.exists() {
                    self.remove_dir(&dir);
                }
            }
            Source::Local { .. } => {}
        }
        self.remove_from_settings(source, scope)
    }

    fn packages(&self, scope: Scope) -> Vec<Value> {
        settings_packages(&self.settings, scope)
    }

    /// Whether settings entry `entry` of `scope` names the same package as
    /// command-line `source`.
    fn matches(&self, entry: &Value, source: &str, scope: Scope) -> bool {
        entry_source(entry).is_some_and(|existing| {
            parse(existing).identity(&self.base(scope)) == parse(source).identity(&self.cwd)
        })
    }

    /// The source as settings store it: a local path relative to the scope's
    /// directory.
    fn settings_source(&self, source: &str, scope: Scope) -> String {
        match parse(source) {
            Source::Local { path } => {
                let resolved = source::local_path(&path, &self.cwd);
                let relative = crate::tools::path::relative(&self.base(scope), &resolved);
                if relative.is_empty() {
                    ".".into()
                } else {
                    relative
                }
            }
            _ => source.to_owned(),
        }
    }

    fn add_to_settings(&mut self, source: &str, scope: Scope) -> Result<(), PackageError> {
        let normalized = self.settings_source(source, scope);
        let mut packages = self.packages(scope);
        match packages
            .iter()
            .position(|entry| self.matches(entry, source, scope))
        {
            Some(index) => {
                if entry_source(&packages[index]) == Some(normalized.as_str()) {
                    return Ok(());
                }
                if let Some(object) = packages[index].as_object_mut() {
                    object.insert("source".into(), Value::String(normalized));
                } else {
                    packages[index] = Value::String(normalized);
                }
            }
            None => packages.push(Value::String(normalized)),
        }
        self.settings
            .set(scope, "packages", Some(Value::Array(packages)))?;
        Ok(())
    }

    fn remove_from_settings(&mut self, source: &str, scope: Scope) -> Result<bool, PackageError> {
        let packages = self.packages(scope);
        let kept: Vec<Value> = packages
            .iter()
            .filter(|entry| !self.matches(entry, source, scope))
            .cloned()
            .collect();
        if kept.len() == packages.len() {
            return Ok(false);
        }
        self.settings
            .set(scope, "packages", Some(Value::Array(kept)))?;
        Ok(true)
    }

    /// The packages settings list: the user's, then the project's.
    pub fn list(&self) -> Vec<Configured> {
        [Scope::Global, Scope::Project]
            .into_iter()
            .flat_map(|scope| {
                self.packages(scope).into_iter().filter_map(move |entry| {
                    let source = entry_source(&entry)?.to_owned();
                    let installed_path = self.installed_path(&source, scope);
                    let filters: Option<FilteredPackage> = entry
                        .is_object()
                        .then(|| serde_json::from_value(entry.clone()).ok())
                        .flatten();
                    let extensions = installed_path
                        .as_deref()
                        .map(|root| {
                            resolve::package_resources(
                                root,
                                filters.as_ref(),
                                source::is_local(&source),
                            )
                            .enabled(resolve::ResourceType::Extensions)
                            .map(|info| PathBuf::from(&info.path))
                            .collect()
                        })
                        .unwrap_or_default();
                    Some(Configured {
                        installed_path,
                        filtered: entry.is_object(),
                        extensions,
                        source,
                        scope,
                    })
                })
            })
            .collect()
    }

    /// pi's `update`: reinstalls unpinned npm packages with newer matching
    /// versions and fast-forwards git packages, for every configured package
    /// or the one `source` names.
    pub async fn update(&mut self, source: Option<&str>) -> Result<(), PackageError> {
        let mut targets = Vec::new();
        for scope in [Scope::Global, Scope::Project] {
            for entry in self.packages(scope) {
                let Some(configured) = entry_source(&entry).map(str::to_owned) else {
                    continue;
                };
                if source.is_none_or(|source| self.matches(&entry, source, scope)) {
                    targets.push((configured, scope));
                }
            }
        }
        if let Some(source) = source
            && targets.is_empty()
        {
            return Err(PackageError::Message(format!(
                "No matching package found for {source}"
            )));
        }
        if crate::tools::external::offline() {
            return Ok(());
        }
        let mut npm_targets = Vec::new();
        let mut git_targets = Vec::new();
        for (configured, scope) in targets {
            match parse(&configured) {
                Source::Npm {
                    name,
                    version,
                    pinned: false,
                } => npm_targets.push((configured, scope, name, version)),
                // pi also reconciles checkouts pinned to a ref.
                Source::Git {
                    repo, reference, ..
                } => git_targets.push((configured, scope, repo, reference)),
                _ => {}
            }
        }
        // pi updates a package whose versions it cannot compare.
        let checks = npm_targets
            .iter()
            .map(|(configured, scope, name, version)| {
                let root = self.install_path(&parse(configured), *scope);
                self.npm_check(name.clone(), version.clone(), root)
            });
        let stale: Vec<bool> = futures_util::stream::iter(checks)
            .buffered(UPDATE_CHECK_CONCURRENCY)
            .map(|newer| newer.unwrap_or(true))
            .collect()
            .await;
        for scope in [Scope::Global, Scope::Project] {
            let batch: Vec<_> = npm_targets
                .iter()
                .zip(&stale)
                .filter(|((_, target_scope, ..), stale)| *target_scope == scope && **stale)
                .map(|(target, _)| target)
                .collect();
            match batch.as_slice() {
                [] => continue,
                [(configured, ..)] => (self.progress)(&format!("Updating {configured}...")),
                _ => {
                    let scope = if scope == Scope::Global {
                        "user"
                    } else {
                        "project"
                    };
                    (self.progress)(&format!("Updating {scope} npm packages..."));
                }
            }
            for (_, _, name, version) in batch {
                self.install_npm(&self.npm_root(scope), name, version.as_deref(), false)
                    .await?;
            }
        }
        for (configured, scope, repo, reference) in git_targets {
            (self.progress)(&format!("Updating {configured}..."));
            let dir = self.install_path(&parse(&configured), scope);
            // pi's `updateGit`: a missing checkout is installed.
            let git_root = self.base(scope).join("git");
            self.install_git(&repo, reference.as_deref(), &dir, Some(&git_root))
                .await?;
        }
        Ok(())
    }

    /// pi's npm update check for package `name` from `version` (a range or
    /// dist-tag, else `latest`) installed in `root`: whether the newest
    /// version is later than the installed one. `None` when either is
    /// unknown. The check owns what it needs, so several can run at once.
    fn npm_check(
        &self,
        name: String,
        version: Option<String>,
        root: PathBuf,
    ) -> impl Future<Output = Option<bool>> + Send + use<> {
        let mut npm = self.npm.clone();
        let command = self.settings.settings().npm_command.clone();
        let cwd = self.cwd.clone();
        async move {
            let installed = npm::installed_version(&root)?;
            let latest = latest_npm_version(
                &mut npm,
                command.as_deref(),
                &cwd,
                &name,
                version.as_deref(),
            )
            .await?;
            npm::newer(&latest, &installed)
        }
    }

    /// pi's `checkForAvailableUpdates`: the installed npm and git packages
    /// that settings list which have a later version than the one installed,
    /// by name: the npm package's, or `<host>/<path>` for git. Local and
    /// pinned sources are skipped, and so is a package whose check fails.
    pub async fn available_updates(&self) -> Vec<String> {
        let project = self.packages(Scope::Project);
        let user = self.packages(Scope::Global);
        let entries = resolve::dedupe_packages(&project, &user, |source, in_scope| {
            parse(source).identity(&self.base(named_scope(in_scope)))
        });
        let checks = entries.into_iter().filter_map(|(entry, in_scope)| {
            let parsed = parse(entry_source(&entry)?);
            let root = self.install_path(&parsed, named_scope(in_scope));
            if !root.exists() {
                return None;
            }
            match parsed {
                Source::Npm {
                    name,
                    version,
                    pinned: false,
                } => {
                    let check = self.npm_check(name.clone(), version, root);
                    Some(Either::Left(async move {
                        (check.await == Some(true)).then_some(name)
                    }))
                }
                Source::Git {
                    host,
                    path,
                    reference: None,
                    ..
                } => Some(Either::Right(async move {
                    git_newer(&root).await.then(|| format!("{host}/{path}"))
                })),
                _ => None,
            }
        });
        let checks: Vec<_> = checks.collect();
        futures_util::stream::iter(checks)
            .buffered(UPDATE_CHECK_CONCURRENCY)
            .filter_map(std::future::ready)
            .collect()
            .await
    }

    /// pi's temporary package for `-e`: where `source` is for this run. npm
    /// and git sources live under `<agent>/tmp/extensions` and are not added
    /// to settings. A missing package is installed, an npm package whose
    /// installed version is outside the source's range is reinstalled, and an
    /// unpinned git checkout is refreshed, kept as it is when that fails.
    /// `offline` skips all three, and a missing package is then `None`.
    pub async fn install_temporary(
        &mut self,
        source: &str,
        offline: bool,
    ) -> Result<Option<PathBuf>, PackageError> {
        match parse(source) {
            Source::Npm { name, version, .. } => {
                let root = self.temporary_dir("npm", "", None)?;
                let dir = root.join("node_modules").join(&name);
                if npm_stale(&dir, version.as_deref()) {
                    if offline {
                        return Ok(None);
                    }
                    self.install_npm(&root, &name, version.as_deref(), true)
                        .await?;
                }
                Ok(Some(dir))
            }
            Source::Git {
                repo,
                host,
                path,
                reference,
            } => {
                let dir =
                    self.temporary_dir(&format!("git-{host}"), &path, reference.as_deref())?;
                if !dir.exists() {
                    if offline {
                        return Ok(None);
                    }
                    self.install_git(&repo, reference.as_deref(), &dir, None)
                        .await?;
                } else if reference.is_none() && !offline {
                    let _ = self.update_git(&dir, None).await;
                }
                Ok(Some(dir))
            }
            Source::Local { path } => Ok(Some(source::local_path(&path, &self.cwd))),
        }
    }

    /// pi's `getTemporaryDir`: `<agent>/tmp/extensions/<prefix>/<hash>/<suffix>`,
    /// where `hash` starts the SHA-256 of `<prefix>-<suffix>[@<reference>]`.
    /// The `extensions` folder is readable by the user only.
    fn temporary_dir(
        &self,
        prefix: &str,
        suffix: &str,
        reference: Option<&str>,
    ) -> Result<PathBuf, PackageError> {
        let folder = self.agent_dir.join("tmp").join("extensions");
        let failed =
            |err: std::io::Error| PackageError::Message(format!("{}: {err}", folder.display()));
        std::fs::create_dir_all(&folder).map_err(failed)?;
        #[cfg(unix)]
        {
            use std::os::unix::fs::PermissionsExt as _;
            std::fs::set_permissions(&folder, std::fs::Permissions::from_mode(0o700))
                .map_err(failed)?;
        }
        let key = match reference {
            Some(reference) => format!("{prefix}-{suffix}@{reference}"),
            None => format!("{prefix}-{suffix}"),
        };
        let hash = crate::time::hex(digest(&SHA256, key.as_ref()).as_ref());
        let root = managed_path(&folder, Path::new(prefix))?;
        managed_path(&root, &Path::new(&hash[..8]).join(suffix))
    }

    /// The installs of pi's `resolve`: the configured npm and git packages
    /// that are missing, and the npm packages whose installed version is
    /// outside the configured range, in the order and at the places of
    /// [`resolve::locate_packages`]. `on_error` receives each failure, with
    /// its source, and its `Break` ends the install and is returned.
    pub async fn install_missing<B>(
        &mut self,
        mut on_error: impl FnMut(&str, String) -> ControlFlow<B>,
    ) -> ControlFlow<B> {
        let packages = resolve::locate_packages(
            &self.packages(Scope::Project),
            &self.packages(Scope::Global),
            |source, in_scope| parse(source).identity(&self.base(named_scope(in_scope))),
        );
        let mut attempted = Vec::new();
        for (_, scope, source, in_scope) in packages {
            // A project-only run installs only the user's packages it could
            // keep. A project's `autoload: false` entry still finds the
            // user's copy.
            let keep = self.project_only.as_ref();
            if scope == "user"
                && keep.is_some_and(|keep| !keep.needs_package(&source, &self.agent_dir))
            {
                continue;
            }
            // An `autoload: false` entry and the user's entry it adjusts
            // install the same package.
            if attempted.contains(&(source.clone(), in_scope)) {
                continue;
            }
            attempted.push((source.clone(), in_scope));
            let parsed = parse(&source);
            let root = self.install_path(&parsed, named_scope(in_scope));
            let installed = match &parsed {
                Source::Npm { name, version, .. } => {
                    if npm_stale(&root, version.as_deref()) {
                        let npm_root = self.npm_root(named_scope(in_scope));
                        self.install_npm(&npm_root, name, version.as_deref(), true)
                            .await
                    } else {
                        Ok(())
                    }
                }
                _ if root.exists() && !clone_marker(&root).exists() => Ok(()),
                Source::Git {
                    repo, reference, ..
                } => {
                    let git_root = self.base(named_scope(in_scope)).join("git");
                    self.install_git(repo, reference.as_deref(), &root, Some(&git_root))
                        .await
                }
                Source::Local { .. } => Ok(()),
            };
            if let Err(err) = installed {
                on_error(&source, format!("Failed to install {source}: {err}"))?;
            }
        }
        ControlFlow::Continue(())
    }
}

/// The file beside git checkout `dir` while its install has not finished:
/// from before the clone starts until its dependencies are installed.
fn clone_marker(dir: &Path) -> PathBuf {
    let mut name = dir.file_name().unwrap_or_default().to_os_string();
    name.push(npm::INCOMPLETE);
    dir.with_file_name(name)
}

/// Whether the npm package at `dir` must be installed: it is missing or its
/// install never finished, or `range` is a version range its installed
/// version does not satisfy. A
/// dist-tag such as `latest` is no range, so it never reinstalls.
fn npm_stale(dir: &Path, range: Option<&str>) -> bool {
    npm::installed_version(dir).is_none_or(|installed| {
        range.is_some_and(|range| npm::is_range(range) && !npm::satisfies(&installed, range))
    })
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn temporary_dirs_are_pis() {
        let agent = std::env::temp_dir().join(format!("yapi-temporary-{}", std::process::id()));
        let manager = PackageManager::new(
            agent.clone(),
            agent.clone(),
            SettingsManager::in_memory(),
            Default::default(),
        );
        let extensions = agent.join("tmp/extensions");
        let dir = |prefix: &str, suffix: &str, reference| {
            manager
                .temporary_dir(prefix, suffix, reference)
                .map_err(|err| err.to_string())
        };
        // Hashes from pi's `getTemporaryDir`.
        assert_eq!(
            dir("npm", "", None).unwrap(),
            extensions.join("npm/f35b2129")
        );
        assert_eq!(
            dir("git-github.com", "user/repo", None).unwrap(),
            extensions.join("git-github.com/338a1076/user/repo")
        );
        assert_eq!(
            dir("git-github.com", "user/repo", Some("v1")).unwrap(),
            extensions.join("git-github.com/18093e51/user/repo")
        );
        let escape = dir("git-github.com", "../../../outside", None).unwrap_err();
        assert!(escape.starts_with("Refusing to use path outside package install root: "));
        std::fs::remove_dir_all(&agent).unwrap();
    }

    /// A fresh directory for test `name`, and a manager whose agent
    /// directory is in it and whose registry refuses connections.
    fn git_scratch(name: &str) -> (PathBuf, PackageManager) {
        let dir = std::env::temp_dir().join(format!("yapi-git-{name}-{}", std::process::id()));
        let _ = std::fs::remove_dir_all(&dir);
        std::fs::create_dir_all(&dir).unwrap();
        let registry = [("registry".to_owned(), "http://127.0.0.1:9/".to_owned())];
        let manager = PackageManager::new(
            dir.clone(),
            dir.join("agent"),
            SettingsManager::in_memory(),
            registry.into(),
        );
        (dir, manager)
    }

    /// Commits everything in the repository `repo` and tags the commit.
    async fn commit(repo: &Path, tag: &str) {
        let config = ["-c", "user.name=yapi", "-c", "user.email=yapi@example.com"];
        let signing = ["-c", "commit.gpgsign=false", "-c", "tag.gpgsign=false"];
        for args in [
            &["add", "-A"][..],
            &["commit", "--allow-empty", "-m", tag],
            &["tag", tag],
        ] {
            let args = [&config[..], &signing[..], args].concat();
            run_git(&args, Some(repo)).await.unwrap();
        }
    }

    fn head(repo: &Path, reference: &str) -> String {
        let output = std::process::Command::new("git")
            .args(["rev-parse", reference])
            .current_dir(repo)
            .output()
            .unwrap();
        String::from_utf8(output.stdout).unwrap()
    }

    /// pi's `installGit` on an existing checkout fetches the new ref, and
    /// keeps the checkout when that fails.
    #[tokio::test]
    async fn new_ref_updates_checkout() {
        let (dir, mut manager) = git_scratch("existing");
        let origin = dir.join("origin");
        run_git(&["init", "-b", "main", origin.to_str().unwrap()], None)
            .await
            .unwrap();
        commit(&origin, "v1").await;
        let checkout = dir.join("agent/git/github.com/user/repo");
        let repo = origin.to_str().unwrap();
        manager
            .install_git(repo, None, &checkout, None)
            .await
            .unwrap();
        // A clone whose install never finished, as when yapi was stopped,
        // is cloned again rather than kept.
        let incomplete = clone_marker(&checkout);
        assert!(!incomplete.exists());
        std::fs::write(&incomplete, "").unwrap();
        std::fs::write(checkout.join("stray"), "").unwrap();
        manager
            .install_git(repo, None, &checkout, None)
            .await
            .unwrap();
        assert!(!incomplete.exists() && !checkout.join("stray").exists());
        commit(&origin, "v2").await;

        manager
            .install_git(repo, Some("v2"), &checkout, None)
            .await
            .unwrap();
        assert_eq!(head(&checkout, "HEAD"), head(&origin, "v2"));
        let missing = manager.install_git(repo, Some("v3"), &checkout, None).await;
        assert!(missing.is_err());
        assert_eq!(head(&checkout, "HEAD"), head(&origin, "v2"));
        std::fs::remove_dir_all(&dir).unwrap();
    }

    /// pi removes a clone whose dependencies fail to install.
    #[tokio::test]
    async fn failed_clone_removed() {
        let (dir, mut manager) = git_scratch("dependencies");
        let origin = dir.join("origin");
        run_git(&["init", "-b", "main", origin.to_str().unwrap()], None)
            .await
            .unwrap();
        let manifest = r#"{"name": "repo", "dependencies": {"left-pad": "^1.0.0"}}"#;
        std::fs::write(origin.join("package.json"), manifest).unwrap();
        commit(&origin, "v1").await;
        let checkout = dir.join("agent/git/github.com/user/repo");
        let installed = manager
            .install_git(origin.to_str().unwrap(), None, &checkout, None)
            .await;
        assert!(installed.is_err());
        assert!(!checkout.exists());
        assert!(dir.join("agent/git").exists());
        std::fs::remove_dir_all(&dir).unwrap();
    }

    /// A checkout that links `node_modules`, or a scope in it, elsewhere gets
    /// no dependencies, so nothing is written through the link.
    #[cfg(unix)]
    #[tokio::test]
    async fn linked_node_modules_refused() {
        for link in ["node_modules", "node_modules/@scope"] {
            let (dir, mut manager) = git_scratch("linked");
            let outside = dir.join("outside");
            std::fs::create_dir_all(&outside).unwrap();
            let origin = dir.join("origin");
            run_git(&["init", "-b", "main", origin.to_str().unwrap()], None)
                .await
                .unwrap();
            let manifest = r#"{"name": "repo", "dependencies": {"@scope/dep": "^1.0.0"}}"#;
            std::fs::write(origin.join("package.json"), manifest).unwrap();
            let link = origin.join(link);
            std::fs::create_dir_all(link.parent().unwrap()).unwrap();
            std::os::unix::fs::symlink(&outside, &link).unwrap();
            commit(&origin, "v1").await;
            let checkout = dir.join("agent/git/github.com/user/repo");
            let error = manager
                .install_git(origin.to_str().unwrap(), None, &checkout, None)
                .await
                .unwrap_err();
            assert!(
                error
                    .to_string()
                    .starts_with("Refusing to install dependencies through the link "),
                "{}: {error}",
                link.display()
            );
            assert!(!checkout.exists());
            std::fs::remove_dir_all(&dir).unwrap();
        }
    }
}