lux_lib/operations/

fetch.rs

use crate::build::utils::recursive_copy_dir;
use crate::config::Config;
use crate::git::url::RemoteGitUrlParseError;
use crate::git::{GitRef, GitSource};
use crate::hash::HasIntegrity;
use crate::lockfile::RemotePackageSourceUrl;
use crate::lua_rockspec::{RemoteRockSource, RockSourceSpec};
use crate::package::PackageSpec;
use crate::reqwest::{RequestBuilderExt, RequestError};
use crate::rockspec::Rockspec;
use crate::{fs, operations};
use auth_git2::{GitAuthenticator, Prompter};
use bon::Builder;
use git2::build::RepoBuilder;
use git2::{FetchOptions, RemoteCallbacks};
use miette::Diagnostic;
use remove_dir_all::remove_dir_all;
use ssri::Integrity;
use std::io;
use std::io::Cursor;
use std::io::Read;
use std::path::{Path, PathBuf};
use thiserror::Error;

use super::DownloadSrcRockError;
use super::UnpackError;

/// A rocks package source fetcher, providing fine-grained control
/// over how a package should be fetched.
#[derive(Builder)]
#[builder(start_fn = new, finish_fn(name = _build, vis = ""))]
pub struct FetchSrc<'a, R: Rockspec> {
    #[builder(start_fn)]
    dest_dir: &'a Path,
    #[builder(start_fn)]
    rockspec: &'a R,
    #[builder(start_fn)]
    config: &'a Config,
    #[builder(setters(vis = "pub(crate)"))]
    source_url: Option<RemotePackageSourceUrl>,
}

#[derive(Debug)]
pub(crate) struct RemotePackageSourceMetadata {
    pub hash: Integrity,
    pub source_url: RemotePackageSourceUrl,
}

impl<R: Rockspec, State> FetchSrcBuilder<'_, R, State>
where
    State: fetch_src_builder::State + fetch_src_builder::IsComplete,
{
    /// Fetch and unpack the source into the `dest_dir`.
    pub async fn fetch(self) -> Result<(), FetchSrcError> {
        self.fetch_internal().await?;
        Ok(())
    }

    /// Fetch and unpack the source into the `dest_dir`,
    /// returning the source `Integrity`.
    pub(crate) async fn fetch_internal(self) -> Result<RemotePackageSourceMetadata, FetchSrcError> {
        let fetch = self._build();
        let rockspec = fetch.rockspec;
        let rock_source = rockspec.source().current_platform();
        let dest_dir = fetch.dest_dir;
        let config = fetch.config;
        // prioritise lockfile source, if present
        let source_spec = match &fetch.source_url {
            Some(source_url) => match source_url {
                RemotePackageSourceUrl::Git {
                    url, checkout_ref, ..
                } => RockSourceSpec::Git(GitSource {
                    url: url.parse()?,
                    git_ref: Some(GitRef::Tag(checkout_ref.clone())),
                }),
                RemotePackageSourceUrl::Url { url } => RockSourceSpec::Url(url.clone()),
                RemotePackageSourceUrl::File { path } => RockSourceSpec::File(path.clone()),
            },
            None => rock_source.source_spec.clone(),
        };
        match fetch_src_impl(source_spec, rockspec, rock_source, dest_dir, config).await {
            Err(err)
                if fetch
                    .source_url
                    .is_some_and(|url| matches!(url, RemotePackageSourceUrl::File { .. })) =>
            {
                // Don't fall back to downloading .src.rock archives if a local source was specified.
                Err(err)
            }
            Err(err) => match &fetch.rockspec.source().current_platform().source_spec {
                RockSourceSpec::Git(_) | RockSourceSpec::Url(_) => {
                    let package = PackageSpec::new(
                        fetch.rockspec.package().clone(),
                        fetch.rockspec.version().clone(),
                    );
                    match FetchSrcRock::new(&package, fetch.dest_dir, fetch.config)
                        .fetch()
                        .await
                    {
                        Ok(metadata) => Ok(metadata),
                        Err(fallback_err) => {
                            tracing::error!("fallback .src.rock download failed: {fallback_err:?}");
                            Err(err)
                        }
                    }
                }
                RockSourceSpec::File(_) => Err(err),
            },
            Ok(metadata) => Ok(metadata),
        }
    }
}

#[derive(Error, Debug, Diagnostic)]
#[non_exhaustive]
pub enum FetchSrcError {
    #[error("failed to clone rock source")]
    #[diagnostic(help("check your network connection and verify the git URL is correct."))]
    GitClone(#[from] git2::Error),
    #[error("git operation failed")]
    #[diagnostic(help("this is a bug, please report it."))]
    GitJoin(#[from] tokio::task::JoinError),
    #[error("failed to parse git URL")]
    #[diagnostic(forward(0))]
    GitUrlParse(#[from] RemoteGitUrlParseError),
    #[error(transparent)]
    #[diagnostic(transparent)]
    Request(#[from] RequestError),
    #[error(transparent)]
    #[diagnostic(transparent)]
    Unpack(#[from] UnpackError),
    #[error(transparent)]
    #[diagnostic(transparent)]
    FetchSrcRock(#[from] FetchSrcRockError),
    #[error("unable to remove the '.git' directory")]
    #[diagnostic(help(
        "check that no process is using the directory and you have write permissions."
    ))]
    CleanGitDir(#[source] io::Error),
    #[error("unable to compute hash")]
    Hash(#[source] io::Error),
    #[error(transparent)]
    #[diagnostic(transparent)]
    Fs(#[from] fs::FsError),
}

impl From<reqwest::Error> for FetchSrcError {
    fn from(err: reqwest::Error) -> Self {
        Self::Request(err.into())
    }
}

/// A rocks package source fetcher, providing fine-grained control
/// over how a package should be fetched.
#[derive(Builder)]
#[builder(start_fn = new, finish_fn(name = _build, vis = ""))]
struct FetchSrcRock<'a> {
    #[builder(start_fn)]
    package: &'a PackageSpec,
    #[builder(start_fn)]
    dest_dir: &'a Path,
    #[builder(start_fn)]
    config: &'a Config,
}

impl<State> FetchSrcRockBuilder<'_, State>
where
    State: fetch_src_rock_builder::State + fetch_src_rock_builder::IsComplete,
{
    pub async fn fetch(self) -> Result<RemotePackageSourceMetadata, FetchSrcRockError> {
        do_fetch_src_rock(self._build()).await
    }
}

#[derive(Error, Debug, Diagnostic)]
#[non_exhaustive]
#[error(transparent)]
pub enum FetchSrcRockError {
    DownloadSrcRock(#[from] DownloadSrcRockError),
    Unpack(#[from] UnpackError),
    Io(#[from] io::Error),
}

/// A no-prompt implementer for auth_git2's prompter
#[derive(Copy, Clone, Debug)]
struct NullPrompter;

impl Prompter for NullPrompter {
    fn prompt_username_password(&mut self, _: &str, _: &git2::Config) -> Option<(String, String)> {
        None
    }

    fn prompt_password(&mut self, _: &str, _: &str, _: &git2::Config) -> Option<String> {
        None
    }

    fn prompt_ssh_key_passphrase(&mut self, _: &Path, _: &git2::Config) -> Option<String> {
        None
    }
}

/// Initialises and checks out git submodules,
/// returning the relative paths of any submodules that were checked out.
fn init_submodules(
    repo: &git2::Repository,
    auth: &GitAuthenticator,
    git_config: &git2::Config,
) -> Result<Vec<PathBuf>, FetchSrcError> {
    let workdir = repo
        .workdir()
        .ok_or_else(|| git2::Error::from_str("cloned repository has no working directory"))?;
    let mut submodule_paths = Vec::new();
    for mut submodule in repo.submodules()? {
        let mut callbacks = RemoteCallbacks::new();
        callbacks.credentials(auth.credentials(git_config));
        let mut fetch_options = FetchOptions::new();
        fetch_options.remote_callbacks(callbacks);
        let mut options = git2::SubmoduleUpdateOptions::new();
        options.fetch(fetch_options);
        submodule.update(true, Some(&mut options))?;
        // The .git file/directory is not deterministic
        let git_file = workdir.join(submodule.path()).join(".git");
        if git_file.is_dir() {
            fs::sync::remove_dir_all(&git_file)?;
        } else {
            fs::sync::remove_file(&git_file)?;
        }
        submodule_paths.push(submodule.path().to_path_buf());
    }
    Ok(submodule_paths)
}

#[tracing::instrument(
    name = "Fetching source",
    level = "info",
    skip_all,
    fields(location = source_spec.to_string()),
)]
async fn fetch_src_impl<R: Rockspec>(
    mut source_spec: RockSourceSpec,
    rockspec: &R,
    rock_source: &RemoteRockSource,
    dest_dir: &Path,
    config: &Config,
) -> Result<RemotePackageSourceMetadata, FetchSrcError> {
    if let Some(vendor_dir) = config.vendor_dir() {
        source_spec = match source_spec {
            // could be a project directory (not vendored) or a local source
            // or a vendored dependency that we have already resolved
            RockSourceSpec::File(_) => source_spec,
            _ => {
                let pkg_vendor_dir =
                    vendor_dir.join(format!("{}@{}", rockspec.package(), rockspec.version()));
                RockSourceSpec::File(pkg_vendor_dir)
            }
        }
    }
    let metadata = match &source_spec {
        RockSourceSpec::Git(git) => {
            let url = git.url.to_string();
            tracing::debug!(message = format!("Cloning {url}").as_str());

            let clone_url = url.clone();
            let clone_dir = dest_dir.to_path_buf();
            let host = git.url.host().to_string();
            let no_prompt = config.no_prompt();
            let access_token = config.access_token(&host);
            let git_ref = git.git_ref.clone();

            let (checkout_ref, has_submodules) = tokio::task::spawn_blocking(move || {
                let mut auth = if no_prompt {
                    GitAuthenticator::default()
                        .try_password_prompt(0)
                        .prompt_ssh_key_password(false)
                        .set_prompter(NullPrompter)
                } else {
                    GitAuthenticator::default()
                };
                if let Some(access_token) = access_token {
                    auth = auth.add_plaintext_credentials(&host, access_token.username(), unsafe {
                        access_token.password()
                    });
                }
                let git_config = git2::Config::open_default()?;
                let mut callbacks = RemoteCallbacks::new();
                callbacks.credentials(auth.credentials(&git_config));
                let mut fetch_options = FetchOptions::new();
                fetch_options.update_fetchhead(false);
                fetch_options.remote_callbacks(callbacks);
                let checkout_ref = match &git_ref {
                    Some(GitRef::Tag(tag)) => Some(tag.as_str()),
                    Some(GitRef::Branch(branch)) => Some(branch.as_str()),
                    None => None,
                };
                if checkout_ref.is_none() {
                    fetch_options.depth(1);
                };
                let mut repo_builder = RepoBuilder::new();
                repo_builder.fetch_options(fetch_options);
                let repo = repo_builder.clone(&clone_url, &clone_dir)?;

                let checkout_ref = match checkout_ref {
                    Some(checkout_ref) => {
                        let ref_name = if let Some(GitRef::Branch(branch)) = &git_ref {
                            format!("origin/{branch}")
                        } else {
                            checkout_ref.to_string()
                        };
                        let (object, _) = repo.revparse_ext(&ref_name)?;
                        repo.checkout_tree(&object, None)?;
                        checkout_ref.to_string()
                    }
                    None => {
                        let head = repo.head()?;
                        let commit = head.peel_to_commit()?;
                        commit.id().to_string()
                    }
                };

                let submodule_paths = init_submodules(&repo, &auth, &git_config)?;

                Ok::<_, FetchSrcError>((checkout_ref, !submodule_paths.is_empty()))
            })
            .await??;

            // The .git directory is not deterministic
            remove_dir_all(dest_dir.join(".git")).map_err(FetchSrcError::CleanGitDir)?;
            let hash = dest_dir.hash().await.map_err(FetchSrcError::Hash)?;
            RemotePackageSourceMetadata {
                hash,
                source_url: RemotePackageSourceUrl::Git {
                    url,
                    checkout_ref,
                    submodules: has_submodules,
                },
            }
        }
        RockSourceSpec::Url(url) => {
            tracing::debug!(message = format!("Downloading {url}").as_str());

            // NOTE: We don't enforce HTTPS when fetching sources because some rockspecs
            // have HTTP URLs in `source.url`.
            let response = crate::reqwest::http_client(config)?
                .get(url.clone())
                .apply_access_token(config, url)
                .send()
                .await?
                .error_for_status()?
                .bytes()
                .await?;
            let hash = response.hash().await.map_err(FetchSrcError::Hash)?;
            let file_name = url
                .path_segments()
                .and_then(|mut segments| segments.next_back())
                .and_then(|name| {
                    if name.is_empty() {
                        None
                    } else {
                        Some(name.to_string())
                    }
                })
                .unwrap_or(url.to_string());
            let cursor = Cursor::new(response);
            let mime_type = infer::get(cursor.get_ref()).map(|file_type| file_type.mime_type());
            operations::unpack::unpack(
                mime_type,
                cursor,
                rock_source.unpack_dir.is_none(),
                file_name,
                dest_dir,
            )
            .await?;
            RemotePackageSourceMetadata {
                hash,
                source_url: RemotePackageSourceUrl::Url { url: url.clone() },
            }
        }
        RockSourceSpec::File(path) => {
            tracing::debug!(message = format!("Copying {}", path.display()).as_str());

            let hash = if path.is_dir() {
                let hash = path.hash().await.map_err(FetchSrcError::Hash)?;
                recursive_copy_dir(path, dest_dir).await?;
                hash
            } else {
                let mut file = fs::sync::open(path)?;
                let mut buffer = Vec::new();
                file.read_to_end(&mut buffer)
                    .map_err(|source| fs::FsError::Read {
                        path: path.to_path_buf(),
                        source,
                    })?;
                let mime_type = infer::get(&buffer).map(|file_type| file_type.mime_type());
                let file_name = path
                    .file_name()
                    .map(|os_str| os_str.to_string_lossy())
                    .unwrap_or(path.to_string_lossy())
                    .to_string();
                operations::unpack::unpack(
                    mime_type,
                    file,
                    rock_source.unpack_dir.is_none(),
                    file_name,
                    dest_dir,
                )
                .await?;
                path.hash().await.map_err(FetchSrcError::Hash)?
            };
            RemotePackageSourceMetadata {
                hash,
                source_url: RemotePackageSourceUrl::File { path: path.clone() },
            }
        }
    };
    Ok(metadata)
}

#[tracing::instrument(
    name = "Fetching src.rock",
    level = "info",
    skip_all,
    fields(package = fetch.package.to_string(),),
)]
async fn do_fetch_src_rock(
    fetch: FetchSrcRock<'_>,
) -> Result<RemotePackageSourceMetadata, FetchSrcRockError> {
    let package = fetch.package;
    let dest_dir = fetch.dest_dir;
    let config = fetch.config;
    let src_rock = operations::download_src_rock(package, config.server(), fetch.config).await?;
    let hash = src_rock.bytes.hash().await?;
    let cursor = Cursor::new(src_rock.bytes);
    let mime_type = infer::get(cursor.get_ref()).map(|file_type| file_type.mime_type());
    operations::unpack::unpack(mime_type, cursor, true, src_rock.file_name, dest_dir).await?;
    Ok(RemotePackageSourceMetadata {
        hash,
        source_url: RemotePackageSourceUrl::Url { url: src_rock.url },
    })
}