lux_lib/operations/

download.rs

use crate::package;
use std::{
    collections::HashMap,
    io::{Cursor, Read},
    path::PathBuf,
    string::FromUtf8Error,
};

use bon::Builder;
use bytes::Bytes;
use miette::Diagnostic;
use thiserror::Error;
use url::{ParseError, Url};

use crate::{
    config::Config,
    fs,
    git::{GitRef, GitSource},
    lockfile::RemotePackageSourceUrl,
    lua_rockspec::{self, RemoteLuaRockspec, RockSourceSpec},
    luarocks,
    package::{RemotePackage, RemotePackageTypeFilterSpec},
    package_db::{self, PackageDB, SearchError},
    remote_package_source::RemotePackageSource,
    reqwest::{RequestBuilderExt, RequestError},
    rockspec::Rockspec,
};

/// Builder for a rock downloader.
pub struct Download<'a> {
    package_req: &'a package::Req,
    package_db: Option<&'a PackageDB>,
    config: &'a Config,
}

impl<'a> Download<'a> {
    /// Construct a new `.src.rock` downloader.
    pub fn new(package_req: &'a package::Req, config: &'a Config) -> Self {
        Self {
            package_req,
            package_db: None,
            config,
        }
    }

    /// Sets the package database to use for searching for packages.
    /// Instantiated from the config if not set.
    pub fn package_db(self, package_db: &'a PackageDB) -> Self {
        Self {
            package_db: Some(package_db),
            ..self
        }
    }

    /// Download the package's Rockspec.
    pub async fn download_rockspec(self) -> Result<DownloadedRockspec, SearchAndDownloadError> {
        match self.package_db {
            Some(db) => download_rockspec(self.package_req, db, self.config).await,
            None => {
                let db = PackageDB::from_config(self.config).await?;
                download_rockspec(self.package_req, &db, self.config).await
            }
        }
    }

    /// Download a `.src.rock` to a file.
    /// `destination_dir` defaults to the current working directory if not set.
    pub async fn download_src_rock_to_file(
        self,
        destination_dir: Option<PathBuf>,
    ) -> Result<DownloadedPackedRock, SearchAndDownloadError> {
        match self.package_db {
            Some(db) => {
                download_src_rock_to_file(self.package_req, destination_dir, db, self.config).await
            }
            None => {
                let db = PackageDB::from_config(self.config).await?;
                download_src_rock_to_file(self.package_req, destination_dir, &db, self.config).await
            }
        }
    }

    /// Search for a `.src.rock` and download it to memory.
    pub async fn search_and_download_src_rock(
        self,
    ) -> Result<DownloadedPackedRockBytes, SearchAndDownloadError> {
        match self.package_db {
            Some(db) => search_and_download_src_rock(self.package_req, db, self.config).await,
            None => {
                let db = PackageDB::from_config(self.config).await?;
                search_and_download_src_rock(self.package_req, &db, self.config).await
            }
        }
    }

    pub(crate) async fn download_remote_rock(
        self,
    ) -> Result<RemoteRockDownload, SearchAndDownloadError> {
        match self.package_db {
            Some(db) => download_remote_rock(self.package_req, db, self.config).await,
            None => {
                let db = PackageDB::from_config(self.config).await?;
                download_remote_rock(self.package_req, &db, self.config).await
            }
        }
    }
}

pub struct DownloadedPackedRockBytes {
    pub name: package::Name,
    pub version: package::Version,
    pub bytes: Bytes,
    pub file_name: String,
    pub url: Url,
}

pub struct DownloadedPackedRock {
    pub name: package::Name,
    pub version: package::Version,
    pub path: PathBuf,
}

/// Remote Lua RockSpec that has been downloaded from a remote server, along with its source metadata
#[derive(Clone, Debug)]
pub struct DownloadedRockspec {
    pub rockspec: RemoteLuaRockspec,
    pub(crate) source: RemotePackageSource,
    pub(crate) source_url: Option<RemotePackageSourceUrl>,
}

#[derive(Clone, Debug)]
pub(crate) enum RemoteRockDownload {
    RockspecOnly {
        rockspec_download: DownloadedRockspec,
    },
    BinaryRock {
        rockspec_download: DownloadedRockspec,
        packed_rock: Bytes,
    },
    SrcRock {
        rockspec_download: DownloadedRockspec,
        src_rock: Bytes,
        source_url: RemotePackageSourceUrl,
    },
}

impl RemoteRockDownload {
    pub fn rockspec(&self) -> &RemoteLuaRockspec {
        &self.rockspec_download().rockspec
    }
    pub fn rockspec_download(&self) -> &DownloadedRockspec {
        match self {
            Self::RockspecOnly { rockspec_download }
            | Self::BinaryRock {
                rockspec_download, ..
            }
            | Self::SrcRock {
                rockspec_download, ..
            } => rockspec_download,
        }
    }

    fn into_rockspec_download(self) -> DownloadedRockspec {
        match self {
            Self::RockspecOnly { rockspec_download }
            | Self::BinaryRock {
                rockspec_download, ..
            }
            | Self::SrcRock {
                rockspec_download, ..
            } => rockspec_download,
        }
    }

    // Instead of downloading a rockspec, generate one from a `Req` and a `RockSourceSpec`.
    pub(crate) fn from_package_req_and_source_spec(
        package_req: package::Req,
        source_spec: RockSourceSpec,
    ) -> Result<Self, SearchAndDownloadError> {
        let package_spec: package::Spec = package_req.try_into()?;
        let source_url = Some(match &source_spec {
            RockSourceSpec::Git(GitSource { url, git_ref }) => {
                let checkout_ref = match git_ref {
                    Some(GitRef::Tag(tag)) => tag.clone(),
                    Some(GitRef::Branch(branch)) => branch.clone(),
                    None => {
                        return Err(SearchAndDownloadError::MissingCheckoutRef(url.to_string()))
                    }
                };
                RemotePackageSourceUrl::Git {
                    url: url.to_string(),
                    checkout_ref,
                    submodules: false, // unknown until the source is fetched
                }
            }
            RockSourceSpec::File(path) => RemotePackageSourceUrl::File { path: path.clone() },
            RockSourceSpec::Url(url) => RemotePackageSourceUrl::Url { url: url.clone() },
            RockSourceSpec::Wally(_) => {
                return Err(SearchAndDownloadError::WallySource(
                    package_spec.name().to_string(),
                ))
            }
        });
        let rockspec = RemoteLuaRockspec::from_source_spec(
            package_spec,
            source_spec,
            Vec::new(),
            HashMap::new(),
        );
        let rockspec_content = rockspec
            .to_lua_remote_rockspec_string()
            .expect("the infallible happened");
        let rockspec_download = DownloadedRockspec {
            rockspec,
            source_url,
            source: RemotePackageSource::RockspecContent(rockspec_content),
        };
        Ok(Self::RockspecOnly { rockspec_download })
    }
}

#[derive(Error, Debug, Diagnostic)]
pub enum DownloadRockspecError {
    #[error("failed to download rockspec")]
    #[diagnostic(transparent)]
    Request(#[from] RequestError),
    #[error("failed to convert rockspec response")]
    #[diagnostic(help(
        r#"the server returned a response that is not valid UTF-8.
check your network connection and server configuration.
if the issue persists, the server may be temporarily unavailable."#
    ))]
    ResponseConversion(#[from] FromUtf8Error),
    #[error("error initialising remote package DB")]
    #[diagnostic(forward(0))]
    PackageDB(#[from] package_db::Error),
    #[error(transparent)]
    #[diagnostic(transparent)]
    DownloadSrcRock(#[from] DownloadSrcRockError),
}

impl From<reqwest::Error> for DownloadRockspecError {
    fn from(err: reqwest::Error) -> Self {
        Self::Request(err.into())
    }
}

/// Find and download a rockspec for a given package requirement
async fn download_rockspec(
    package_req: &package::Req,
    package_db: &PackageDB,
    config: &Config,
) -> Result<DownloadedRockspec, SearchAndDownloadError> {
    Ok(download_remote_rock(package_req, package_db, config)
        .await?
        .into_rockspec_download())
}

pub(crate) async fn download_exact_rockspec(
    package: &package::Spec,
    package_db: &PackageDB,
    config: &Config,
) -> Result<DownloadedRockspec, SearchAndDownloadError> {
    let package_req = package.clone().into();
    let remote = package_db.find(&package_req, None).await?;
    let remote_package = RemotePackage {
        spec: package.clone(),
        ..remote
    };
    Ok(download_found_rock(remote_package, config)
        .await?
        .into_rockspec_download())
}

#[tracing::instrument(
    name = "Downloading rock",
    level = "info",
    skip_all,
    fields(package = package_req.to_string(),),
)]
async fn download_remote_rock(
    package_req: &package::Req,
    package_db: &PackageDB,
    config: &Config,
) -> Result<RemoteRockDownload, SearchAndDownloadError> {
    let remote_package = package_db.find(package_req, None).await?;
    download_found_rock(remote_package, config).await
}

async fn download_found_rock(
    remote_package: RemotePackage,
    config: &Config,
) -> Result<RemoteRockDownload, SearchAndDownloadError> {
    match &remote_package.source {
        RemotePackageSource::LuarocksRockspec(url) => {
            let package = &remote_package.spec;
            let rockspec_name = format!("{}-{}.rockspec", package.name(), package.version());
            let url = url
                .join(&rockspec_name)
                .map_err(|source| SearchAndDownloadError::Parse {
                    source,
                    url: format!("{}/{}", url, rockspec_name),
                })?;
            let bytes = crate::reqwest::https_client(config)?
                .get(url.clone())
                .apply_access_token(config, &url)
                .send()
                .await?
                .error_for_status()?
                .bytes()
                .await?;
            let content = String::from_utf8(bytes.into())?;
            let rockspec = DownloadedRockspec {
                rockspec: RemoteLuaRockspec::new(&content)
                    .map_err(|err| SearchAndDownloadError::Rockspec(Box::new(err)))?,
                source: remote_package.source,
                source_url: remote_package.source_url,
            };
            Ok(RemoteRockDownload::RockspecOnly {
                rockspec_download: rockspec,
            })
        }
        RemotePackageSource::RockspecContent(content) => {
            let rockspec = DownloadedRockspec {
                rockspec: RemoteLuaRockspec::new(content)
                    .map_err(|err| SearchAndDownloadError::Rockspec(Box::new(err)))?,
                source: remote_package.source,
                source_url: remote_package.source_url,
            };
            Ok(RemoteRockDownload::RockspecOnly {
                rockspec_download: rockspec,
            })
        }
        RemotePackageSource::LuarocksBinaryRock(url) => {
            // prioritise lockfile source_url
            let url = if let Some(RemotePackageSourceUrl::Url { url }) = &remote_package.source_url
            {
                url
            } else {
                url
            };
            let rock = download_binary_rock(&remote_package.spec, url, config).await?;
            let rockspec = DownloadedRockspec {
                rockspec: unpack_rockspec(&rock).await?,
                source: remote_package.source,
                source_url: remote_package.source_url,
            };
            Ok(RemoteRockDownload::BinaryRock {
                rockspec_download: rockspec,
                packed_rock: rock.bytes,
            })
        }
        RemotePackageSource::LuarocksSrcRock(url) => {
            // prioritise lockfile source_url
            let url = if let Some(RemotePackageSourceUrl::Url { url }) = &remote_package.source_url
            {
                url.clone()
            } else {
                url.clone()
            };
            let rock = download_src_rock(&remote_package.spec, &url, config).await?;
            let rockspec = DownloadedRockspec {
                rockspec: unpack_rockspec(&rock).await?,
                source: remote_package.source,
                source_url: remote_package.source_url,
            };
            Ok(RemoteRockDownload::SrcRock {
                rockspec_download: rockspec,
                src_rock: rock.bytes,
                source_url: RemotePackageSourceUrl::Url { url },
            })
        }
        RemotePackageSource::Wally(_) => Err(SearchAndDownloadError::WallySource(
            remote_package.spec.name().to_string(),
        )),
        RemotePackageSource::Local => Err(SearchAndDownloadError::LocalSource),
        #[cfg(test)]
        RemotePackageSource::Test => unimplemented!(),
    }
}

#[derive(Error, Debug, Diagnostic)]
pub enum SearchAndDownloadError {
    #[error("failed to parse rock URL '{url}'")]
    Parse {
        source: url::ParseError,
        url: String,
    },
    #[error(transparent)]
    #[diagnostic(transparent)]
    Search(#[from] SearchError),
    #[error(transparent)]
    #[diagnostic(transparent)]
    Download(#[from] DownloadSrcRockError),
    #[error(transparent)]
    #[diagnostic(transparent)]
    DownloadRockspec(#[from] DownloadRockspecError),
    #[error(transparent)]
    #[diagnostic(transparent)]
    Fs(#[from] fs::Error),
    #[error("UTF-8 conversion failed")]
    #[diagnostic(help(
        r#"the server returned a response that is not valid UTF-8.
check your network connection and server configuration.
if the issue persists, the server may be temporarily unavailable."#
    ))]
    Utf8(#[from] FromUtf8Error),
    #[error(transparent)]
    #[diagnostic(transparent)]
    Rockspec(Box<lua_rockspec::Error>),
    #[error("error initialising remote package DB")]
    #[diagnostic(forward(0))]
    PackageDB(#[from] package_db::Error),
    #[error("failed to read packed rock {0}:\n{1}")]
    #[diagnostic(help(
        r#"the downloaded rock may be corrupted.
check your network connection and rerun the command to re-download it."#
    ))]
    ZipRead(String, zip::result::ZipError),
    #[error("failed to extract packed rock {0}:\n{1}")]
    #[diagnostic(help(
        r#"the downloaded rock may be corrupted.
check your network connection and rerun the command to re-download it."#
    ))]
    ZipExtract(String, zip::result::ZipError),
    #[error("{0} not found in the packed rock.")]
    #[diagnostic(help(
        r#"the packed rock does not contain the expected rockspec.
check your network connection and rerun the command to re-download it."#
    ))]
    RockspecNotFoundInPackedRock(String),
    #[error(transparent)]
    #[diagnostic(transparent)]
    PackageSpecFromPackageReq(#[from] crate::package::SpecFromReqError),
    #[error("git source {0} without a revision or tag.")]
    #[diagnostic(help(
        r#"lux requires a pinned revision or tag to ensure reproducible builds.
without one, the same version could resolve to different code at different times.
try a different version, or report it to the package maintainer."#
    ))]
    MissingCheckoutRef(String),
    #[error("cannot download from a local rock source.")]
    #[diagnostic(
        help(
            r#"lux cannot download rocks from a local path.
for local dependencies, use `path` in your lux.toml."#
        ),
        url("https://lux.lumen-labs.org/reference/lux-toml#local-dependencies")
    )]
    LocalSource,
    #[error("cannot download from a local rock or embedded rockspec source.")]
    #[diagnostic(
        help(
            r#"lux cannot download rocks from a local path.
for local dependencies, use `path` in your lux.toml."#
        ),
        url("https://lux.lumen-labs.org/reference/lux-toml#local-dependencies")
    )]
    NonURLSource,
    #[error("failed to resolve wally package '{0}'")]
    #[diagnostic(
        code(lux_lib::wally::unresolved_source),
        help("this is a bug, please report it.")
    )]
    WallySource(String),
    #[error("client error")]
    #[diagnostic(transparent)]
    Request(#[from] RequestError),
}

impl From<reqwest::Error> for SearchAndDownloadError {
    fn from(err: reqwest::Error) -> Self {
        Self::Request(err.into())
    }
}

async fn search_and_download_src_rock(
    package_req: &package::Req,
    package_db: &PackageDB,
    config: &Config,
) -> Result<DownloadedPackedRockBytes, SearchAndDownloadError> {
    let filter = Some(RemotePackageTypeFilterSpec {
        rockspec: false,
        binary: false,
        src: true,
    });
    let remote_package = package_db.find(package_req, filter).await?;
    let source_url = remote_package
        .source
        .url()
        .ok_or(SearchAndDownloadError::NonURLSource)?;
    Ok(download_src_rock(&remote_package.spec, &source_url, config).await?)
}

#[derive(Error, Debug, Diagnostic)]
pub enum DownloadSrcRockError {
    #[error("failed to download source rock")]
    #[diagnostic(transparent)]
    Request(#[from] RequestError),
    #[error("failed to parse source rock URL")]
    Parse(#[from] ParseError),
}

impl From<reqwest::Error> for DownloadSrcRockError {
    fn from(err: reqwest::Error) -> Self {
        Self::Request(err.into())
    }
}

#[tracing::instrument(name = "Downloading src.rock", skip_all)]
pub(crate) async fn download_src_rock(
    package: &package::Spec,
    server_url: &Url,
    config: &Config,
) -> Result<DownloadedPackedRockBytes, DownloadSrcRockError> {
    ArchiveDownload::new()
        .package(package)
        .server_url(server_url)
        .config(config)
        .ext("src.rock")
        .download()
        .await
}

#[tracing::instrument(name = "Downloading binary rock", skip_all)]
pub(crate) async fn download_binary_rock(
    package: &package::Spec,
    server_url: &Url,
    config: &Config,
) -> Result<DownloadedPackedRockBytes, DownloadSrcRockError> {
    let ext = format!("{}.rock", luarocks::current_platform_luarocks_identifier());
    ArchiveDownload::new()
        .package(package)
        .server_url(server_url)
        .config(config)
        .ext(&ext)
        .fallback_ext("all.rock")
        .download()
        .await
}

#[tracing::instrument(name = "Downloading package", skip_all)]
async fn download_src_rock_to_file(
    package_req: &package::Req,
    destination_dir: Option<PathBuf>,
    package_db: &PackageDB,
    config: &Config,
) -> Result<DownloadedPackedRock, SearchAndDownloadError> {
    let rock = search_and_download_src_rock(package_req, package_db, config).await?;
    let full_rock_name = mk_packed_rock_name(&rock.name, &rock.version, "src.rock");
    fs::tokio::write(
        destination_dir
            .map(|dest| dest.join(&full_rock_name))
            .unwrap_or_else(|| full_rock_name.clone().into()),
        &rock.bytes,
    )
    .await?;

    Ok(DownloadedPackedRock {
        name: rock.name.to_owned(),
        version: rock.version.to_owned(),
        path: full_rock_name.into(),
    })
}

#[derive(Builder)]
#[builder(start_fn = new, finish_fn(name = _build, vis = ""))]
struct ArchiveDownload<'a> {
    package: &'a package::Spec,

    server_url: &'a Url,

    ext: &'a str,

    fallback_ext: Option<&'a str>,

    config: &'a Config,
}

impl<State> ArchiveDownloadBuilder<'_, State>
where
    State: archive_download_builder::State + archive_download_builder::IsComplete,
{
    async fn download(self) -> Result<DownloadedPackedRockBytes, DownloadSrcRockError> {
        let args = self._build();
        download_impl(args).await
    }
}

#[tracing::instrument(
    name = "Downloading",
    level = "info",
    skip_all,
    fields(
        package = args.package.name().to_string(),
        version = args.package.version().to_string(),
        server = args.server_url.to_string(),
    ),
)]
async fn download_impl(
    args: ArchiveDownload<'_>,
) -> Result<DownloadedPackedRockBytes, DownloadSrcRockError> {
    let package = args.package;
    let ext = args.ext;
    let server_url = args.server_url;
    let full_rock_name = mk_packed_rock_name(package.name(), package.version(), ext);
    tracing::debug!(message = format!("📥 Downloading {full_rock_name}").as_str());
    let url = server_url.join(&full_rock_name)?;
    let response = crate::reqwest::https_client(args.config)?
        .get(url.clone())
        .apply_access_token(args.config, &url)
        .send()
        .await?;
    let bytes = if response.status().is_success() {
        response.bytes().await
    } else {
        match args.fallback_ext {
            Some(ext) => {
                let full_rock_name = mk_packed_rock_name(package.name(), package.version(), ext);
                let url = server_url.join(&full_rock_name)?;
                crate::reqwest::https_client(args.config)?
                    .get(url.clone())
                    .apply_access_token(args.config, &url)
                    .send()
                    .await?
                    .error_for_status()?
                    .bytes()
                    .await
            }
            None => response.error_for_status()?.bytes().await,
        }
    }?;
    Ok(DownloadedPackedRockBytes {
        name: package.name().clone(),
        version: package.version().clone(),
        bytes,
        file_name: full_rock_name,
        url,
    })
}
fn mk_packed_rock_name(name: &package::Name, version: &package::Version, ext: &str) -> String {
    format!("{name}-{version}.{ext}")
}

pub(crate) async fn unpack_rockspec(
    rock: &DownloadedPackedRockBytes,
) -> Result<RemoteLuaRockspec, SearchAndDownloadError> {
    let cursor = Cursor::new(&rock.bytes);
    let rockspec_file_name = format!("{}-{}.rockspec", rock.name, rock.version);
    let mut zip = zip::ZipArchive::new(cursor)
        .map_err(|err| SearchAndDownloadError::ZipRead(rock.file_name.clone(), err))?;
    let rockspec_index = (0..zip.len())
        .find(|&i| {
            unsafe { zip.by_index(i).unwrap_unchecked() }
                .name()
                .is_ok_and(|name| name.eq(&rockspec_file_name))
        })
        .ok_or(SearchAndDownloadError::RockspecNotFoundInPackedRock(
            rockspec_file_name.to_string(),
        ))?;
    let mut rockspec_file = zip
        .by_index(rockspec_index)
        .map_err(|err| SearchAndDownloadError::ZipExtract(rock.file_name.clone(), err))?;
    let mut content = String::new();
    rockspec_file
        .read_to_string(&mut content)
        .map_err(|source| fs::Error::ReadToString {
            path: PathBuf::from(&rockspec_file_name),
            source,
        })?;
    let rockspec = RemoteLuaRockspec::new(&content)
        .map_err(|err| SearchAndDownloadError::Rockspec(Box::new(err)))?;
    Ok(rockspec)
}