lux_lib/pipeline/

materialise.rs

use std::collections::HashMap;
use std::io;
use std::io::Cursor;
use std::path::{Path, PathBuf};

use bon::Builder;
use bytes::Bytes;
use futures::{StreamExt, TryStreamExt};
use miette::Diagnostic;
use tempfile::TempDir;
use thiserror::Error;

use crate::{
    build::{BuildError, RemotePackageSourceSpec, SrcRockSource},
    config::Config,
    fs::{self, FsError},
    hash::HasIntegrity,
    lockfile::{
        LockedPackage, LockedPackageHashes, LockedPackageId, LockedPackageSpec,
        RemotePackageSourceUrl,
    },
    lua_rockspec::{RemoteLuaRockspec, RockSourceSpec},
    operations::{
        single_non_rockspec_entry, unpack_src_rock, FetchSrc, RemotePackageSourceMetadata,
    },
    remote_package_source::RemotePackageSource,
    rockspec::Rockspec,
    tree::EntryType,
};

use super::{
    resolve::{ResolvedArtifacts, ResolvedPackage},
    Artifacts,
};

#[derive(Clone, Debug)]
pub(crate) enum PackageSource {
    SourceDir(PathBuf),
    Archive(Bytes),
}

pub(crate) struct MaterialisedPackage {
    pub(crate) package: LockedPackage,
    pub(crate) rockspec: RemoteLuaRockspec,
    pub(crate) entry_type: EntryType,
    pub(crate) build_behaviour: crate::build::BuildBehaviour,
    pub(crate) artifact: PackageSource,
    pub(crate) temp_dir: Option<TempDir>,
}

pub(crate) type MaterialiseArtifacts = Artifacts<HashMap<LockedPackageId, MaterialisedPackage>>;

#[derive(Error, Debug, Diagnostic)]
#[non_exhaustive]
pub enum Error {
    #[error(transparent)]
    #[diagnostic(transparent)]
    Fs(#[from] FsError),
    #[error(transparent)]
    #[diagnostic(transparent)]
    Build(#[from] Box<BuildError>),
    #[error("failed to hash the source for '{0}'")]
    Hash(String, #[source] io::Error),
    #[error("missing source url for '{0}'")]
    MissingSourceUrl(String),
}

impl From<BuildError> for Error {
    fn from(source: BuildError) -> Self {
        Self::Build(Box::new(source))
    }
}

#[derive(Builder)]
#[builder(start_fn = new, finish_fn(name = _build, vis = ""))]
pub(crate) struct Materialise<'a> {
    #[builder(start_fn)]
    pub(crate) config: &'a Config,
    pub(crate) resolved: ResolvedArtifacts,
}

impl<State> MaterialiseBuilder<'_, State>
where
    State: materialise_builder::State + materialise_builder::IsComplete,
{
    pub(crate) async fn materialise(self) -> Result<MaterialiseArtifacts, Error> {
        let args = self._build();
        let config = args.config;

        let artifacts: MaterialiseArtifacts = args
            .resolved
            .iter()
            .map(|(section, packages)| (section, packages.map(|_| HashMap::new())))
            .collect();

        let jobs = args.resolved.into_iter().flat_map(|(section, packages)| {
            packages
                .into_iter()
                .flatten()
                .map(move |(id, package)| (section, id, package))
        });

        futures::stream::iter(jobs)
            .map(|(section, id, package)| async move {
                materialise(config, package)
                    .await
                    .map(|package| (section, id, package))
            })
            .buffered(config.max_jobs())
            .try_fold(
                artifacts,
                |mut artifacts, (section, id, package)| async move {
                    artifacts
                        .get_mut(section)
                        .get_or_insert_with(HashMap::new)
                        .insert(id, package);
                    Ok(artifacts)
                },
            )
            .await
    }
}

async fn materialise(
    config: &Config,
    resolved: ResolvedPackage,
) -> Result<MaterialisedPackage, Error> {
    let ResolvedPackage {
        spec,
        rockspec,
        source,
        source_url,
        entry_type,
        build_behaviour,
        artifact,
    } = resolved;

    let name = spec.name().to_string();

    let (package, artifact, temp_dir) = match artifact {
        Some(PackageSource::Archive(bytes)) => match &source {
            RemotePackageSource::LuarocksBinaryRock(url) => {
                let binary_url = url.clone();
                let rockspec_hash = rockspec
                    .hash()
                    .await
                    .map_err(|err| Error::Hash(name.clone(), err))?;
                let source_hash = bytes
                    .hash()
                    .await
                    .map_err(|err| Error::Hash(name.clone(), err))?;
                let hashes = LockedPackageHashes {
                    rockspec: rockspec_hash,
                    source: source_hash,
                };
                let package = LockedPackage::new(
                    spec,
                    source,
                    Some(RemotePackageSourceUrl::Url { url: binary_url }),
                    hashes,
                );
                (package, PackageSource::Archive(bytes), None)
            }
            _ => {
                let source_url = source_url
                    .clone()
                    .ok_or_else(|| Error::MissingSourceUrl(name.clone()))?;
                let source_spec =
                    RemotePackageSourceSpec::SrcRock(SrcRockSource { bytes, source_url });
                let (package, dir, source_root) =
                    fetch_and_hash_source(&rockspec, spec, Some(source_spec), Some(source), config)
                        .await?;
                let artifact = PackageSource::SourceDir(source_root);
                (package, artifact, Some(dir))
            }
        },
        Some(PackageSource::SourceDir(path)) => {
            let rockspec_hash = rockspec
                .hash()
                .await
                .map_err(|err| Error::Hash(name.clone(), err))?;
            let source_hash = path
                .hash()
                .await
                .map_err(|err| Error::Hash(name.clone(), err))?;
            let hashes = LockedPackageHashes {
                rockspec: rockspec_hash,
                source: source_hash,
            };
            let package = LockedPackage::new(spec, source, source_url, hashes);
            (package, PackageSource::SourceDir(path), None)
        }
        None => {
            let source_spec = RemotePackageSourceSpec::RockSpec(source_url);
            let (package, dir, source_root) =
                fetch_and_hash_source(&rockspec, spec, Some(source_spec), Some(source), config)
                    .await?;
            let artifact = PackageSource::SourceDir(source_root);
            (package, artifact, Some(dir))
        }
    };

    Ok(MaterialisedPackage {
        package,
        rockspec,
        entry_type,
        build_behaviour,
        artifact,
        temp_dir,
    })
}

fn is_src_rock_url(source_url: &RemotePackageSourceUrl) -> bool {
    match source_url {
        RemotePackageSourceUrl::Url { url } => url.path().ends_with(".src.rock"),
        RemotePackageSourceUrl::File { path } => path.to_string_lossy().ends_with(".src.rock"),
        RemotePackageSourceUrl::Git { .. } => false,
    }
}

/// Resolves the source root inside an unpacked source rock.
///
/// The source root is `<unpacked>/<repository>` when that directory is the only
/// entry besides the rockspec. Otherwise the source root is the unpacked root.
///
/// This matches the wrapper directory luarocks adds when packing a Git source rock,
/// while leaving unwrapped source trees untouched.
///
/// For a git source rock containing `foo/`:
///
/// | unpacked contents        | returned source root |
/// |--------------------------|----------------------|
/// | `foo/` and the rockspec  | `<unpacked>/foo`     |
/// | source files at the root | `<unpacked>`         |
fn src_rock_source_root<R: Rockspec>(rockspec: &R, unpacked_root: &Path) -> PathBuf {
    if let RockSourceSpec::Git(git) = &rockspec.source().current_platform().source_spec {
        let repo_dir = unpacked_root.join(git.url.repo());
        if let Ok(Some(entry)) = single_non_rockspec_entry(unpacked_root) {
            if entry == repo_dir && entry.is_dir() {
                return repo_dir;
            }
        }
    }
    unpacked_root.to_path_buf()
}

pub(crate) async fn fetch_and_hash_source<R: Rockspec + HasIntegrity>(
    rockspec: &R,
    spec: LockedPackageSpec,
    source_spec: Option<RemotePackageSourceSpec>,
    source: Option<RemotePackageSource>,
    config: &Config,
) -> Result<(LockedPackage, TempDir, PathBuf), BuildError> {
    let temp_dir = fs::tempfile::tempdir()?;
    let is_src_rock = match &source_spec {
        Some(RemotePackageSourceSpec::SrcRock(_)) => true,
        Some(RemotePackageSourceSpec::RockSpec(Some(source_url))) => {
            config.vendor_dir().is_none() && is_src_rock_url(source_url)
        }
        _ => false,
    };

    let source_metadata = match source_spec {
        Some(RemotePackageSourceSpec::SrcRock(SrcRockSource { bytes, source_url })) => {
            let hash = bytes.hash().await?;
            let cursor = Cursor::new(bytes);
            unpack_src_rock(cursor, temp_dir.path().to_path_buf())
                .await
                .map_err(BuildError::UnpackSrcRock)?;
            RemotePackageSourceMetadata { hash, source_url }
        }
        Some(RemotePackageSourceSpec::RockSpec(source_url)) => {
            FetchSrc::new(temp_dir.path(), rockspec, config)
                .maybe_source_url(source_url)
                .fetch_internal()
                .await?
        }
        None => {
            FetchSrc::new(temp_dir.path(), rockspec, config)
                .fetch_internal()
                .await?
        }
    };

    let hashes = LockedPackageHashes {
        rockspec: rockspec.hash().await?,
        source: source_metadata.hash.clone(),
    };

    let package = LockedPackage::new(
        spec,
        source
            .map(Result::Ok)
            .unwrap_or_else(|| {
                rockspec
                    .to_lua_remote_rockspec_string()
                    .map(RemotePackageSource::RockspecContent)
            })
            .unwrap_or(RemotePackageSource::Local),
        Some(source_metadata.source_url.clone()),
        hashes,
    );

    let source_root = if is_src_rock {
        src_rock_source_root(rockspec, temp_dir.path())
    } else {
        temp_dir.path().to_path_buf()
    };

    Ok((package, temp_dir, source_root))
}