lux_lib/pipeline/
materialise.rs

1use std::collections::HashMap;
2use std::io;
3use std::io::Cursor;
4use std::path::{Path, PathBuf};
5
6use bon::Builder;
7use bytes::Bytes;
8use futures::{StreamExt, TryStreamExt};
9use miette::Diagnostic;
10use tempfile::TempDir;
11use thiserror::Error;
12
13use crate::{
14    build::{self, RemotePackageSourceSpec, SrcRockSource},
15    config::Config,
16    fs,
17    hash::HasIntegrity,
18    lockfile::{
19        LockedPackage, LockedPackageHashes, LockedPackageId, LockedPackageSpec,
20        RemotePackageSourceUrl,
21    },
22    lua_rockspec::{RemoteLuaRockspec, RockSourceSpec},
23    operations::{
24        single_non_rockspec_entry, unpack_src_rock, FetchSrc, RemotePackageSourceMetadata,
25    },
26    remote_package_source::RemotePackageSource,
27    rockspec::Rockspec,
28    tree::EntryType,
29};
30
31use super::{
32    resolve::{ResolvedArtifacts, ResolvedPackage},
33    Artifacts,
34};
35
36#[derive(Clone, Debug)]
37pub(crate) enum PackageSource {
38    SourceDir(PathBuf),
39    Archive(Bytes),
40}
41
42pub(crate) struct MaterialisedPackage {
43    pub(crate) package: LockedPackage,
44    pub(crate) rockspec: RemoteLuaRockspec,
45    pub(crate) entry_type: EntryType,
46    pub(crate) build_behaviour: crate::build::BuildBehaviour,
47    pub(crate) artifact: PackageSource,
48    pub(crate) temp_dir: Option<TempDir>,
49}
50
51pub(crate) type MaterialiseArtifacts = Artifacts<HashMap<LockedPackageId, MaterialisedPackage>>;
52
53#[derive(Error, Debug, Diagnostic)]
54#[non_exhaustive]
55pub enum Error {
56    #[error(transparent)]
57    #[diagnostic(transparent)]
58    Fs(#[from] fs::Error),
59    #[error(transparent)]
60    #[diagnostic(transparent)]
61    Build(#[from] Box<build::Error>),
62    #[error("failed to hash the source for '{0}'")]
63    Hash(String, #[source] io::Error),
64    #[error("missing source url for '{0}'")]
65    MissingSourceUrl(String),
66}
67
68impl From<build::Error> for Error {
69    fn from(source: build::Error) -> Self {
70        Self::Build(Box::new(source))
71    }
72}
73
74#[derive(Builder)]
75#[builder(start_fn = new, finish_fn(name = _build, vis = ""))]
76pub(crate) struct Materialise<'a> {
77    #[builder(start_fn)]
78    pub(crate) config: &'a Config,
79    pub(crate) resolved: ResolvedArtifacts,
80}
81
82impl<State> MaterialiseBuilder<'_, State>
83where
84    State: materialise_builder::State + materialise_builder::IsComplete,
85{
86    pub(crate) async fn materialise(self) -> Result<MaterialiseArtifacts, Error> {
87        let args = self._build();
88        let config = args.config;
89
90        let artifacts: MaterialiseArtifacts = args
91            .resolved
92            .iter()
93            .map(|(section, packages)| (section, packages.map(|_| HashMap::new())))
94            .collect();
95
96        let jobs = args.resolved.into_iter().flat_map(|(section, packages)| {
97            packages
98                .into_iter()
99                .flatten()
100                .map(move |(id, package)| (section, id, package))
101        });
102
103        futures::stream::iter(jobs)
104            .map(|(section, id, package)| async move {
105                materialise(config, package)
106                    .await
107                    .map(|package| (section, id, package))
108            })
109            .buffered(config.max_jobs())
110            .try_fold(
111                artifacts,
112                |mut artifacts, (section, id, package)| async move {
113                    artifacts
114                        .get_mut(section)
115                        .get_or_insert_with(HashMap::new)
116                        .insert(id, package);
117                    Ok(artifacts)
118                },
119            )
120            .await
121    }
122}
123
124async fn materialise(
125    config: &Config,
126    resolved: ResolvedPackage,
127) -> Result<MaterialisedPackage, Error> {
128    let ResolvedPackage {
129        spec,
130        rockspec,
131        source,
132        source_url,
133        entry_type,
134        build_behaviour,
135        artifact,
136    } = resolved;
137
138    let name = spec.name().to_string();
139
140    let (package, artifact, temp_dir) = match artifact {
141        Some(PackageSource::Archive(bytes)) => match &source {
142            RemotePackageSource::LuarocksBinaryRock(url) => {
143                let binary_url = url.clone();
144                let rockspec_hash = rockspec
145                    .hash()
146                    .await
147                    .map_err(|err| Error::Hash(name.clone(), err))?;
148                let source_hash = bytes
149                    .hash()
150                    .await
151                    .map_err(|err| Error::Hash(name.clone(), err))?;
152                let hashes = LockedPackageHashes {
153                    rockspec: rockspec_hash,
154                    source: source_hash,
155                };
156                let package = LockedPackage::new(
157                    spec,
158                    source,
159                    Some(RemotePackageSourceUrl::Url { url: binary_url }),
160                    hashes,
161                );
162                (package, PackageSource::Archive(bytes), None)
163            }
164            _ => {
165                let source_url = source_url
166                    .clone()
167                    .ok_or_else(|| Error::MissingSourceUrl(name.clone()))?;
168                let source_spec =
169                    RemotePackageSourceSpec::SrcRock(SrcRockSource { bytes, source_url });
170                let (package, dir, source_root) =
171                    fetch_and_hash_source(&rockspec, spec, Some(source_spec), Some(source), config)
172                        .await?;
173                let artifact = PackageSource::SourceDir(source_root);
174                (package, artifact, Some(dir))
175            }
176        },
177        Some(PackageSource::SourceDir(path)) => {
178            let rockspec_hash = rockspec
179                .hash()
180                .await
181                .map_err(|err| Error::Hash(name.clone(), err))?;
182            let source_hash = path
183                .hash()
184                .await
185                .map_err(|err| Error::Hash(name.clone(), err))?;
186            let hashes = LockedPackageHashes {
187                rockspec: rockspec_hash,
188                source: source_hash,
189            };
190            let package = LockedPackage::new(spec, source, source_url, hashes);
191            (package, PackageSource::SourceDir(path), None)
192        }
193        None => {
194            let source_spec = RemotePackageSourceSpec::RockSpec(source_url);
195            let (package, dir, source_root) =
196                fetch_and_hash_source(&rockspec, spec, Some(source_spec), Some(source), config)
197                    .await?;
198            let artifact = PackageSource::SourceDir(source_root);
199            (package, artifact, Some(dir))
200        }
201    };
202
203    Ok(MaterialisedPackage {
204        package,
205        rockspec,
206        entry_type,
207        build_behaviour,
208        artifact,
209        temp_dir,
210    })
211}
212
213fn is_src_rock_url(source_url: &RemotePackageSourceUrl) -> bool {
214    match source_url {
215        RemotePackageSourceUrl::Url { url } => url.path().ends_with(".src.rock"),
216        RemotePackageSourceUrl::File { path } => path.to_string_lossy().ends_with(".src.rock"),
217        RemotePackageSourceUrl::Git { .. } => false,
218    }
219}
220
221/// Resolves the source root inside an unpacked source rock.
222///
223/// The source root is `<unpacked>/<repository>` when that directory is the only
224/// entry besides the rockspec. Otherwise the source root is the unpacked root.
225///
226/// This matches the wrapper directory luarocks adds when packing a Git source rock,
227/// while leaving unwrapped source trees untouched.
228///
229/// For a git source rock containing `foo/`:
230///
231/// | unpacked contents        | returned source root |
232/// |--------------------------|----------------------|
233/// | `foo/` and the rockspec  | `<unpacked>/foo`     |
234/// | source files at the root | `<unpacked>`         |
235fn src_rock_source_root<R: Rockspec>(rockspec: &R, unpacked_root: &Path) -> PathBuf {
236    if let RockSourceSpec::Git(git) = &rockspec.source().current_platform().source_spec {
237        let repo_dir = unpacked_root.join(git.url.repo());
238        if let Ok(Some(entry)) = single_non_rockspec_entry(unpacked_root) {
239            if entry == repo_dir && entry.is_dir() {
240                return repo_dir;
241            }
242        }
243    }
244    unpacked_root.to_path_buf()
245}
246
247pub(crate) async fn fetch_and_hash_source<R: Rockspec + HasIntegrity>(
248    rockspec: &R,
249    spec: LockedPackageSpec,
250    source_spec: Option<RemotePackageSourceSpec>,
251    source: Option<RemotePackageSource>,
252    config: &Config,
253) -> Result<(LockedPackage, TempDir, PathBuf), build::Error> {
254    let temp_dir = fs::tempfile::tempdir()?;
255    let is_src_rock = match &source_spec {
256        Some(RemotePackageSourceSpec::SrcRock(_)) => true,
257        Some(RemotePackageSourceSpec::RockSpec(Some(source_url))) => {
258            config.vendor_dir().is_none() && is_src_rock_url(source_url)
259        }
260        _ => false,
261    };
262
263    let source_metadata = match source_spec {
264        Some(RemotePackageSourceSpec::SrcRock(SrcRockSource { bytes, source_url })) => {
265            let hash = bytes.hash().await?;
266            let cursor = Cursor::new(bytes);
267            unpack_src_rock(cursor, temp_dir.path().to_path_buf())
268                .await
269                .map_err(build::Error::UnpackSrcRock)?;
270            RemotePackageSourceMetadata { hash, source_url }
271        }
272        Some(RemotePackageSourceSpec::RockSpec(source_url)) => {
273            FetchSrc::new(temp_dir.path(), rockspec, config)
274                .maybe_source_url(source_url)
275                .fetch_internal()
276                .await?
277        }
278        None => {
279            FetchSrc::new(temp_dir.path(), rockspec, config)
280                .fetch_internal()
281                .await?
282        }
283    };
284
285    let hashes = LockedPackageHashes {
286        rockspec: rockspec.hash().await?,
287        source: source_metadata.hash.clone(),
288    };
289
290    let package = LockedPackage::new(
291        spec,
292        source
293            .map(Result::Ok)
294            .unwrap_or_else(|| {
295                rockspec
296                    .to_lua_remote_rockspec_string()
297                    .map(RemotePackageSource::RockspecContent)
298            })
299            .unwrap_or(RemotePackageSource::Local),
300        Some(source_metadata.source_url.clone()),
301        hashes,
302    );
303
304    let source_root = if is_src_rock {
305        src_rock_source_root(rockspec, temp_dir.path())
306    } else {
307        temp_dir.path().to_path_buf()
308    };
309
310    Ok((package, temp_dir, source_root))
311}
lux_lib/pipeline/materialise.rs

lux_lib/pipeline/
materialise.rs
