lux_lib/operations/
vendor.rs

1use std::{
2    io::{self, Cursor},
3    path::{Path, PathBuf},
4    sync::Arc,
5};
6
7use bon::Builder;
8use bytes::Bytes;
9use futures::StreamExt;
10use itertools::Itertools;
11use path_slash::PathExt;
12use strum::IntoEnumIterator;
13use thiserror::Error;
14use tokio::{fs::File, io::AsyncWriteExt};
15
16use crate::{
17    build::{RemotePackageSourceSpec, SrcRockSource},
18    config::Config,
19    lockfile::{LocalPackageLockType, ReadOnly},
20    lua_rockspec::RemoteLuaRockspec,
21    operations::{
22        self,
23        resolve::{PackageInstallData, Resolve, ResolveDependenciesError},
24        DownloadedRockspec, FetchSrcError, PackageInstallSpec, UnpackError,
25    },
26    package::PackageReq,
27    progress::{MultiProgress, Progress, ProgressBar},
28    project::{project_toml::LocalProjectTomlValidationError, Project, ProjectError},
29    remote_package_db::{RemotePackageDB, RemotePackageDBError},
30    rockspec::Rockspec,
31    tree::EntryType,
32};
33
34pub enum VendorTarget {
35    /// Vendor dependencies of a Lux project
36    Project(Project),
37    /// Vendor dependencies of a Lua RockSpec
38    Rockspec(RemoteLuaRockspec),
39}
40
41/// Vendor a project's dependencies into the specified directory at `<vendor_dir>`.
42/// After this command completes the vendor directory specified by `<vendor_dir>`
43/// will contain all remote sources from dependencies specified.
44#[derive(Builder)]
45#[builder(start_fn = new, finish_fn(name = _build, vis = ""))]
46pub struct Vendor<'a> {
47    target: VendorTarget,
48
49    /// The directory in which to vendor the dependencies.
50    vendor_dir: PathBuf,
51
52    /// Ignore the project's lockfile.
53    no_lock: Option<bool>,
54
55    /// Don't delete the `<vendor-dir>` when vendoring,{n}
56    /// but rather keep all existing contents of the vendor directory.
57    no_delete: Option<bool>,
58
59    config: &'a Config,
60
61    progress: Option<Arc<Progress<MultiProgress>>>,
62}
63
64#[derive(Error, Debug)]
65pub enum VendorError {
66    #[error(transparent)]
67    Project(#[from] ProjectError),
68    #[error("project validation failed:\n{0}")]
69    LocalProjectTomlValidation(#[from] LocalProjectTomlValidationError),
70    #[error("error initialising remote package DB:\n{0}")]
71    RemotePackageDB(#[from] RemotePackageDBError),
72    #[error("failed to resolve dependencies:\n{0}")]
73    ResolveDependencies(#[from] ResolveDependenciesError),
74    #[error("failed to delete vendor directory {0}:\n{1}")]
75    DeleteVendorDir(String, io::Error),
76    #[error("failed to create vendor directory {0}:\n{1}")]
77    CreateVendorDir(String, io::Error),
78    #[error("failed to create {0}:\n{1}")]
79    CreateSrcRock(String, io::Error),
80    #[error("failed to vendor Lua RockSpec:\n{0}")]
81    LuaRockSpec(String),
82    #[error("failed to write Lua RockSpec {0}:\n{1}")]
83    WriteLuaRockSpec(String, io::Error),
84    #[error("failed to unpack src.rock:\n{0}")]
85    Unpack(#[from] UnpackError),
86    #[error("failed to fetch rock source:\n{0}")]
87    FetchSrc(#[from] FetchSrcError),
88}
89
90impl<State> VendorBuilder<'_, State>
91where
92    State: vendor_builder::State + vendor_builder::IsComplete,
93{
94    pub async fn vendor_dependencies(self) -> Result<(), VendorError> {
95        do_vendor_dependencies(self._build()).await
96    }
97}
98
99async fn do_vendor_dependencies(args: Vendor<'_>) -> Result<(), VendorError> {
100    let vendor_dir = args.vendor_dir;
101    let no_delete = args.no_delete.unwrap_or(false);
102    let no_lock = args.no_lock.unwrap_or(false);
103    let target = args.target;
104    let config = args.config;
105    let progress = match args.progress {
106        Some(p) => p,
107        None => MultiProgress::new_arc(args.config),
108    };
109    let mut all_packages = Vec::new();
110
111    for lock_type in LocalPackageLockType::iter() {
112        let (package_db, install_specs) =
113            mk_resolve_args(lock_type, no_lock, &target, config, progress.clone()).await?;
114
115        let (dep_tx, mut dep_rx) = tokio::sync::mpsc::unbounded_channel();
116        Resolve::<'_, ReadOnly>::new()
117            .dependencies_tx(dep_tx.clone())
118            .build_dependencies_tx(dep_tx)
119            .packages(install_specs)
120            .package_db(Arc::new(package_db))
121            .config(config)
122            .progress(progress.clone())
123            .get_all_dependencies()
124            .await?;
125
126        while let Some(dep) = dep_rx.recv().await {
127            all_packages.push(dep);
128        }
129    }
130
131    if !no_delete && vendor_dir.exists() {
132        tokio::fs::remove_dir_all(&vendor_dir)
133            .await
134            .map_err(|err| {
135                VendorError::DeleteVendorDir(vendor_dir.to_slash_lossy().to_string(), err)
136            })?;
137    }
138
139    vendor_sources(Arc::new(vendor_dir), progress, config.clone(), all_packages).await
140}
141
142async fn mk_resolve_args(
143    lock_type: LocalPackageLockType,
144    no_lock: bool,
145    target: &VendorTarget,
146    config: &Config,
147    progress: Arc<Progress<MultiProgress>>,
148) -> Result<(RemotePackageDB, Vec<PackageInstallSpec>), VendorError> {
149    match &target {
150        VendorTarget::Project(project) => {
151            let toml = project.toml().into_local()?;
152            let lockfile = project.lockfile()?;
153            let package_db = if !no_lock {
154                lockfile.local_pkg_lock(&lock_type).clone().into()
155            } else {
156                let bar = progress.map(|p| p.new_bar());
157                RemotePackageDB::from_config(config, &bar).await?
158            };
159            let mut install_specs = mk_dependencies_vec(&lock_type, &toml)?;
160            if lock_type == LocalPackageLockType::Test {
161                for test_spec_dependency in toml
162                    .test()
163                    .current_platform()
164                    .test_dependencies(project)
165                    .iter()
166                    .cloned()
167                    .map(|dep| PackageInstallSpec::new(dep, EntryType::Entrypoint).build())
168                {
169                    install_specs.push(test_spec_dependency);
170                }
171            }
172            Ok((package_db, install_specs))
173        }
174        VendorTarget::Rockspec(remote_lua_rockspec) => {
175            let bar = progress.map(|p| p.new_bar());
176            let package_db = RemotePackageDB::from_config(config, &bar).await?;
177            let install_specs = mk_dependencies_vec(&lock_type, remote_lua_rockspec)?;
178            Ok((package_db, install_specs))
179        }
180    }
181}
182
183fn mk_dependencies_vec<R: Rockspec>(
184    lock_type: &LocalPackageLockType,
185    rockspec: &R,
186) -> Result<Vec<PackageInstallSpec>, LocalProjectTomlValidationError> {
187    let dependencies: Vec<&PackageReq> = match lock_type {
188        LocalPackageLockType::Regular => rockspec
189            .dependencies()
190            .current_platform()
191            .iter()
192            .map(|dep| dep.package_req())
193            .collect_vec(),
194        LocalPackageLockType::Test => rockspec
195            .test_dependencies()
196            .current_platform()
197            .iter()
198            .map(|dep| dep.package_req())
199            .collect_vec(),
200        LocalPackageLockType::Build => rockspec
201            .build_dependencies()
202            .current_platform()
203            .iter()
204            .map(|dep| dep.package_req())
205            .collect_vec(),
206    };
207
208    Ok(dependencies
209        .into_iter()
210        .unique()
211        .cloned()
212        .map(|dep| PackageInstallSpec::new(dep, EntryType::Entrypoint).build())
213        .collect_vec())
214}
215
216async fn vendor_sources(
217    vendor_dir: Arc<PathBuf>,
218    progress: Arc<Progress<MultiProgress>>,
219    config: Config,
220    packages: Vec<PackageInstallData>,
221) -> Result<(), VendorError> {
222    futures::stream::iter(packages.into_iter().map(|dep| {
223        let vendor_dir = Arc::clone(&vendor_dir);
224        let progress = Arc::clone(&progress);
225        let config = config.clone();
226        tokio::spawn(async move {
227            match dep.downloaded_rock {
228                crate::operations::RemoteRockDownload::RockspecOnly { rockspec_download } => {
229                    vendor_rockspec_sources(
230                        &vendor_dir,
231                        rockspec_download,
232                        None,
233                        &config,
234                        &progress,
235                    )
236                    .await?
237                }
238                crate::operations::RemoteRockDownload::BinaryRock {
239                    rockspec_download,
240                    packed_rock,
241                } => {
242                    vendor_binary_rock(&vendor_dir, rockspec_download, packed_rock, &progress)
243                        .await?
244                }
245                crate::operations::RemoteRockDownload::SrcRock {
246                    rockspec_download,
247                    src_rock,
248                    source_url,
249                } => {
250                    let src_rock_source = SrcRockSource {
251                        bytes: src_rock,
252                        source_url,
253                    };
254                    vendor_rockspec_sources(
255                        &vendor_dir,
256                        rockspec_download,
257                        Some(src_rock_source),
258                        &config,
259                        &progress,
260                    )
261                    .await?
262                }
263            };
264            Ok::<_, VendorError>(())
265        })
266    }))
267    .buffered(config.max_jobs())
268    .collect::<Vec<_>>()
269    .await
270    .into_iter()
271    .flatten()
272    .try_collect()
273}
274
275async fn vendor_rockspec_sources(
276    vendor_dir: &Path,
277    rockspec_download: DownloadedRockspec,
278    src_rock_source: Option<SrcRockSource>,
279    config: &Config,
280    progress: &Progress<MultiProgress>,
281) -> Result<(), VendorError> {
282    let rockspec = rockspec_download.rockspec;
283    let package = rockspec.package();
284    let version = rockspec.version();
285    let package_version_str = format!("{}@{}", package, version);
286    let bar = progress.map(|p| {
287        p.add(ProgressBar::from(format!(
288            "💼 Vendoring source of {}",
289            &package_version_str,
290        )))
291    });
292    let source_spec = match src_rock_source {
293        Some(src_rock_source) => RemotePackageSourceSpec::SrcRock(src_rock_source),
294        None => RemotePackageSourceSpec::RockSpec(rockspec_download.source_url),
295    };
296
297    let package_vendor_dir = vendor_dir.join(&package_version_str);
298
299    tokio::fs::create_dir_all(&package_vendor_dir)
300        .await
301        .map_err(|err| {
302            VendorError::CreateVendorDir(package_vendor_dir.to_slash_lossy().to_string(), err)
303        })?;
304
305    let rockspec_lua_content = rockspec
306        .to_lua_remote_rockspec_string()
307        .map_err(|err| VendorError::LuaRockSpec(err.to_string()))?;
308
309    let rockspec_file_name = format!("{}-{}.rockspec", package, version);
310    let rockspec_path = vendor_dir.join(rockspec_file_name);
311    tokio::fs::write(&rockspec_path, rockspec_lua_content)
312        .await
313        .map_err(|err| {
314            VendorError::WriteLuaRockSpec(rockspec_path.to_slash_lossy().to_string(), err)
315        })?;
316
317    match source_spec {
318        RemotePackageSourceSpec::SrcRock(SrcRockSource {
319            bytes,
320            source_url: _,
321        }) => {
322            let cursor = Cursor::new(&bytes);
323            operations::unpack_src_rock(cursor, package_vendor_dir, &bar).await?;
324        }
325        RemotePackageSourceSpec::RockSpec(source_url) => {
326            operations::FetchSrc::new(&package_vendor_dir, &rockspec, config, &bar)
327                .maybe_source_url(source_url)
328                .fetch_internal()
329                .await?;
330        }
331    }
332
333    bar.map(|bar| bar.finish_and_clear());
334
335    Ok(())
336}
337
338async fn vendor_binary_rock(
339    vendor_dir: &Path,
340    rockspec_download: DownloadedRockspec,
341    packed_rock: Bytes,
342    progress: &Progress<MultiProgress>,
343) -> Result<(), VendorError> {
344    let rockspec = rockspec_download.rockspec;
345    let package = rockspec.package();
346    let version = rockspec.version();
347
348    let file_name = format!("{}@{}.rock", package, version);
349
350    let bar = progress.map(|p| {
351        p.add(ProgressBar::from(format!(
352            "💼 Vendoring pre-built binary .rock: {}",
353            &file_name,
354        )))
355    });
356
357    tokio::fs::create_dir_all(&vendor_dir)
358        .await
359        .map_err(|err| {
360            VendorError::CreateVendorDir(vendor_dir.to_slash_lossy().to_string(), err)
361        })?;
362
363    let dest_file = vendor_dir.join(&file_name);
364    let mut file = File::create(&dest_file)
365        .await
366        .map_err(|err| VendorError::CreateSrcRock(dest_file.to_slash_lossy().to_string(), err))?;
367    file.write_all(&packed_rock)
368        .await
369        .map_err(|err| VendorError::CreateSrcRock(dest_file.to_slash_lossy().to_string(), err))?;
370
371    bar.map(|bar| bar.finish_and_clear());
372
373    Ok(())
374}
lux_lib/operations/vendor.rs

lux_lib/operations/
vendor.rs
