lux_lib/operations/
vendor.rs

1use std::{
2    io::{self, Cursor},
3    path::{Path, PathBuf},
4    sync::Arc,
5};
6
7use bon::Builder;
8use bytes::Bytes;
9use futures::StreamExt;
10use itertools::Itertools;
11use path_slash::PathExt;
12use strum::IntoEnumIterator;
13use thiserror::Error;
14use tokio::{fs::File, io::AsyncWriteExt};
15
16use crate::{
17    build::{RemotePackageSourceSpec, SrcRockSource},
18    config::Config,
19    lockfile::{LocalPackageLockType, ReadOnly},
20    lua_rockspec::RemoteLuaRockspec,
21    operations::{
22        self,
23        resolve::{PackageInstallData, Resolve, ResolveDependenciesError},
24        DownloadedRockspec, FetchSrcError, PackageInstallSpec, UnpackError,
25    },
26    package::PackageReq,
27    progress::{MultiProgress, Progress, ProgressBar},
28    project::project_toml::LocalProjectTomlValidationError,
29    remote_package_db::{RemotePackageDB, RemotePackageDBError},
30    rockspec::Rockspec,
31    tree::EntryType,
32    workspace::{Workspace, WorkspaceError},
33};
34
35#[allow(clippy::large_enum_variant)]
36pub enum VendorTarget {
37    /// Vendor dependencies of a Lux workspace
38    Workspace(Workspace),
39
40    /// Vendor dependencies of a Lua RockSpec
41    Rockspec(RemoteLuaRockspec),
42}
43
44/// Vendor a project's dependencies into the specified directory at `<vendor_dir>`.
45/// After this command completes the vendor directory specified by `<vendor_dir>`
46/// will contain all remote sources from dependencies specified.
47#[derive(Builder)]
48#[builder(start_fn = new, finish_fn(name = _build, vis = ""))]
49pub struct Vendor<'a> {
50    target: VendorTarget,
51
52    /// The directory in which to vendor the dependencies.
53    vendor_dir: PathBuf,
54
55    /// Ignore the project's lockfile.
56    no_lock: Option<bool>,
57
58    /// Don't delete the `<vendor-dir>` when vendoring,{n}
59    /// but rather keep all existing contents of the vendor directory.
60    no_delete: Option<bool>,
61
62    config: &'a Config,
63
64    progress: Option<Arc<Progress<MultiProgress>>>,
65}
66
67#[derive(Error, Debug)]
68pub enum VendorError {
69    #[error(transparent)]
70    Workspace(#[from] WorkspaceError),
71    #[error("project validation failed:\n{0}")]
72    LocalProjectTomlValidation(#[from] LocalProjectTomlValidationError),
73    #[error("error initialising remote package DB:\n{0}")]
74    RemotePackageDB(#[from] RemotePackageDBError),
75    #[error("failed to resolve dependencies:\n{0}")]
76    ResolveDependencies(#[from] ResolveDependenciesError),
77    #[error("failed to delete vendor directory {0}:\n{1}")]
78    DeleteVendorDir(String, io::Error),
79    #[error("failed to create vendor directory {0}:\n{1}")]
80    CreateVendorDir(String, io::Error),
81    #[error("failed to create {0}:\n{1}")]
82    CreateSrcRock(String, io::Error),
83    #[error("failed to vendor Lua RockSpec:\n{0}")]
84    LuaRockSpec(String),
85    #[error("failed to write Lua RockSpec {0}:\n{1}")]
86    WriteLuaRockSpec(String, io::Error),
87    #[error("failed to unpack src.rock:\n{0}")]
88    Unpack(#[from] UnpackError),
89    #[error("failed to fetch rock source:\n{0}")]
90    FetchSrc(#[from] FetchSrcError),
91}
92
93impl<State> VendorBuilder<'_, State>
94where
95    State: vendor_builder::State + vendor_builder::IsComplete,
96{
97    pub async fn vendor_dependencies(self) -> Result<(), VendorError> {
98        do_vendor_dependencies(self._build()).await
99    }
100}
101
102async fn do_vendor_dependencies(args: Vendor<'_>) -> Result<(), VendorError> {
103    let vendor_dir = args.vendor_dir;
104    let no_delete = args.no_delete.unwrap_or(false);
105    let no_lock = args.no_lock.unwrap_or(false);
106    let target = args.target;
107    let config = args.config;
108    let progress = match args.progress {
109        Some(p) => p,
110        None => MultiProgress::new_arc(args.config),
111    };
112    let mut all_packages = Vec::new();
113
114    for lock_type in LocalPackageLockType::iter() {
115        let (package_db, install_specs) =
116            mk_resolve_args(lock_type, no_lock, &target, config, progress.clone()).await?;
117
118        let (dep_tx, mut dep_rx) = tokio::sync::mpsc::unbounded_channel();
119        Resolve::<'_, ReadOnly>::new()
120            .dependencies_tx(dep_tx.clone())
121            .build_dependencies_tx(dep_tx)
122            .packages(install_specs)
123            .package_db(Arc::new(package_db))
124            .config(config)
125            .progress(progress.clone())
126            .get_all_dependencies()
127            .await?;
128
129        while let Some(dep) = dep_rx.recv().await {
130            all_packages.push(dep);
131        }
132    }
133
134    if !no_delete && vendor_dir.exists() {
135        tokio::fs::remove_dir_all(&vendor_dir)
136            .await
137            .map_err(|err| {
138                VendorError::DeleteVendorDir(vendor_dir.to_slash_lossy().to_string(), err)
139            })?;
140    }
141
142    vendor_sources(Arc::new(vendor_dir), progress, config.clone(), all_packages).await
143}
144
145async fn mk_resolve_args(
146    lock_type: LocalPackageLockType,
147    no_lock: bool,
148    target: &VendorTarget,
149    config: &Config,
150    progress: Arc<Progress<MultiProgress>>,
151) -> Result<(RemotePackageDB, Vec<PackageInstallSpec>), VendorError> {
152    match &target {
153        VendorTarget::Workspace(workspace) => {
154            let lockfile = workspace.lockfile()?;
155            let package_db = if !no_lock {
156                lockfile.local_pkg_lock(&lock_type).clone().into()
157            } else {
158                let bar = progress.map(|p| p.new_bar());
159                RemotePackageDB::from_config(config, &bar).await?
160            };
161            let mut install_specs = Vec::new();
162            for project in workspace.members() {
163                let toml = project.toml().into_local()?;
164                push_dependencies(&lock_type, &toml, &mut install_specs)?;
165                if lock_type == LocalPackageLockType::Test {
166                    for test_spec_dependency in toml
167                        .test()
168                        .current_platform()
169                        .test_dependencies(project)
170                        .iter()
171                        .cloned()
172                        .map(|dep| PackageInstallSpec::new(dep, EntryType::Entrypoint).build())
173                    {
174                        install_specs.push(test_spec_dependency);
175                    }
176                }
177            }
178            Ok((package_db, install_specs))
179        }
180        VendorTarget::Rockspec(remote_lua_rockspec) => {
181            let bar = progress.map(|p| p.new_bar());
182            let package_db = RemotePackageDB::from_config(config, &bar).await?;
183            let mut install_specs = Vec::new();
184            push_dependencies(&lock_type, remote_lua_rockspec, &mut install_specs)?;
185            Ok((package_db, install_specs))
186        }
187    }
188}
189
190fn push_dependencies<R: Rockspec>(
191    lock_type: &LocalPackageLockType,
192    rockspec: &R,
193    install_specs: &mut Vec<PackageInstallSpec>,
194) -> Result<(), LocalProjectTomlValidationError> {
195    let dependencies: Vec<&PackageReq> = match lock_type {
196        LocalPackageLockType::Regular => rockspec
197            .dependencies()
198            .current_platform()
199            .iter()
200            .map(|dep| dep.package_req())
201            .collect_vec(),
202        LocalPackageLockType::Test => rockspec
203            .test_dependencies()
204            .current_platform()
205            .iter()
206            .map(|dep| dep.package_req())
207            .collect_vec(),
208        LocalPackageLockType::Build => rockspec
209            .build_dependencies()
210            .current_platform()
211            .iter()
212            .map(|dep| dep.package_req())
213            .collect_vec(),
214    };
215    install_specs.extend(
216        dependencies
217            .into_iter()
218            .unique()
219            .cloned()
220            .map(|dep| PackageInstallSpec::new(dep, EntryType::Entrypoint).build())
221            .collect_vec(),
222    );
223    Ok(())
224}
225
226async fn vendor_sources(
227    vendor_dir: Arc<PathBuf>,
228    progress: Arc<Progress<MultiProgress>>,
229    config: Config,
230    packages: Vec<PackageInstallData>,
231) -> Result<(), VendorError> {
232    futures::stream::iter(packages.into_iter().map(|dep| {
233        let vendor_dir = Arc::clone(&vendor_dir);
234        let progress = Arc::clone(&progress);
235        let config = config.clone();
236        tokio::spawn(async move {
237            match dep.downloaded_rock {
238                crate::operations::RemoteRockDownload::RockspecOnly { rockspec_download } => {
239                    vendor_rockspec_sources(
240                        &vendor_dir,
241                        rockspec_download,
242                        None,
243                        &config,
244                        &progress,
245                    )
246                    .await?
247                }
248                crate::operations::RemoteRockDownload::BinaryRock {
249                    rockspec_download,
250                    packed_rock,
251                } => {
252                    vendor_binary_rock(&vendor_dir, rockspec_download, packed_rock, &progress)
253                        .await?
254                }
255                crate::operations::RemoteRockDownload::SrcRock {
256                    rockspec_download,
257                    src_rock,
258                    source_url,
259                } => {
260                    let src_rock_source = SrcRockSource {
261                        bytes: src_rock,
262                        source_url,
263                    };
264                    vendor_rockspec_sources(
265                        &vendor_dir,
266                        rockspec_download,
267                        Some(src_rock_source),
268                        &config,
269                        &progress,
270                    )
271                    .await?
272                }
273            };
274            Ok::<_, VendorError>(())
275        })
276    }))
277    .buffered(config.max_jobs())
278    .collect::<Vec<_>>()
279    .await
280    .into_iter()
281    .flatten()
282    .try_collect()
283}
284
285async fn vendor_rockspec_sources(
286    vendor_dir: &Path,
287    rockspec_download: DownloadedRockspec,
288    src_rock_source: Option<SrcRockSource>,
289    config: &Config,
290    progress: &Progress<MultiProgress>,
291) -> Result<(), VendorError> {
292    let rockspec = rockspec_download.rockspec;
293    let package = rockspec.package();
294    let version = rockspec.version();
295    let package_version_str = format!("{}@{}", package, version);
296    let bar = progress.map(|p| {
297        p.add(ProgressBar::from(format!(
298            "💼 Vendoring source of {}",
299            &package_version_str,
300        )))
301    });
302    let source_spec = match src_rock_source {
303        Some(src_rock_source) => RemotePackageSourceSpec::SrcRock(src_rock_source),
304        None => RemotePackageSourceSpec::RockSpec(rockspec_download.source_url),
305    };
306
307    let package_vendor_dir = vendor_dir.join(&package_version_str);
308
309    tokio::fs::create_dir_all(&package_vendor_dir)
310        .await
311        .map_err(|err| {
312            VendorError::CreateVendorDir(package_vendor_dir.to_slash_lossy().to_string(), err)
313        })?;
314
315    let rockspec_lua_content = rockspec
316        .to_lua_remote_rockspec_string()
317        .map_err(|err| VendorError::LuaRockSpec(err.to_string()))?;
318
319    let rockspec_file_name = format!("{}-{}.rockspec", package, version);
320    let rockspec_path = vendor_dir.join(rockspec_file_name);
321    tokio::fs::write(&rockspec_path, rockspec_lua_content)
322        .await
323        .map_err(|err| {
324            VendorError::WriteLuaRockSpec(rockspec_path.to_slash_lossy().to_string(), err)
325        })?;
326
327    match source_spec {
328        RemotePackageSourceSpec::SrcRock(SrcRockSource {
329            bytes,
330            source_url: _,
331        }) => {
332            let cursor = Cursor::new(&bytes);
333            operations::unpack_src_rock(cursor, package_vendor_dir, &bar).await?;
334        }
335        RemotePackageSourceSpec::RockSpec(source_url) => {
336            operations::FetchSrc::new(&package_vendor_dir, &rockspec, config, &bar)
337                .maybe_source_url(source_url)
338                .fetch_internal()
339                .await?;
340        }
341    }
342
343    bar.map(|bar| bar.finish_and_clear());
344
345    Ok(())
346}
347
348async fn vendor_binary_rock(
349    vendor_dir: &Path,
350    rockspec_download: DownloadedRockspec,
351    packed_rock: Bytes,
352    progress: &Progress<MultiProgress>,
353) -> Result<(), VendorError> {
354    let rockspec = rockspec_download.rockspec;
355    let package = rockspec.package();
356    let version = rockspec.version();
357
358    let file_name = format!("{}@{}.rock", package, version);
359
360    let bar = progress.map(|p| {
361        p.add(ProgressBar::from(format!(
362            "💼 Vendoring pre-built binary .rock: {}",
363            &file_name,
364        )))
365    });
366
367    tokio::fs::create_dir_all(&vendor_dir)
368        .await
369        .map_err(|err| {
370            VendorError::CreateVendorDir(vendor_dir.to_slash_lossy().to_string(), err)
371        })?;
372
373    let dest_file = vendor_dir.join(&file_name);
374    let mut file = File::create(&dest_file)
375        .await
376        .map_err(|err| VendorError::CreateSrcRock(dest_file.to_slash_lossy().to_string(), err))?;
377    file.write_all(&packed_rock)
378        .await
379        .map_err(|err| VendorError::CreateSrcRock(dest_file.to_slash_lossy().to_string(), err))?;
380
381    bar.map(|bar| bar.finish_and_clear());
382
383    Ok(())
384}
lux_lib/operations/vendor.rs

lux_lib/operations/
vendor.rs
