lux_lib/operations/
vendor.rs

1use std::{
2    io::{self, Cursor},
3    path::{Path, PathBuf},
4    sync::Arc,
5};
6
7use bon::Builder;
8use bytes::Bytes;
9use futures::StreamExt;
10use itertools::Itertools;
11use miette::Diagnostic;
12use path_slash::PathExt;
13use strum::IntoEnumIterator;
14use thiserror::Error;
15use tokio::{fs::File, io::AsyncWriteExt};
16
17use crate::{
18    build::{RemotePackageSourceSpec, SrcRockSource},
19    config::Config,
20    lockfile::{LocalPackageLockType, ReadOnly},
21    lua_rockspec::RemoteLuaRockspec,
22    operations::{
23        self,
24        resolve::{PackageInstallData, Resolve, ResolveDependenciesError},
25        DownloadedRockspec, FetchSrcError, PackageInstallSpec, UnpackError,
26    },
27    package::PackageReq,
28    progress::{MultiProgress, Progress, ProgressBar},
29    project::project_toml::LocalProjectTomlValidationError,
30    remote_package_db::{RemotePackageDB, RemotePackageDBError},
31    rockspec::Rockspec,
32    tree::EntryType,
33    workspace::{Workspace, WorkspaceError},
34};
35
36#[allow(clippy::large_enum_variant)]
37pub enum VendorTarget {
38    /// Vendor dependencies of a Lux workspace
39    Workspace(Workspace),
40
41    /// Vendor dependencies of a Lua RockSpec
42    Rockspec(RemoteLuaRockspec),
43}
44
45/// Vendor a project's dependencies into the specified directory at `<vendor_dir>`.
46/// After this command completes the vendor directory specified by `<vendor_dir>`
47/// will contain all remote sources from dependencies specified.
48#[derive(Builder)]
49#[builder(start_fn = new, finish_fn(name = _build, vis = ""))]
50pub struct Vendor<'a> {
51    target: VendorTarget,
52
53    /// The directory in which to vendor the dependencies.
54    vendor_dir: PathBuf,
55
56    /// Ignore the project's lockfile.
57    no_lock: Option<bool>,
58
59    /// Don't delete the `<vendor-dir>` when vendoring,{n}
60    /// but rather keep all existing contents of the vendor directory.
61    no_delete: Option<bool>,
62
63    config: &'a Config,
64
65    progress: Option<Arc<Progress<MultiProgress>>>,
66}
67
68#[derive(Error, Debug, Diagnostic)]
69pub enum VendorError {
70    #[error(transparent)]
71    #[diagnostic(transparent)]
72    Workspace(#[from] WorkspaceError),
73    #[error("project validation failed:\n{0}")]
74    #[diagnostic(forward(0))]
75    LocalProjectTomlValidation(#[from] LocalProjectTomlValidationError),
76    #[error("error initialising remote package DB:\n{0}")]
77    #[diagnostic(forward(0))]
78    RemotePackageDB(#[from] RemotePackageDBError),
79    #[error("failed to resolve dependencies:\n{0}")]
80    #[diagnostic(forward(0))]
81    ResolveDependencies(#[from] ResolveDependenciesError),
82    #[error("failed to delete vendor directory {0}:\n{1}")]
83    DeleteVendorDir(String, io::Error),
84    #[error("failed to create vendor directory {0}:\n{1}")]
85    CreateVendorDir(String, io::Error),
86    #[error("failed to create {0}:\n{1}")]
87    CreateSrcRock(String, io::Error),
88    #[error("failed to vendor Lua RockSpec:\n{0}")]
89    LuaRockSpec(String),
90    #[error("failed to write Lua RockSpec {0}:\n{1}")]
91    WriteLuaRockSpec(String, io::Error),
92    #[error("failed to unpack src.rock:\n{0}")]
93    #[diagnostic(forward(0))]
94    Unpack(#[from] UnpackError),
95    #[error("failed to fetch rock source:\n{0}")]
96    #[diagnostic(forward(0))]
97    FetchSrc(#[from] FetchSrcError),
98}
99
100impl<State> VendorBuilder<'_, State>
101where
102    State: vendor_builder::State + vendor_builder::IsComplete,
103{
104    pub async fn vendor_dependencies(self) -> Result<(), VendorError> {
105        do_vendor_dependencies(self._build()).await
106    }
107}
108
109async fn do_vendor_dependencies(args: Vendor<'_>) -> Result<(), VendorError> {
110    let vendor_dir = args.vendor_dir;
111    let no_delete = args.no_delete.unwrap_or(false);
112    let no_lock = args.no_lock.unwrap_or(false);
113    let target = args.target;
114    let config = args.config;
115    let progress = match args.progress {
116        Some(p) => p,
117        None => MultiProgress::new_arc(args.config),
118    };
119    let mut all_packages = Vec::new();
120
121    for lock_type in LocalPackageLockType::iter() {
122        let (package_db, install_specs) =
123            mk_resolve_args(lock_type, no_lock, &target, config, progress.clone()).await?;
124
125        let (dep_tx, mut dep_rx) = tokio::sync::mpsc::unbounded_channel();
126        Resolve::<'_, ReadOnly>::new()
127            .dependencies_tx(dep_tx.clone())
128            .build_dependencies_tx(dep_tx)
129            .packages(install_specs)
130            .package_db(Arc::new(package_db))
131            .config(config)
132            .progress(progress.clone())
133            .get_all_dependencies()
134            .await?;
135
136        while let Some(dep) = dep_rx.recv().await {
137            all_packages.push(dep);
138        }
139    }
140
141    if !no_delete && vendor_dir.exists() {
142        tokio::fs::remove_dir_all(&vendor_dir)
143            .await
144            .map_err(|err| {
145                VendorError::DeleteVendorDir(vendor_dir.to_slash_lossy().to_string(), err)
146            })?;
147    }
148
149    vendor_sources(Arc::new(vendor_dir), progress, config.clone(), all_packages).await
150}
151
152async fn mk_resolve_args(
153    lock_type: LocalPackageLockType,
154    no_lock: bool,
155    target: &VendorTarget,
156    config: &Config,
157    progress: Arc<Progress<MultiProgress>>,
158) -> Result<(RemotePackageDB, Vec<PackageInstallSpec>), VendorError> {
159    match &target {
160        VendorTarget::Workspace(workspace) => {
161            let lockfile = workspace.lockfile()?;
162            let package_db = if !no_lock {
163                lockfile.local_pkg_lock(&lock_type).clone().into()
164            } else {
165                let bar = progress.map(|p| p.new_bar());
166                RemotePackageDB::from_config(config, &bar).await?
167            };
168            let mut install_specs = Vec::new();
169            for project in workspace.members() {
170                let toml = project.toml().into_local()?;
171                push_dependencies(&lock_type, &toml, &mut install_specs)?;
172                if lock_type == LocalPackageLockType::Test {
173                    for test_spec_dependency in toml
174                        .test()
175                        .current_platform()
176                        .test_dependencies(project)
177                        .iter()
178                        .cloned()
179                        .map(|dep| PackageInstallSpec::new(dep, EntryType::Entrypoint).build())
180                    {
181                        install_specs.push(test_spec_dependency);
182                    }
183                }
184            }
185            Ok((package_db, install_specs))
186        }
187        VendorTarget::Rockspec(remote_lua_rockspec) => {
188            let bar = progress.map(|p| p.new_bar());
189            let package_db = RemotePackageDB::from_config(config, &bar).await?;
190            let mut install_specs = Vec::new();
191            push_dependencies(&lock_type, remote_lua_rockspec, &mut install_specs)?;
192            Ok((package_db, install_specs))
193        }
194    }
195}
196
197fn push_dependencies<R: Rockspec>(
198    lock_type: &LocalPackageLockType,
199    rockspec: &R,
200    install_specs: &mut Vec<PackageInstallSpec>,
201) -> Result<(), LocalProjectTomlValidationError> {
202    let dependencies: Vec<&PackageReq> = match lock_type {
203        LocalPackageLockType::Regular => rockspec
204            .dependencies()
205            .current_platform()
206            .iter()
207            .map(|dep| dep.package_req())
208            .collect_vec(),
209        LocalPackageLockType::Test => rockspec
210            .test_dependencies()
211            .current_platform()
212            .iter()
213            .map(|dep| dep.package_req())
214            .collect_vec(),
215        LocalPackageLockType::Build => rockspec
216            .build_dependencies()
217            .current_platform()
218            .iter()
219            .map(|dep| dep.package_req())
220            .collect_vec(),
221    };
222    install_specs.extend(
223        dependencies
224            .into_iter()
225            .unique()
226            .cloned()
227            .map(|dep| PackageInstallSpec::new(dep, EntryType::Entrypoint).build())
228            .collect_vec(),
229    );
230    Ok(())
231}
232
233async fn vendor_sources(
234    vendor_dir: Arc<PathBuf>,
235    progress: Arc<Progress<MultiProgress>>,
236    config: Config,
237    packages: Vec<PackageInstallData>,
238) -> Result<(), VendorError> {
239    futures::stream::iter(packages.into_iter().map(|dep| {
240        let vendor_dir = Arc::clone(&vendor_dir);
241        let progress = Arc::clone(&progress);
242        let config = config.clone();
243        tokio::spawn(async move {
244            match dep.downloaded_rock {
245                crate::operations::RemoteRockDownload::RockspecOnly { rockspec_download } => {
246                    vendor_rockspec_sources(
247                        &vendor_dir,
248                        rockspec_download,
249                        None,
250                        &config,
251                        &progress,
252                    )
253                    .await?
254                }
255                crate::operations::RemoteRockDownload::BinaryRock {
256                    rockspec_download,
257                    packed_rock,
258                } => {
259                    vendor_binary_rock(&vendor_dir, rockspec_download, packed_rock, &progress)
260                        .await?
261                }
262                crate::operations::RemoteRockDownload::SrcRock {
263                    rockspec_download,
264                    src_rock,
265                    source_url,
266                } => {
267                    let src_rock_source = SrcRockSource {
268                        bytes: src_rock,
269                        source_url,
270                    };
271                    vendor_rockspec_sources(
272                        &vendor_dir,
273                        rockspec_download,
274                        Some(src_rock_source),
275                        &config,
276                        &progress,
277                    )
278                    .await?
279                }
280            };
281            Ok::<_, VendorError>(())
282        })
283    }))
284    .buffered(config.max_jobs())
285    .collect::<Vec<_>>()
286    .await
287    .into_iter()
288    .flatten()
289    .try_collect()
290}
291
292async fn vendor_rockspec_sources(
293    vendor_dir: &Path,
294    rockspec_download: DownloadedRockspec,
295    src_rock_source: Option<SrcRockSource>,
296    config: &Config,
297    progress: &Progress<MultiProgress>,
298) -> Result<(), VendorError> {
299    let rockspec = rockspec_download.rockspec;
300    let package = rockspec.package();
301    let version = rockspec.version();
302    let package_version_str = format!("{}@{}", package, version);
303    let bar = progress.map(|p| {
304        p.add(ProgressBar::from(format!(
305            "💼 Vendoring source of {}",
306            &package_version_str,
307        )))
308    });
309    let source_spec = match src_rock_source {
310        Some(src_rock_source) => RemotePackageSourceSpec::SrcRock(src_rock_source),
311        None => RemotePackageSourceSpec::RockSpec(rockspec_download.source_url),
312    };
313
314    let package_vendor_dir = vendor_dir.join(&package_version_str);
315
316    tokio::fs::create_dir_all(&package_vendor_dir)
317        .await
318        .map_err(|err| {
319            VendorError::CreateVendorDir(package_vendor_dir.to_slash_lossy().to_string(), err)
320        })?;
321
322    let rockspec_lua_content = rockspec
323        .to_lua_remote_rockspec_string()
324        .map_err(|err| VendorError::LuaRockSpec(err.to_string()))?;
325
326    let rockspec_file_name = format!("{}-{}.rockspec", package, version);
327    let rockspec_path = vendor_dir.join(rockspec_file_name);
328    tokio::fs::write(&rockspec_path, rockspec_lua_content)
329        .await
330        .map_err(|err| {
331            VendorError::WriteLuaRockSpec(rockspec_path.to_slash_lossy().to_string(), err)
332        })?;
333
334    match source_spec {
335        RemotePackageSourceSpec::SrcRock(SrcRockSource {
336            bytes,
337            source_url: _,
338        }) => {
339            let cursor = Cursor::new(&bytes);
340            operations::unpack_src_rock(cursor, package_vendor_dir, &bar).await?;
341        }
342        RemotePackageSourceSpec::RockSpec(source_url) => {
343            operations::FetchSrc::new(&package_vendor_dir, &rockspec, config, &bar)
344                .maybe_source_url(source_url)
345                .fetch_internal()
346                .await?;
347        }
348    }
349
350    bar.map(|bar| bar.finish_and_clear());
351
352    Ok(())
353}
354
355async fn vendor_binary_rock(
356    vendor_dir: &Path,
357    rockspec_download: DownloadedRockspec,
358    packed_rock: Bytes,
359    progress: &Progress<MultiProgress>,
360) -> Result<(), VendorError> {
361    let rockspec = rockspec_download.rockspec;
362    let package = rockspec.package();
363    let version = rockspec.version();
364
365    let file_name = format!("{}@{}.rock", package, version);
366
367    let bar = progress.map(|p| {
368        p.add(ProgressBar::from(format!(
369            "💼 Vendoring pre-built binary .rock: {}",
370            &file_name,
371        )))
372    });
373
374    tokio::fs::create_dir_all(&vendor_dir)
375        .await
376        .map_err(|err| {
377            VendorError::CreateVendorDir(vendor_dir.to_slash_lossy().to_string(), err)
378        })?;
379
380    let dest_file = vendor_dir.join(&file_name);
381    let mut file = File::create(&dest_file)
382        .await
383        .map_err(|err| VendorError::CreateSrcRock(dest_file.to_slash_lossy().to_string(), err))?;
384    file.write_all(&packed_rock)
385        .await
386        .map_err(|err| VendorError::CreateSrcRock(dest_file.to_slash_lossy().to_string(), err))?;
387
388    bar.map(|bar| bar.finish_and_clear());
389
390    Ok(())
391}
lux_lib/operations/vendor.rs

lux_lib/operations/
vendor.rs
