lux_lib/operations/
vendor.rs

1use std::{
2    io::{self, Cursor},
3    path::{Path, PathBuf},
4    sync::Arc,
5};
6
7use bon::Builder;
8use bytes::Bytes;
9use futures::StreamExt;
10use itertools::Itertools;
11use path_slash::PathExt;
12use strum::IntoEnumIterator;
13use thiserror::Error;
14use tokio::{fs::File, io::AsyncWriteExt};
15
16use crate::{
17    build::{RemotePackageSourceSpec, SrcRockSource},
18    config::Config,
19    lockfile::{LocalPackageLockType, ReadOnly},
20    lua_rockspec::RemoteLuaRockspec,
21    operations::{
22        self,
23        resolve::{PackageInstallData, Resolve, ResolveDependenciesError},
24        DownloadedRockspec, FetchSrcError, PackageInstallSpec, UnpackError,
25    },
26    package::PackageReq,
27    progress::{MultiProgress, Progress, ProgressBar},
28    project::project_toml::LocalProjectTomlValidationError,
29    remote_package_db::{RemotePackageDB, RemotePackageDBError},
30    rockspec::Rockspec,
31    tree::EntryType,
32    workspace::{Workspace, WorkspaceError},
33};
34
35pub enum VendorTarget {
36    /// Vendor dependencies of a Lux workspace
37    Workspace(Workspace),
38    /// Vendor dependencies of a Lua RockSpec
39    Rockspec(RemoteLuaRockspec),
40}
41
42/// Vendor a project's dependencies into the specified directory at `<vendor_dir>`.
43/// After this command completes the vendor directory specified by `<vendor_dir>`
44/// will contain all remote sources from dependencies specified.
45#[derive(Builder)]
46#[builder(start_fn = new, finish_fn(name = _build, vis = ""))]
47pub struct Vendor<'a> {
48    target: VendorTarget,
49
50    /// The directory in which to vendor the dependencies.
51    vendor_dir: PathBuf,
52
53    /// Ignore the project's lockfile.
54    no_lock: Option<bool>,
55
56    /// Don't delete the `<vendor-dir>` when vendoring,{n}
57    /// but rather keep all existing contents of the vendor directory.
58    no_delete: Option<bool>,
59
60    config: &'a Config,
61
62    progress: Option<Arc<Progress<MultiProgress>>>,
63}
64
65#[derive(Error, Debug)]
66pub enum VendorError {
67    #[error(transparent)]
68    Workspace(#[from] WorkspaceError),
69    #[error("project validation failed:\n{0}")]
70    LocalProjectTomlValidation(#[from] LocalProjectTomlValidationError),
71    #[error("error initialising remote package DB:\n{0}")]
72    RemotePackageDB(#[from] RemotePackageDBError),
73    #[error("failed to resolve dependencies:\n{0}")]
74    ResolveDependencies(#[from] ResolveDependenciesError),
75    #[error("failed to delete vendor directory {0}:\n{1}")]
76    DeleteVendorDir(String, io::Error),
77    #[error("failed to create vendor directory {0}:\n{1}")]
78    CreateVendorDir(String, io::Error),
79    #[error("failed to create {0}:\n{1}")]
80    CreateSrcRock(String, io::Error),
81    #[error("failed to vendor Lua RockSpec:\n{0}")]
82    LuaRockSpec(String),
83    #[error("failed to write Lua RockSpec {0}:\n{1}")]
84    WriteLuaRockSpec(String, io::Error),
85    #[error("failed to unpack src.rock:\n{0}")]
86    Unpack(#[from] UnpackError),
87    #[error("failed to fetch rock source:\n{0}")]
88    FetchSrc(#[from] FetchSrcError),
89}
90
91impl<State> VendorBuilder<'_, State>
92where
93    State: vendor_builder::State + vendor_builder::IsComplete,
94{
95    pub async fn vendor_dependencies(self) -> Result<(), VendorError> {
96        do_vendor_dependencies(self._build()).await
97    }
98}
99
100async fn do_vendor_dependencies(args: Vendor<'_>) -> Result<(), VendorError> {
101    let vendor_dir = args.vendor_dir;
102    let no_delete = args.no_delete.unwrap_or(false);
103    let no_lock = args.no_lock.unwrap_or(false);
104    let target = args.target;
105    let config = args.config;
106    let progress = match args.progress {
107        Some(p) => p,
108        None => MultiProgress::new_arc(args.config),
109    };
110    let mut all_packages = Vec::new();
111
112    for lock_type in LocalPackageLockType::iter() {
113        let (package_db, install_specs) =
114            mk_resolve_args(lock_type, no_lock, &target, config, progress.clone()).await?;
115
116        let (dep_tx, mut dep_rx) = tokio::sync::mpsc::unbounded_channel();
117        Resolve::<'_, ReadOnly>::new()
118            .dependencies_tx(dep_tx.clone())
119            .build_dependencies_tx(dep_tx)
120            .packages(install_specs)
121            .package_db(Arc::new(package_db))
122            .config(config)
123            .progress(progress.clone())
124            .get_all_dependencies()
125            .await?;
126
127        while let Some(dep) = dep_rx.recv().await {
128            all_packages.push(dep);
129        }
130    }
131
132    if !no_delete && vendor_dir.exists() {
133        tokio::fs::remove_dir_all(&vendor_dir)
134            .await
135            .map_err(|err| {
136                VendorError::DeleteVendorDir(vendor_dir.to_slash_lossy().to_string(), err)
137            })?;
138    }
139
140    vendor_sources(Arc::new(vendor_dir), progress, config.clone(), all_packages).await
141}
142
143async fn mk_resolve_args(
144    lock_type: LocalPackageLockType,
145    no_lock: bool,
146    target: &VendorTarget,
147    config: &Config,
148    progress: Arc<Progress<MultiProgress>>,
149) -> Result<(RemotePackageDB, Vec<PackageInstallSpec>), VendorError> {
150    match &target {
151        VendorTarget::Workspace(workspace) => {
152            let lockfile = workspace.lockfile()?;
153            let package_db = if !no_lock {
154                lockfile.local_pkg_lock(&lock_type).clone().into()
155            } else {
156                let bar = progress.map(|p| p.new_bar());
157                RemotePackageDB::from_config(config, &bar).await?
158            };
159            let mut install_specs = Vec::new();
160            for project in workspace.members() {
161                let toml = project.toml().into_local()?;
162                push_dependencies(&lock_type, &toml, &mut install_specs)?;
163                if lock_type == LocalPackageLockType::Test {
164                    for test_spec_dependency in toml
165                        .test()
166                        .current_platform()
167                        .test_dependencies(project)
168                        .iter()
169                        .cloned()
170                        .map(|dep| PackageInstallSpec::new(dep, EntryType::Entrypoint).build())
171                    {
172                        install_specs.push(test_spec_dependency);
173                    }
174                }
175            }
176            Ok((package_db, install_specs))
177        }
178        VendorTarget::Rockspec(remote_lua_rockspec) => {
179            let bar = progress.map(|p| p.new_bar());
180            let package_db = RemotePackageDB::from_config(config, &bar).await?;
181            let mut install_specs = Vec::new();
182            push_dependencies(&lock_type, remote_lua_rockspec, &mut install_specs)?;
183            Ok((package_db, install_specs))
184        }
185    }
186}
187
188fn push_dependencies<R: Rockspec>(
189    lock_type: &LocalPackageLockType,
190    rockspec: &R,
191    install_specs: &mut Vec<PackageInstallSpec>,
192) -> Result<(), LocalProjectTomlValidationError> {
193    let dependencies: Vec<&PackageReq> = match lock_type {
194        LocalPackageLockType::Regular => rockspec
195            .dependencies()
196            .current_platform()
197            .iter()
198            .map(|dep| dep.package_req())
199            .collect_vec(),
200        LocalPackageLockType::Test => rockspec
201            .test_dependencies()
202            .current_platform()
203            .iter()
204            .map(|dep| dep.package_req())
205            .collect_vec(),
206        LocalPackageLockType::Build => rockspec
207            .build_dependencies()
208            .current_platform()
209            .iter()
210            .map(|dep| dep.package_req())
211            .collect_vec(),
212    };
213    install_specs.extend(
214        dependencies
215            .into_iter()
216            .unique()
217            .cloned()
218            .map(|dep| PackageInstallSpec::new(dep, EntryType::Entrypoint).build())
219            .collect_vec(),
220    );
221    Ok(())
222}
223
224async fn vendor_sources(
225    vendor_dir: Arc<PathBuf>,
226    progress: Arc<Progress<MultiProgress>>,
227    config: Config,
228    packages: Vec<PackageInstallData>,
229) -> Result<(), VendorError> {
230    futures::stream::iter(packages.into_iter().map(|dep| {
231        let vendor_dir = Arc::clone(&vendor_dir);
232        let progress = Arc::clone(&progress);
233        let config = config.clone();
234        tokio::spawn(async move {
235            match dep.downloaded_rock {
236                crate::operations::RemoteRockDownload::RockspecOnly { rockspec_download } => {
237                    vendor_rockspec_sources(
238                        &vendor_dir,
239                        rockspec_download,
240                        None,
241                        &config,
242                        &progress,
243                    )
244                    .await?
245                }
246                crate::operations::RemoteRockDownload::BinaryRock {
247                    rockspec_download,
248                    packed_rock,
249                } => {
250                    vendor_binary_rock(&vendor_dir, rockspec_download, packed_rock, &progress)
251                        .await?
252                }
253                crate::operations::RemoteRockDownload::SrcRock {
254                    rockspec_download,
255                    src_rock,
256                    source_url,
257                } => {
258                    let src_rock_source = SrcRockSource {
259                        bytes: src_rock,
260                        source_url,
261                    };
262                    vendor_rockspec_sources(
263                        &vendor_dir,
264                        rockspec_download,
265                        Some(src_rock_source),
266                        &config,
267                        &progress,
268                    )
269                    .await?
270                }
271            };
272            Ok::<_, VendorError>(())
273        })
274    }))
275    .buffered(config.max_jobs())
276    .collect::<Vec<_>>()
277    .await
278    .into_iter()
279    .flatten()
280    .try_collect()
281}
282
283async fn vendor_rockspec_sources(
284    vendor_dir: &Path,
285    rockspec_download: DownloadedRockspec,
286    src_rock_source: Option<SrcRockSource>,
287    config: &Config,
288    progress: &Progress<MultiProgress>,
289) -> Result<(), VendorError> {
290    let rockspec = rockspec_download.rockspec;
291    let package = rockspec.package();
292    let version = rockspec.version();
293    let package_version_str = format!("{}@{}", package, version);
294    let bar = progress.map(|p| {
295        p.add(ProgressBar::from(format!(
296            "💼 Vendoring source of {}",
297            &package_version_str,
298        )))
299    });
300    let source_spec = match src_rock_source {
301        Some(src_rock_source) => RemotePackageSourceSpec::SrcRock(src_rock_source),
302        None => RemotePackageSourceSpec::RockSpec(rockspec_download.source_url),
303    };
304
305    let package_vendor_dir = vendor_dir.join(&package_version_str);
306
307    tokio::fs::create_dir_all(&package_vendor_dir)
308        .await
309        .map_err(|err| {
310            VendorError::CreateVendorDir(package_vendor_dir.to_slash_lossy().to_string(), err)
311        })?;
312
313    let rockspec_lua_content = rockspec
314        .to_lua_remote_rockspec_string()
315        .map_err(|err| VendorError::LuaRockSpec(err.to_string()))?;
316
317    let rockspec_file_name = format!("{}-{}.rockspec", package, version);
318    let rockspec_path = vendor_dir.join(rockspec_file_name);
319    tokio::fs::write(&rockspec_path, rockspec_lua_content)
320        .await
321        .map_err(|err| {
322            VendorError::WriteLuaRockSpec(rockspec_path.to_slash_lossy().to_string(), err)
323        })?;
324
325    match source_spec {
326        RemotePackageSourceSpec::SrcRock(SrcRockSource {
327            bytes,
328            source_url: _,
329        }) => {
330            let cursor = Cursor::new(&bytes);
331            operations::unpack_src_rock(cursor, package_vendor_dir, &bar).await?;
332        }
333        RemotePackageSourceSpec::RockSpec(source_url) => {
334            operations::FetchSrc::new(&package_vendor_dir, &rockspec, config, &bar)
335                .maybe_source_url(source_url)
336                .fetch_internal()
337                .await?;
338        }
339    }
340
341    bar.map(|bar| bar.finish_and_clear());
342
343    Ok(())
344}
345
346async fn vendor_binary_rock(
347    vendor_dir: &Path,
348    rockspec_download: DownloadedRockspec,
349    packed_rock: Bytes,
350    progress: &Progress<MultiProgress>,
351) -> Result<(), VendorError> {
352    let rockspec = rockspec_download.rockspec;
353    let package = rockspec.package();
354    let version = rockspec.version();
355
356    let file_name = format!("{}@{}.rock", package, version);
357
358    let bar = progress.map(|p| {
359        p.add(ProgressBar::from(format!(
360            "💼 Vendoring pre-built binary .rock: {}",
361            &file_name,
362        )))
363    });
364
365    tokio::fs::create_dir_all(&vendor_dir)
366        .await
367        .map_err(|err| {
368            VendorError::CreateVendorDir(vendor_dir.to_slash_lossy().to_string(), err)
369        })?;
370
371    let dest_file = vendor_dir.join(&file_name);
372    let mut file = File::create(&dest_file)
373        .await
374        .map_err(|err| VendorError::CreateSrcRock(dest_file.to_slash_lossy().to_string(), err))?;
375    file.write_all(&packed_rock)
376        .await
377        .map_err(|err| VendorError::CreateSrcRock(dest_file.to_slash_lossy().to_string(), err))?;
378
379    bar.map(|bar| bar.finish_and_clear());
380
381    Ok(())
382}
lux_lib/operations/vendor.rs

lux_lib/operations/
vendor.rs
