lux_lib/operations/
download.rs

1use std::{
2    collections::HashMap,
3    io::{Cursor, Read},
4    path::PathBuf,
5    string::FromUtf8Error,
6};
7
8use bon::Builder;
9use bytes::Bytes;
10use miette::Diagnostic;
11use thiserror::Error;
12use url::{ParseError, Url};
13
14use crate::{
15    config::Config,
16    fs,
17    git::{GitRef, GitSource},
18    lockfile::RemotePackageSourceUrl,
19    lua_rockspec::{LuaRockspecError, RemoteLuaRockspec, RockSourceSpec},
20    luarocks,
21    package::{
22        PackageName, PackageReq, PackageSpec, PackageSpecFromPackageReqError, PackageVersion,
23        RemotePackageTypeFilterSpec,
24    },
25    package_db::{PackageDB, PackageDBError, SearchError},
26    remote_package_source::RemotePackageSource,
27    reqwest::{RequestBuilderExt, RequestError},
28    rockspec::Rockspec,
29};
30
31/// Builder for a rock downloader.
32pub struct Download<'a> {
33    package_req: &'a PackageReq,
34    package_db: Option<&'a PackageDB>,
35    config: &'a Config,
36}
37
38impl<'a> Download<'a> {
39    /// Construct a new `.src.rock` downloader.
40    pub fn new(package_req: &'a PackageReq, config: &'a Config) -> Self {
41        Self {
42            package_req,
43            package_db: None,
44            config,
45        }
46    }
47
48    /// Sets the package database to use for searching for packages.
49    /// Instantiated from the config if not set.
50    pub fn package_db(self, package_db: &'a PackageDB) -> Self {
51        Self {
52            package_db: Some(package_db),
53            ..self
54        }
55    }
56
57    /// Download the package's Rockspec.
58    pub async fn download_rockspec(self) -> Result<DownloadedRockspec, SearchAndDownloadError> {
59        match self.package_db {
60            Some(db) => download_rockspec(self.package_req, db, self.config).await,
61            None => {
62                let db = PackageDB::from_config(self.config).await?;
63                download_rockspec(self.package_req, &db, self.config).await
64            }
65        }
66    }
67
68    /// Download a `.src.rock` to a file.
69    /// `destination_dir` defaults to the current working directory if not set.
70    pub async fn download_src_rock_to_file(
71        self,
72        destination_dir: Option<PathBuf>,
73    ) -> Result<DownloadedPackedRock, SearchAndDownloadError> {
74        match self.package_db {
75            Some(db) => {
76                download_src_rock_to_file(self.package_req, destination_dir, db, self.config).await
77            }
78            None => {
79                let db = PackageDB::from_config(self.config).await?;
80                download_src_rock_to_file(self.package_req, destination_dir, &db, self.config).await
81            }
82        }
83    }
84
85    /// Search for a `.src.rock` and download it to memory.
86    pub async fn search_and_download_src_rock(
87        self,
88    ) -> Result<DownloadedPackedRockBytes, SearchAndDownloadError> {
89        match self.package_db {
90            Some(db) => search_and_download_src_rock(self.package_req, db, self.config).await,
91            None => {
92                let db = PackageDB::from_config(self.config).await?;
93                search_and_download_src_rock(self.package_req, &db, self.config).await
94            }
95        }
96    }
97
98    pub(crate) async fn download_remote_rock(
99        self,
100    ) -> Result<RemoteRockDownload, SearchAndDownloadError> {
101        match self.package_db {
102            Some(db) => download_remote_rock(self.package_req, db, self.config).await,
103            None => {
104                let db = PackageDB::from_config(self.config).await?;
105                download_remote_rock(self.package_req, &db, self.config).await
106            }
107        }
108    }
109}
110
111pub struct DownloadedPackedRockBytes {
112    pub name: PackageName,
113    pub version: PackageVersion,
114    pub bytes: Bytes,
115    pub file_name: String,
116    pub url: Url,
117}
118
119pub struct DownloadedPackedRock {
120    pub name: PackageName,
121    pub version: PackageVersion,
122    pub path: PathBuf,
123}
124
125/// Remote Lua RockSpec that has been downloaded from a remote server, along with its source metadata
126#[derive(Clone, Debug)]
127pub struct DownloadedRockspec {
128    pub rockspec: RemoteLuaRockspec,
129    pub(crate) source: RemotePackageSource,
130    pub(crate) source_url: Option<RemotePackageSourceUrl>,
131}
132
133#[derive(Clone, Debug)]
134pub(crate) enum RemoteRockDownload {
135    RockspecOnly {
136        rockspec_download: DownloadedRockspec,
137    },
138    BinaryRock {
139        rockspec_download: DownloadedRockspec,
140        packed_rock: Bytes,
141    },
142    SrcRock {
143        rockspec_download: DownloadedRockspec,
144        src_rock: Bytes,
145        source_url: RemotePackageSourceUrl,
146    },
147}
148
149impl RemoteRockDownload {
150    pub fn rockspec(&self) -> &RemoteLuaRockspec {
151        &self.rockspec_download().rockspec
152    }
153    pub fn rockspec_download(&self) -> &DownloadedRockspec {
154        match self {
155            Self::RockspecOnly { rockspec_download }
156            | Self::BinaryRock {
157                rockspec_download, ..
158            }
159            | Self::SrcRock {
160                rockspec_download, ..
161            } => rockspec_download,
162        }
163    }
164    // Instead of downloading a rockspec, generate one from a `PackageReq` and a `RockSourceSpec`.
165    pub(crate) fn from_package_req_and_source_spec(
166        package_req: PackageReq,
167        source_spec: RockSourceSpec,
168    ) -> Result<Self, SearchAndDownloadError> {
169        let package_spec: PackageSpec = package_req.try_into()?;
170        let source_url = Some(match &source_spec {
171            RockSourceSpec::Git(GitSource { url, git_ref }) => {
172                let checkout_ref = match git_ref {
173                    Some(GitRef::Tag(tag)) => tag.clone(),
174                    Some(GitRef::Branch(branch)) => branch.clone(),
175                    None => {
176                        return Err(SearchAndDownloadError::MissingCheckoutRef(url.to_string()))
177                    }
178                };
179                RemotePackageSourceUrl::Git {
180                    url: url.to_string(),
181                    checkout_ref,
182                    submodules: false, // unknown until the source is fetched
183                }
184            }
185            RockSourceSpec::File(path) => RemotePackageSourceUrl::File { path: path.clone() },
186            RockSourceSpec::Url(url) => RemotePackageSourceUrl::Url { url: url.clone() },
187            RockSourceSpec::Wally(_) => {
188                return Err(SearchAndDownloadError::WallySource(
189                    package_spec.name().to_string(),
190                ))
191            }
192        });
193        let rockspec = RemoteLuaRockspec::from_source_spec(
194            package_spec,
195            source_spec,
196            Vec::new(),
197            HashMap::new(),
198        );
199        let rockspec_content = rockspec
200            .to_lua_remote_rockspec_string()
201            .expect("the infallible happened");
202        let rockspec_download = DownloadedRockspec {
203            rockspec,
204            source_url,
205            source: RemotePackageSource::RockspecContent(rockspec_content),
206        };
207        Ok(Self::RockspecOnly { rockspec_download })
208    }
209}
210
211#[derive(Error, Debug, Diagnostic)]
212pub enum DownloadRockspecError {
213    #[error("failed to download rockspec")]
214    #[diagnostic(transparent)]
215    Request(#[from] RequestError),
216    #[error("failed to convert rockspec response")]
217    #[diagnostic(help(
218        r#"the server returned a response that is not valid UTF-8.
219check your network connection and server configuration.
220if the issue persists, the server may be temporarily unavailable."#
221    ))]
222    ResponseConversion(#[from] FromUtf8Error),
223    #[error("error initialising remote package DB")]
224    #[diagnostic(forward(0))]
225    PackageDB(#[from] PackageDBError),
226    #[error(transparent)]
227    #[diagnostic(transparent)]
228    DownloadSrcRock(#[from] DownloadSrcRockError),
229}
230
231impl From<reqwest::Error> for DownloadRockspecError {
232    fn from(err: reqwest::Error) -> Self {
233        Self::Request(err.into())
234    }
235}
236
237/// Find and download a rockspec for a given package requirement
238async fn download_rockspec(
239    package_req: &PackageReq,
240    package_db: &PackageDB,
241    config: &Config,
242) -> Result<DownloadedRockspec, SearchAndDownloadError> {
243    let rockspec = match download_remote_rock(package_req, package_db, config).await? {
244        RemoteRockDownload::RockspecOnly {
245            rockspec_download: rockspec,
246        } => rockspec,
247        RemoteRockDownload::BinaryRock {
248            rockspec_download: rockspec,
249            ..
250        } => rockspec,
251        RemoteRockDownload::SrcRock {
252            rockspec_download: rockspec,
253            ..
254        } => rockspec,
255    };
256    Ok(rockspec)
257}
258
259#[tracing::instrument(
260    name = "Downloading rock",
261    level = "info",
262    skip_all,
263    fields(package = package_req.to_string(),),
264)]
265async fn download_remote_rock(
266    package_req: &PackageReq,
267    package_db: &PackageDB,
268    config: &Config,
269) -> Result<RemoteRockDownload, SearchAndDownloadError> {
270    let remote_package = package_db.find(package_req, None).await?;
271    match &remote_package.source {
272        RemotePackageSource::LuarocksRockspec(url) => {
273            let package = &remote_package.spec;
274            let rockspec_name = format!("{}-{}.rockspec", package.name(), package.version());
275            let url = url
276                .join(&rockspec_name)
277                .map_err(|source| SearchAndDownloadError::Parse {
278                    source,
279                    url: format!("{}/{}", url, rockspec_name),
280                })?;
281            let bytes = crate::reqwest::https_client(config)?
282                .get(url.clone())
283                .apply_access_token(config, &url)
284                .send()
285                .await?
286                .error_for_status()?
287                .bytes()
288                .await?;
289            let content = String::from_utf8(bytes.into())?;
290            let rockspec = DownloadedRockspec {
291                rockspec: RemoteLuaRockspec::new(&content)
292                    .map_err(|err| SearchAndDownloadError::Rockspec(Box::new(err)))?,
293                source: remote_package.source,
294                source_url: remote_package.source_url,
295            };
296            Ok(RemoteRockDownload::RockspecOnly {
297                rockspec_download: rockspec,
298            })
299        }
300        RemotePackageSource::RockspecContent(content) => {
301            let rockspec = DownloadedRockspec {
302                rockspec: RemoteLuaRockspec::new(content)
303                    .map_err(|err| SearchAndDownloadError::Rockspec(Box::new(err)))?,
304                source: remote_package.source,
305                source_url: remote_package.source_url,
306            };
307            Ok(RemoteRockDownload::RockspecOnly {
308                rockspec_download: rockspec,
309            })
310        }
311        RemotePackageSource::LuarocksBinaryRock(url) => {
312            // prioritise lockfile source_url
313            let url = if let Some(RemotePackageSourceUrl::Url { url }) = &remote_package.source_url
314            {
315                url
316            } else {
317                url
318            };
319            let rock = download_binary_rock(&remote_package.spec, url, config).await?;
320            let rockspec = DownloadedRockspec {
321                rockspec: unpack_rockspec(&rock).await?,
322                source: remote_package.source,
323                source_url: remote_package.source_url,
324            };
325            Ok(RemoteRockDownload::BinaryRock {
326                rockspec_download: rockspec,
327                packed_rock: rock.bytes,
328            })
329        }
330        RemotePackageSource::LuarocksSrcRock(url) => {
331            // prioritise lockfile source_url
332            let url = if let Some(RemotePackageSourceUrl::Url { url }) = &remote_package.source_url
333            {
334                url.clone()
335            } else {
336                url.clone()
337            };
338            let rock = download_src_rock(&remote_package.spec, &url, config).await?;
339            let rockspec = DownloadedRockspec {
340                rockspec: unpack_rockspec(&rock).await?,
341                source: remote_package.source,
342                source_url: remote_package.source_url,
343            };
344            Ok(RemoteRockDownload::SrcRock {
345                rockspec_download: rockspec,
346                src_rock: rock.bytes,
347                source_url: RemotePackageSourceUrl::Url { url },
348            })
349        }
350        RemotePackageSource::Wally(_) => Err(SearchAndDownloadError::WallySource(
351            remote_package.spec.name().to_string(),
352        )),
353        RemotePackageSource::Local => Err(SearchAndDownloadError::LocalSource),
354        #[cfg(test)]
355        RemotePackageSource::Test => unimplemented!(),
356    }
357}
358
359#[derive(Error, Debug, Diagnostic)]
360pub enum SearchAndDownloadError {
361    #[error("failed to parse rock URL '{url}'")]
362    Parse {
363        source: url::ParseError,
364        url: String,
365    },
366    #[error(transparent)]
367    #[diagnostic(transparent)]
368    Search(#[from] SearchError),
369    #[error(transparent)]
370    #[diagnostic(transparent)]
371    Download(#[from] DownloadSrcRockError),
372    #[error(transparent)]
373    #[diagnostic(transparent)]
374    DownloadRockspec(#[from] DownloadRockspecError),
375    #[error(transparent)]
376    #[diagnostic(transparent)]
377    Fs(#[from] fs::FsError),
378    #[error("UTF-8 conversion failed")]
379    #[diagnostic(help(
380        r#"the server returned a response that is not valid UTF-8.
381check your network connection and server configuration.
382if the issue persists, the server may be temporarily unavailable."#
383    ))]
384    Utf8(#[from] FromUtf8Error),
385    #[error(transparent)]
386    #[diagnostic(transparent)]
387    Rockspec(Box<LuaRockspecError>),
388    #[error("error initialising remote package DB")]
389    #[diagnostic(forward(0))]
390    PackageDB(#[from] PackageDBError),
391    #[error("failed to read packed rock {0}:\n{1}")]
392    #[diagnostic(help(
393        r#"the downloaded rock may be corrupted.
394check your network connection and rerun the command to re-download it."#
395    ))]
396    ZipRead(String, zip::result::ZipError),
397    #[error("failed to extract packed rock {0}:\n{1}")]
398    #[diagnostic(help(
399        r#"the downloaded rock may be corrupted.
400check your network connection and rerun the command to re-download it."#
401    ))]
402    ZipExtract(String, zip::result::ZipError),
403    #[error("{0} not found in the packed rock.")]
404    #[diagnostic(help(
405        r#"the packed rock does not contain the expected rockspec.
406check your network connection and rerun the command to re-download it."#
407    ))]
408    RockspecNotFoundInPackedRock(String),
409    #[error(transparent)]
410    #[diagnostic(transparent)]
411    PackageSpecFromPackageReq(#[from] PackageSpecFromPackageReqError),
412    #[error("git source {0} without a revision or tag.")]
413    #[diagnostic(help(
414        r#"lux requires a pinned revision or tag to ensure reproducible builds.
415without one, the same version could resolve to different code at different times.
416try a different version, or report it to the package maintainer."#
417    ))]
418    MissingCheckoutRef(String),
419    #[error("cannot download from a local rock source.")]
420    #[diagnostic(
421        help(
422            r#"lux cannot download rocks from a local path.
423for local dependencies, use `path` in your lux.toml."#
424        ),
425        url("https://lux.lumen-labs.org/reference/lux-toml#local-dependencies")
426    )]
427    LocalSource,
428    #[error("cannot download from a local rock or embedded rockspec source.")]
429    #[diagnostic(
430        help(
431            r#"lux cannot download rocks from a local path.
432for local dependencies, use `path` in your lux.toml."#
433        ),
434        url("https://lux.lumen-labs.org/reference/lux-toml#local-dependencies")
435    )]
436    NonURLSource,
437    #[error("failed to resolve wally package '{0}'")]
438    #[diagnostic(
439        code(lux_lib::wally::unresolved_source),
440        help("this is a bug, please report it.")
441    )]
442    WallySource(String),
443    #[error("client error")]
444    #[diagnostic(transparent)]
445    Request(#[from] RequestError),
446}
447
448impl From<reqwest::Error> for SearchAndDownloadError {
449    fn from(err: reqwest::Error) -> Self {
450        Self::Request(err.into())
451    }
452}
453
454async fn search_and_download_src_rock(
455    package_req: &PackageReq,
456    package_db: &PackageDB,
457    config: &Config,
458) -> Result<DownloadedPackedRockBytes, SearchAndDownloadError> {
459    let filter = Some(RemotePackageTypeFilterSpec {
460        rockspec: false,
461        binary: false,
462        src: true,
463    });
464    let remote_package = package_db.find(package_req, filter).await?;
465    let source_url = remote_package
466        .source
467        .url()
468        .ok_or(SearchAndDownloadError::NonURLSource)?;
469    Ok(download_src_rock(&remote_package.spec, &source_url, config).await?)
470}
471
472#[derive(Error, Debug, Diagnostic)]
473pub enum DownloadSrcRockError {
474    #[error("failed to download source rock")]
475    #[diagnostic(transparent)]
476    Request(#[from] RequestError),
477    #[error("failed to parse source rock URL")]
478    Parse(#[from] ParseError),
479}
480
481impl From<reqwest::Error> for DownloadSrcRockError {
482    fn from(err: reqwest::Error) -> Self {
483        Self::Request(err.into())
484    }
485}
486
487#[tracing::instrument(name = "Downloading src.rock", skip_all)]
488pub(crate) async fn download_src_rock(
489    package: &PackageSpec,
490    server_url: &Url,
491    config: &Config,
492) -> Result<DownloadedPackedRockBytes, DownloadSrcRockError> {
493    ArchiveDownload::new()
494        .package(package)
495        .server_url(server_url)
496        .config(config)
497        .ext("src.rock")
498        .download()
499        .await
500}
501
502#[tracing::instrument(name = "Downloading binary rock", skip_all)]
503pub(crate) async fn download_binary_rock(
504    package: &PackageSpec,
505    server_url: &Url,
506    config: &Config,
507) -> Result<DownloadedPackedRockBytes, DownloadSrcRockError> {
508    let ext = format!("{}.rock", luarocks::current_platform_luarocks_identifier());
509    ArchiveDownload::new()
510        .package(package)
511        .server_url(server_url)
512        .config(config)
513        .ext(&ext)
514        .fallback_ext("all.rock")
515        .download()
516        .await
517}
518
519#[tracing::instrument(name = "Downloading package", skip_all)]
520async fn download_src_rock_to_file(
521    package_req: &PackageReq,
522    destination_dir: Option<PathBuf>,
523    package_db: &PackageDB,
524    config: &Config,
525) -> Result<DownloadedPackedRock, SearchAndDownloadError> {
526    let rock = search_and_download_src_rock(package_req, package_db, config).await?;
527    let full_rock_name = mk_packed_rock_name(&rock.name, &rock.version, "src.rock");
528    fs::tokio::write(
529        destination_dir
530            .map(|dest| dest.join(&full_rock_name))
531            .unwrap_or_else(|| full_rock_name.clone().into()),
532        &rock.bytes,
533    )
534    .await?;
535
536    Ok(DownloadedPackedRock {
537        name: rock.name.to_owned(),
538        version: rock.version.to_owned(),
539        path: full_rock_name.into(),
540    })
541}
542
543#[derive(Builder)]
544#[builder(start_fn = new, finish_fn(name = _build, vis = ""))]
545struct ArchiveDownload<'a> {
546    package: &'a PackageSpec,
547
548    server_url: &'a Url,
549
550    ext: &'a str,
551
552    fallback_ext: Option<&'a str>,
553
554    config: &'a Config,
555}
556
557impl<State> ArchiveDownloadBuilder<'_, State>
558where
559    State: archive_download_builder::State + archive_download_builder::IsComplete,
560{
561    async fn download(self) -> Result<DownloadedPackedRockBytes, DownloadSrcRockError> {
562        let args = self._build();
563        download_impl(args).await
564    }
565}
566
567#[tracing::instrument(
568    name = "Downloading",
569    level = "info",
570    skip_all,
571    fields(
572        package = args.package.name().to_string(),
573        version = args.package.version().to_string(),
574        server = args.server_url.to_string(),
575    ),
576)]
577async fn download_impl(
578    args: ArchiveDownload<'_>,
579) -> Result<DownloadedPackedRockBytes, DownloadSrcRockError> {
580    let package = args.package;
581    let ext = args.ext;
582    let server_url = args.server_url;
583    let full_rock_name = mk_packed_rock_name(package.name(), package.version(), ext);
584    tracing::debug!(message = format!("📥 Downloading {full_rock_name}").as_str());
585    let url = server_url.join(&full_rock_name)?;
586    let response = crate::reqwest::https_client(args.config)?
587        .get(url.clone())
588        .apply_access_token(args.config, &url)
589        .send()
590        .await?;
591    let bytes = if response.status().is_success() {
592        response.bytes().await
593    } else {
594        match args.fallback_ext {
595            Some(ext) => {
596                let full_rock_name = mk_packed_rock_name(package.name(), package.version(), ext);
597                let url = server_url.join(&full_rock_name)?;
598                crate::reqwest::https_client(args.config)?
599                    .get(url.clone())
600                    .apply_access_token(args.config, &url)
601                    .send()
602                    .await?
603                    .error_for_status()?
604                    .bytes()
605                    .await
606            }
607            None => response.error_for_status()?.bytes().await,
608        }
609    }?;
610    Ok(DownloadedPackedRockBytes {
611        name: package.name().clone(),
612        version: package.version().clone(),
613        bytes,
614        file_name: full_rock_name,
615        url,
616    })
617}
618fn mk_packed_rock_name(name: &PackageName, version: &PackageVersion, ext: &str) -> String {
619    format!("{name}-{version}.{ext}")
620}
621
622pub(crate) async fn unpack_rockspec(
623    rock: &DownloadedPackedRockBytes,
624) -> Result<RemoteLuaRockspec, SearchAndDownloadError> {
625    let cursor = Cursor::new(&rock.bytes);
626    let rockspec_file_name = format!("{}-{}.rockspec", rock.name, rock.version);
627    let mut zip = zip::ZipArchive::new(cursor)
628        .map_err(|err| SearchAndDownloadError::ZipRead(rock.file_name.clone(), err))?;
629    let rockspec_index = (0..zip.len())
630        .find(|&i| {
631            unsafe { zip.by_index(i).unwrap_unchecked() }
632                .name()
633                .is_ok_and(|name| name.eq(&rockspec_file_name))
634        })
635        .ok_or(SearchAndDownloadError::RockspecNotFoundInPackedRock(
636            rockspec_file_name.to_string(),
637        ))?;
638    let mut rockspec_file = zip
639        .by_index(rockspec_index)
640        .map_err(|err| SearchAndDownloadError::ZipExtract(rock.file_name.clone(), err))?;
641    let mut content = String::new();
642    rockspec_file
643        .read_to_string(&mut content)
644        .map_err(|source| fs::FsError::ReadToString {
645            path: PathBuf::from(&rockspec_file_name),
646            source,
647        })?;
648    let rockspec = RemoteLuaRockspec::new(&content)
649        .map_err(|err| SearchAndDownloadError::Rockspec(Box::new(err)))?;
650    Ok(rockspec)
651}
lux_lib/operations/download.rs

lux_lib/operations/
download.rs
