lux_lib/operations/
fetch.rs

1use crate::build::utils::recursive_copy_dir;
2use crate::config::Config;
3use crate::git::url::RemoteGitUrlParseError;
4use crate::git::GitSource;
5use crate::hash::HasIntegrity;
6use crate::lockfile::RemotePackageSourceUrl;
7use crate::lua_rockspec::RockSourceSpec;
8use crate::operations;
9use crate::package::PackageSpec;
10use crate::rockspec::Rockspec;
11use auth_git2::{GitAuthenticator, Prompter};
12use bon::Builder;
13use git2::build::RepoBuilder;
14use git2::{FetchOptions, RemoteCallbacks};
15use miette::Diagnostic;
16use remove_dir_all::remove_dir_all;
17use ssri::Integrity;
18use std::fs::File;
19use std::io;
20use std::io::Cursor;
21use std::io::Read;
22use std::path::Path;
23use std::path::PathBuf;
24use thiserror::Error;
25use tracing::span;
26
27use super::DownloadSrcRockError;
28use super::UnpackError;
29
30/// A rocks package source fetcher, providing fine-grained control
31/// over how a package should be fetched.
32#[derive(Builder)]
33#[builder(start_fn = new, finish_fn(name = _build, vis = ""))]
34pub struct FetchSrc<'a, R: Rockspec> {
35    #[builder(start_fn)]
36    dest_dir: &'a Path,
37    #[builder(start_fn)]
38    rockspec: &'a R,
39    #[builder(start_fn)]
40    config: &'a Config,
41    #[builder(setters(vis = "pub(crate)"))]
42    source_url: Option<RemotePackageSourceUrl>,
43}
44
45#[derive(Debug)]
46pub(crate) struct RemotePackageSourceMetadata {
47    pub hash: Integrity,
48    pub source_url: RemotePackageSourceUrl,
49}
50
51impl<R: Rockspec, State> FetchSrcBuilder<'_, R, State>
52where
53    State: fetch_src_builder::State + fetch_src_builder::IsComplete,
54{
55    /// Fetch and unpack the source into the `dest_dir`.
56    pub async fn fetch(self) -> Result<(), FetchSrcError> {
57        self.fetch_internal().await?;
58        Ok(())
59    }
60
61    /// Fetch and unpack the source into the `dest_dir`,
62    /// returning the source `Integrity`.
63    pub(crate) async fn fetch_internal(self) -> Result<RemotePackageSourceMetadata, FetchSrcError> {
64        let fetch = self._build();
65        match do_fetch_src(&fetch).await {
66            Err(err)
67                if fetch
68                    .source_url
69                    .is_some_and(|url| matches!(url, RemotePackageSourceUrl::File { .. })) =>
70            {
71                // Don't fall back to downloading .src.rock archives if a local source was specified.
72                Err(err)
73            }
74            Err(err) => match &fetch.rockspec.source().current_platform().source_spec {
75                RockSourceSpec::Git(_) | RockSourceSpec::Url(_) => {
76                    let package = PackageSpec::new(
77                        fetch.rockspec.package().clone(),
78                        fetch.rockspec.version().clone(),
79                    );
80                    let metadata = FetchSrcRock::new(&package, fetch.dest_dir, fetch.config)
81                        .fetch()
82                        .await?;
83                    Ok(metadata)
84                }
85                RockSourceSpec::File(_) => Err(err),
86            },
87            Ok(metadata) => Ok(metadata),
88        }
89    }
90}
91
92#[derive(Error, Debug, Diagnostic)]
93#[non_exhaustive]
94pub enum FetchSrcError {
95    #[error("failed to clone rock source:\n{0}")]
96    #[diagnostic(help("check your network connection and verify the git URL is correct."))]
97    GitClone(#[from] git2::Error),
98    #[error("failed to parse git URL:\n{0}")]
99    #[diagnostic(forward(0))]
100    GitUrlParse(#[from] RemoteGitUrlParseError),
101    #[error(transparent)]
102    #[diagnostic(help("check your network connection."))]
103    Request(#[from] reqwest::Error),
104    #[error(transparent)]
105    #[diagnostic(transparent)]
106    Unpack(#[from] UnpackError),
107    #[error(transparent)]
108    #[diagnostic(transparent)]
109    FetchSrcRock(#[from] FetchSrcRockError),
110    #[error("unable to remove the '.git' directory:\n{0}")]
111    #[diagnostic(help(
112        "check that no process is using the directory and you have write permissions."
113    ))]
114    CleanGitDir(io::Error),
115    #[error("unable to compute hash:\n{0}")]
116    Hash(io::Error),
117    #[error("unable to copy {src} to {dest}:\n{err}")]
118    #[diagnostic(help("check that the source exists and the destination is writable."))]
119    CopyDir {
120        src: PathBuf,
121        dest: PathBuf,
122        err: io::Error,
123    },
124    #[error("unable to open {file}:\n{err}")]
125    #[diagnostic(help("check that the file exists and is readable."))]
126    FileOpen { file: PathBuf, err: io::Error },
127    #[error("unable to read {file}:\n{err}")]
128    #[diagnostic(help("check that the file exists and is readable."))]
129    FileRead { file: PathBuf, err: io::Error },
130}
131
132/// A rocks package source fetcher, providing fine-grained control
133/// over how a package should be fetched.
134#[derive(Builder)]
135#[builder(start_fn = new, finish_fn(name = _build, vis = ""))]
136struct FetchSrcRock<'a> {
137    #[builder(start_fn)]
138    package: &'a PackageSpec,
139    #[builder(start_fn)]
140    dest_dir: &'a Path,
141    #[builder(start_fn)]
142    config: &'a Config,
143}
144
145impl<State> FetchSrcRockBuilder<'_, State>
146where
147    State: fetch_src_rock_builder::State + fetch_src_rock_builder::IsComplete,
148{
149    pub async fn fetch(self) -> Result<RemotePackageSourceMetadata, FetchSrcRockError> {
150        do_fetch_src_rock(self._build()).await
151    }
152}
153
154#[derive(Error, Debug, Diagnostic)]
155#[non_exhaustive]
156#[error(transparent)]
157pub enum FetchSrcRockError {
158    DownloadSrcRock(#[from] DownloadSrcRockError),
159    Unpack(#[from] UnpackError),
160    Io(#[from] io::Error),
161}
162
163/// A no-prompt implementer for auth_git2's prompter
164#[derive(Copy, Clone, Debug)]
165struct NullPrompter;
166
167impl Prompter for NullPrompter {
168    fn prompt_username_password(&mut self, _: &str, _: &git2::Config) -> Option<(String, String)> {
169        None
170    }
171
172    fn prompt_password(&mut self, _: &str, _: &str, _: &git2::Config) -> Option<String> {
173        None
174    }
175
176    fn prompt_ssh_key_passphrase(&mut self, _: &Path, _: &git2::Config) -> Option<String> {
177        None
178    }
179}
180
181async fn do_fetch_src<R: Rockspec>(
182    fetch: &FetchSrc<'_, R>,
183) -> Result<RemotePackageSourceMetadata, FetchSrcError> {
184    let rockspec = fetch.rockspec;
185    let rock_source = rockspec.source().current_platform();
186    let dest_dir = fetch.dest_dir;
187    let config = fetch.config;
188    // prioritise lockfile source, if present
189    let mut source_spec = match &fetch.source_url {
190        Some(source_url) => match source_url {
191            RemotePackageSourceUrl::Git { url, checkout_ref } => RockSourceSpec::Git(GitSource {
192                url: url.parse()?,
193                checkout_ref: Some(checkout_ref.clone()),
194            }),
195            RemotePackageSourceUrl::Url { url } => RockSourceSpec::Url(url.clone()),
196            RemotePackageSourceUrl::File { path } => RockSourceSpec::File(path.clone()),
197        },
198        None => rock_source.source_spec.clone(),
199    };
200    let span = span!(
201        tracing::Level::INFO,
202        "Fetching source",
203        location = source_spec.to_string(),
204    );
205    let _enter = span.enter();
206
207    if let Some(vendor_dir) = config.vendor_dir() {
208        source_spec = match source_spec {
209            // could be a project directory (not vendored) or a local source
210            // or a vendored dependency that we have already resolved
211            RockSourceSpec::File(_) => source_spec,
212            _ => {
213                let pkg_vendor_dir =
214                    vendor_dir.join(format!("{}@{}", rockspec.package(), rockspec.version()));
215                RockSourceSpec::File(pkg_vendor_dir)
216            }
217        }
218    }
219    let metadata = match &source_spec {
220        RockSourceSpec::Git(git) => {
221            let url = git.url.to_string();
222            tracing::debug!(message = format!("Cloning {url}").as_str());
223
224            let auth = if config.no_prompt() {
225                GitAuthenticator::default()
226                    .try_password_prompt(0)
227                    .prompt_ssh_key_password(false)
228                    .set_prompter(NullPrompter)
229            } else {
230                GitAuthenticator::default()
231            };
232            let git_config = git2::Config::open_default()?;
233            let mut callbacks = RemoteCallbacks::new();
234            callbacks.credentials(auth.credentials(&git_config));
235            let mut fetch_options = FetchOptions::new();
236            fetch_options.update_fetchhead(false);
237            fetch_options.remote_callbacks(callbacks);
238            if git.checkout_ref.is_none() {
239                fetch_options.depth(1);
240            };
241            let mut repo_builder = RepoBuilder::new();
242            repo_builder.fetch_options(fetch_options);
243            let repo = repo_builder.clone(&url, dest_dir)?;
244
245            let checkout_ref = match &git.checkout_ref {
246                Some(checkout_ref) => {
247                    let (object, _) = repo.revparse_ext(checkout_ref)?;
248                    repo.checkout_tree(&object, None)?;
249                    checkout_ref.clone()
250                }
251                None => {
252                    let head = repo.head()?;
253                    let commit = head.peel_to_commit()?;
254                    commit.id().to_string()
255                }
256            };
257            // The .git directory is not deterministic
258            remove_dir_all(dest_dir.join(".git")).map_err(FetchSrcError::CleanGitDir)?;
259            let hash = fetch.dest_dir.hash().map_err(FetchSrcError::Hash)?;
260            RemotePackageSourceMetadata {
261                hash,
262                source_url: RemotePackageSourceUrl::Git { url, checkout_ref },
263            }
264        }
265        RockSourceSpec::Url(url) => {
266            tracing::debug!(message = format!("📥 Downloading {url}").as_str());
267
268            // NOTE: We don't enforce HTTPS when fetching sources because some rockspecs
269            // have HTTP URLs in `source.url`.
270            let response = crate::reqwest::new_http_client(config)?
271                .get(url.clone())
272                .send()
273                .await?
274                .error_for_status()?
275                .bytes()
276                .await?;
277            let hash = response.hash().map_err(FetchSrcError::Hash)?;
278            let file_name = url
279                .path_segments()
280                .and_then(|mut segments| segments.next_back())
281                .and_then(|name| {
282                    if name.is_empty() {
283                        None
284                    } else {
285                        Some(name.to_string())
286                    }
287                })
288                .unwrap_or(url.to_string());
289            let cursor = Cursor::new(response);
290            let mime_type = infer::get(cursor.get_ref()).map(|file_type| file_type.mime_type());
291            operations::unpack::unpack(
292                mime_type,
293                cursor,
294                rock_source.unpack_dir.is_none(),
295                file_name,
296                dest_dir,
297            )
298            .await?;
299            RemotePackageSourceMetadata {
300                hash,
301                source_url: RemotePackageSourceUrl::Url { url: url.clone() },
302            }
303        }
304        RockSourceSpec::File(path) => {
305            tracing::debug!(message = format!("📋 Copying {}", path.display()).as_str());
306
307            let hash = if path.is_dir() {
308                recursive_copy_dir(&path.to_path_buf(), dest_dir)
309                    .await
310                    .map_err(|err| FetchSrcError::CopyDir {
311                        src: path.to_path_buf(),
312                        dest: dest_dir.to_path_buf(),
313                        err,
314                    })?;
315                dest_dir.hash().map_err(FetchSrcError::Hash)?
316            } else {
317                let mut file = File::open(path).map_err(|err| FetchSrcError::FileOpen {
318                    file: path.clone(),
319                    err,
320                })?;
321                let mut buffer = Vec::new();
322                file.read_to_end(&mut buffer)
323                    .map_err(|err| FetchSrcError::FileRead {
324                        file: path.clone(),
325                        err,
326                    })?;
327                let mime_type = infer::get(&buffer).map(|file_type| file_type.mime_type());
328                let file_name = path
329                    .file_name()
330                    .map(|os_str| os_str.to_string_lossy())
331                    .unwrap_or(path.to_string_lossy())
332                    .to_string();
333                operations::unpack::unpack(
334                    mime_type,
335                    file,
336                    rock_source.unpack_dir.is_none(),
337                    file_name,
338                    dest_dir,
339                )
340                .await?;
341                path.hash().map_err(FetchSrcError::Hash)?
342            };
343            RemotePackageSourceMetadata {
344                hash,
345                source_url: RemotePackageSourceUrl::File { path: path.clone() },
346            }
347        }
348    };
349    Ok(metadata)
350}
351
352async fn do_fetch_src_rock(
353    fetch: FetchSrcRock<'_>,
354) -> Result<RemotePackageSourceMetadata, FetchSrcRockError> {
355    let package = fetch.package;
356    let span = span!(
357        tracing::Level::INFO,
358        "Fetching src.rock",
359        package = package.to_string(),
360    );
361    let _enter = span.enter();
362
363    let dest_dir = fetch.dest_dir;
364    let config = fetch.config;
365    let src_rock = operations::download_src_rock(package, config.server(), fetch.config).await?;
366    let hash = src_rock.bytes.hash()?;
367    let cursor = Cursor::new(src_rock.bytes);
368    let mime_type = infer::get(cursor.get_ref()).map(|file_type| file_type.mime_type());
369    operations::unpack::unpack(mime_type, cursor, true, src_rock.file_name, dest_dir).await?;
370    Ok(RemotePackageSourceMetadata {
371        hash,
372        source_url: RemotePackageSourceUrl::Url { url: src_rock.url },
373    })
374}
lux_lib/operations/fetch.rs

lux_lib/operations/
fetch.rs
