lux_lib/operations/
fetch.rs

1use crate::build::utils::recursive_copy_dir;
2use crate::config::Config;
3use crate::git::url::RemoteGitUrlParseError;
4use crate::git::GitSource;
5use crate::hash::HasIntegrity;
6use crate::lockfile::RemotePackageSourceUrl;
7use crate::lua_rockspec::RockSourceSpec;
8use crate::operations;
9use crate::package::PackageSpec;
10use crate::rockspec::Rockspec;
11use auth_git2::{GitAuthenticator, Prompter};
12use bon::Builder;
13use git2::build::RepoBuilder;
14use git2::{FetchOptions, RemoteCallbacks};
15use miette::Diagnostic;
16use remove_dir_all::remove_dir_all;
17use ssri::Integrity;
18use std::fs::File;
19use std::io;
20use std::io::Cursor;
21use std::io::Read;
22use std::path::Path;
23use std::path::PathBuf;
24use thiserror::Error;
25use tracing::span;
26
27use super::DownloadSrcRockError;
28use super::UnpackError;
29
30/// A rocks package source fetcher, providing fine-grained control
31/// over how a package should be fetched.
32#[derive(Builder)]
33#[builder(start_fn = new, finish_fn(name = _build, vis = ""))]
34pub struct FetchSrc<'a, R: Rockspec> {
35    #[builder(start_fn)]
36    dest_dir: &'a Path,
37    #[builder(start_fn)]
38    rockspec: &'a R,
39    #[builder(start_fn)]
40    config: &'a Config,
41    #[builder(setters(vis = "pub(crate)"))]
42    source_url: Option<RemotePackageSourceUrl>,
43}
44
45#[derive(Debug)]
46pub(crate) struct RemotePackageSourceMetadata {
47    pub hash: Integrity,
48    pub source_url: RemotePackageSourceUrl,
49}
50
51impl<R: Rockspec, State> FetchSrcBuilder<'_, R, State>
52where
53    State: fetch_src_builder::State + fetch_src_builder::IsComplete,
54{
55    /// Fetch and unpack the source into the `dest_dir`.
56    pub async fn fetch(self) -> Result<(), FetchSrcError> {
57        self.fetch_internal().await?;
58        Ok(())
59    }
60
61    /// Fetch and unpack the source into the `dest_dir`,
62    /// returning the source `Integrity`.
63    pub(crate) async fn fetch_internal(self) -> Result<RemotePackageSourceMetadata, FetchSrcError> {
64        let fetch = self._build();
65        match do_fetch_src(&fetch).await {
66            Err(err)
67                if fetch
68                    .source_url
69                    .is_some_and(|url| matches!(url, RemotePackageSourceUrl::File { .. })) =>
70            {
71                // Don't fall back to downloading .src.rock archives if a local source was specified.
72                Err(err)
73            }
74            Err(err) => match &fetch.rockspec.source().current_platform().source_spec {
75                RockSourceSpec::Git(_) | RockSourceSpec::Url(_) => {
76                    let package = PackageSpec::new(
77                        fetch.rockspec.package().clone(),
78                        fetch.rockspec.version().clone(),
79                    );
80                    let metadata = FetchSrcRock::new(&package, fetch.dest_dir, fetch.config)
81                        .fetch()
82                        .await?;
83                    Ok(metadata)
84                }
85                RockSourceSpec::File(_) => Err(err),
86            },
87            Ok(metadata) => Ok(metadata),
88        }
89    }
90}
91
92#[derive(Error, Debug, Diagnostic)]
93pub enum FetchSrcError {
94    #[error("failed to clone rock source:\n{0}")]
95    #[diagnostic(help("check your network connection and verify the git URL is correct."))]
96    GitClone(#[from] git2::Error),
97    #[error("failed to parse git URL:\n{0}")]
98    #[diagnostic(forward(0))]
99    GitUrlParse(#[from] RemoteGitUrlParseError),
100    #[error(transparent)]
101    #[diagnostic(help("check your network connection."))]
102    Request(#[from] reqwest::Error),
103    #[error(transparent)]
104    #[diagnostic(transparent)]
105    Unpack(#[from] UnpackError),
106    #[error(transparent)]
107    #[diagnostic(transparent)]
108    FetchSrcRock(#[from] FetchSrcRockError),
109    #[error("unable to remove the '.git' directory:\n{0}")]
110    #[diagnostic(help(
111        "check that no process is using the directory and you have write permissions."
112    ))]
113    CleanGitDir(io::Error),
114    #[error("unable to compute hash:\n{0}")]
115    Hash(io::Error),
116    #[error("unable to copy {src} to {dest}:\n{err}")]
117    #[diagnostic(help("check that the source exists and the destination is writable."))]
118    CopyDir {
119        src: PathBuf,
120        dest: PathBuf,
121        err: io::Error,
122    },
123    #[error("unable to open {file}:\n{err}")]
124    #[diagnostic(help("check that the file exists and is readable."))]
125    FileOpen { file: PathBuf, err: io::Error },
126    #[error("unable to read {file}:\n{err}")]
127    #[diagnostic(help("check that the file exists and is readable."))]
128    FileRead { file: PathBuf, err: io::Error },
129}
130
131/// A rocks package source fetcher, providing fine-grained control
132/// over how a package should be fetched.
133#[derive(Builder)]
134#[builder(start_fn = new, finish_fn(name = _build, vis = ""))]
135struct FetchSrcRock<'a> {
136    #[builder(start_fn)]
137    package: &'a PackageSpec,
138    #[builder(start_fn)]
139    dest_dir: &'a Path,
140    #[builder(start_fn)]
141    config: &'a Config,
142}
143
144impl<State> FetchSrcRockBuilder<'_, State>
145where
146    State: fetch_src_rock_builder::State + fetch_src_rock_builder::IsComplete,
147{
148    pub async fn fetch(self) -> Result<RemotePackageSourceMetadata, FetchSrcRockError> {
149        do_fetch_src_rock(self._build()).await
150    }
151}
152
153#[derive(Error, Debug, Diagnostic)]
154#[error(transparent)]
155pub enum FetchSrcRockError {
156    DownloadSrcRock(#[from] DownloadSrcRockError),
157    Unpack(#[from] UnpackError),
158    Io(#[from] io::Error),
159}
160
161/// A no-prompt implementer for auth_git2's prompter
162#[derive(Copy, Clone, Debug)]
163struct NullPrompter;
164
165impl Prompter for NullPrompter {
166    fn prompt_username_password(&mut self, _: &str, _: &git2::Config) -> Option<(String, String)> {
167        None
168    }
169
170    fn prompt_password(&mut self, _: &str, _: &str, _: &git2::Config) -> Option<String> {
171        None
172    }
173
174    fn prompt_ssh_key_passphrase(&mut self, _: &Path, _: &git2::Config) -> Option<String> {
175        None
176    }
177}
178
179async fn do_fetch_src<R: Rockspec>(
180    fetch: &FetchSrc<'_, R>,
181) -> Result<RemotePackageSourceMetadata, FetchSrcError> {
182    let rockspec = fetch.rockspec;
183    let rock_source = rockspec.source().current_platform();
184    let dest_dir = fetch.dest_dir;
185    let config = fetch.config;
186    // prioritise lockfile source, if present
187    let mut source_spec = match &fetch.source_url {
188        Some(source_url) => match source_url {
189            RemotePackageSourceUrl::Git { url, checkout_ref } => RockSourceSpec::Git(GitSource {
190                url: url.parse()?,
191                checkout_ref: Some(checkout_ref.clone()),
192            }),
193            RemotePackageSourceUrl::Url { url } => RockSourceSpec::Url(url.clone()),
194            RemotePackageSourceUrl::File { path } => RockSourceSpec::File(path.clone()),
195        },
196        None => rock_source.source_spec.clone(),
197    };
198    let span = span!(
199        tracing::Level::INFO,
200        "Fetching source",
201        location = source_spec.to_string(),
202    );
203    let _enter = span.enter();
204
205    if let Some(vendor_dir) = config.vendor_dir() {
206        source_spec = match source_spec {
207            // could be a project directory (not vendored) or a local source
208            // or a vendored dependency that we have already resolved
209            RockSourceSpec::File(_) => source_spec,
210            _ => {
211                let pkg_vendor_dir =
212                    vendor_dir.join(format!("{}@{}", rockspec.package(), rockspec.version()));
213                RockSourceSpec::File(pkg_vendor_dir)
214            }
215        }
216    }
217    let metadata = match &source_spec {
218        RockSourceSpec::Git(git) => {
219            let url = git.url.to_string();
220            tracing::debug!(message = format!("Cloning {url}").as_str());
221
222            let auth = if config.no_prompt() {
223                GitAuthenticator::default()
224                    .try_password_prompt(0)
225                    .prompt_ssh_key_password(false)
226                    .set_prompter(NullPrompter)
227            } else {
228                GitAuthenticator::default()
229            };
230            let git_config = git2::Config::open_default()?;
231            let mut callbacks = RemoteCallbacks::new();
232            callbacks.credentials(auth.credentials(&git_config));
233            let mut fetch_options = FetchOptions::new();
234            fetch_options.update_fetchhead(false);
235            fetch_options.remote_callbacks(callbacks);
236            if git.checkout_ref.is_none() {
237                fetch_options.depth(1);
238            };
239            let mut repo_builder = RepoBuilder::new();
240            repo_builder.fetch_options(fetch_options);
241            let repo = repo_builder.clone(&url, dest_dir)?;
242
243            let checkout_ref = match &git.checkout_ref {
244                Some(checkout_ref) => {
245                    let (object, _) = repo.revparse_ext(checkout_ref)?;
246                    repo.checkout_tree(&object, None)?;
247                    checkout_ref.clone()
248                }
249                None => {
250                    let head = repo.head()?;
251                    let commit = head.peel_to_commit()?;
252                    commit.id().to_string()
253                }
254            };
255            // The .git directory is not deterministic
256            remove_dir_all(dest_dir.join(".git")).map_err(FetchSrcError::CleanGitDir)?;
257            let hash = fetch.dest_dir.hash().map_err(FetchSrcError::Hash)?;
258            RemotePackageSourceMetadata {
259                hash,
260                source_url: RemotePackageSourceUrl::Git { url, checkout_ref },
261            }
262        }
263        RockSourceSpec::Url(url) => {
264            tracing::debug!(message = format!("📥 Downloading {url}").as_str());
265
266            // NOTE: We don't enforce HTTPS when fetching sources because some rockspecs
267            // have HTTP URLs in `source.url`.
268            let response = crate::reqwest::new_http_client(config)?
269                .get(url.clone())
270                .send()
271                .await?
272                .error_for_status()?
273                .bytes()
274                .await?;
275            let hash = response.hash().map_err(FetchSrcError::Hash)?;
276            let file_name = url
277                .path_segments()
278                .and_then(|mut segments| segments.next_back())
279                .and_then(|name| {
280                    if name.is_empty() {
281                        None
282                    } else {
283                        Some(name.to_string())
284                    }
285                })
286                .unwrap_or(url.to_string());
287            let cursor = Cursor::new(response);
288            let mime_type = infer::get(cursor.get_ref()).map(|file_type| file_type.mime_type());
289            operations::unpack::unpack(
290                mime_type,
291                cursor,
292                rock_source.unpack_dir.is_none(),
293                file_name,
294                dest_dir,
295            )
296            .await?;
297            RemotePackageSourceMetadata {
298                hash,
299                source_url: RemotePackageSourceUrl::Url { url: url.clone() },
300            }
301        }
302        RockSourceSpec::File(path) => {
303            tracing::debug!(message = format!("📋 Copying {}", path.display()).as_str());
304
305            let hash = if path.is_dir() {
306                recursive_copy_dir(&path.to_path_buf(), dest_dir)
307                    .await
308                    .map_err(|err| FetchSrcError::CopyDir {
309                        src: path.to_path_buf(),
310                        dest: dest_dir.to_path_buf(),
311                        err,
312                    })?;
313                dest_dir.hash().map_err(FetchSrcError::Hash)?
314            } else {
315                let mut file = File::open(path).map_err(|err| FetchSrcError::FileOpen {
316                    file: path.clone(),
317                    err,
318                })?;
319                let mut buffer = Vec::new();
320                file.read_to_end(&mut buffer)
321                    .map_err(|err| FetchSrcError::FileRead {
322                        file: path.clone(),
323                        err,
324                    })?;
325                let mime_type = infer::get(&buffer).map(|file_type| file_type.mime_type());
326                let file_name = path
327                    .file_name()
328                    .map(|os_str| os_str.to_string_lossy())
329                    .unwrap_or(path.to_string_lossy())
330                    .to_string();
331                operations::unpack::unpack(
332                    mime_type,
333                    file,
334                    rock_source.unpack_dir.is_none(),
335                    file_name,
336                    dest_dir,
337                )
338                .await?;
339                path.hash().map_err(FetchSrcError::Hash)?
340            };
341            RemotePackageSourceMetadata {
342                hash,
343                source_url: RemotePackageSourceUrl::File { path: path.clone() },
344            }
345        }
346    };
347    Ok(metadata)
348}
349
350async fn do_fetch_src_rock(
351    fetch: FetchSrcRock<'_>,
352) -> Result<RemotePackageSourceMetadata, FetchSrcRockError> {
353    let package = fetch.package;
354    let span = span!(
355        tracing::Level::INFO,
356        "Fetching src.rock",
357        package = package.to_string(),
358    );
359    let _enter = span.enter();
360
361    let dest_dir = fetch.dest_dir;
362    let config = fetch.config;
363    let src_rock = operations::download_src_rock(package, config.server(), fetch.config).await?;
364    let hash = src_rock.bytes.hash()?;
365    let cursor = Cursor::new(src_rock.bytes);
366    let mime_type = infer::get(cursor.get_ref()).map(|file_type| file_type.mime_type());
367    operations::unpack::unpack(mime_type, cursor, true, src_rock.file_name, dest_dir).await?;
368    Ok(RemotePackageSourceMetadata {
369        hash,
370        source_url: RemotePackageSourceUrl::Url { url: src_rock.url },
371    })
372}
lux_lib/operations/fetch.rs

lux_lib/operations/
fetch.rs
