lux_lib/operations/
fetch.rs

1use crate::config::Config;
2use crate::git::url::RemoteGitUrlParseError;
3use crate::git::{GitRef, GitSource};
4use crate::hash::HasIntegrity;
5use crate::lockfile::RemotePackageSourceUrl;
6use crate::lua_rockspec::{RemoteRockSource, RockSourceSpec};
7use crate::package::PackageSpec;
8use crate::reqwest::{RequestBuilderExt, RequestError};
9use crate::rockspec::Rockspec;
10use crate::{fs, operations};
11use auth_git2::{GitAuthenticator, Prompter};
12use bon::Builder;
13use git2::build::RepoBuilder;
14use git2::{FetchOptions, RemoteCallbacks};
15use miette::Diagnostic;
16use remove_dir_all::remove_dir_all;
17use ssri::Integrity;
18use std::io;
19use std::io::Cursor;
20use std::io::Read;
21use std::path::{Path, PathBuf};
22use thiserror::Error;
23
24use super::DownloadSrcRockError;
25use super::UnpackError;
26
27/// A rocks package source fetcher, providing fine-grained control
28/// over how a package should be fetched.
29#[derive(Builder)]
30#[builder(start_fn = new, finish_fn(name = _build, vis = ""))]
31pub struct FetchSrc<'a, R: Rockspec> {
32    #[builder(start_fn)]
33    dest_dir: &'a Path,
34    #[builder(start_fn)]
35    rockspec: &'a R,
36    #[builder(start_fn)]
37    config: &'a Config,
38    #[builder(setters(vis = "pub(crate)"))]
39    source_url: Option<RemotePackageSourceUrl>,
40}
41
42#[derive(Debug)]
43pub(crate) struct RemotePackageSourceMetadata {
44    pub hash: Integrity,
45    pub source_url: RemotePackageSourceUrl,
46}
47
48impl<R: Rockspec, State> FetchSrcBuilder<'_, R, State>
49where
50    State: fetch_src_builder::State + fetch_src_builder::IsComplete,
51{
52    /// Fetch and unpack the source into the `dest_dir`.
53    pub async fn fetch(self) -> Result<(), FetchSrcError> {
54        self.fetch_internal().await?;
55        Ok(())
56    }
57
58    /// Fetch and unpack the source into the `dest_dir`,
59    /// returning the source `Integrity`.
60    pub(crate) async fn fetch_internal(self) -> Result<RemotePackageSourceMetadata, FetchSrcError> {
61        let fetch = self._build();
62        let rockspec = fetch.rockspec;
63        let rock_source = rockspec.source().current_platform();
64        let dest_dir = fetch.dest_dir;
65        let config = fetch.config;
66        // prioritise lockfile source, if present
67        let source_spec = match &fetch.source_url {
68            Some(source_url) => match source_url {
69                RemotePackageSourceUrl::Git {
70                    url, checkout_ref, ..
71                } => RockSourceSpec::Git(GitSource {
72                    url: url.parse()?,
73                    git_ref: Some(GitRef::Tag(checkout_ref.clone())),
74                }),
75                RemotePackageSourceUrl::Url { url } => RockSourceSpec::Url(url.clone()),
76                RemotePackageSourceUrl::File { path } => RockSourceSpec::File(path.clone()),
77            },
78            None => rock_source.source_spec.clone(),
79        };
80        match fetch_src_impl(source_spec, rockspec, rock_source, dest_dir, config).await {
81            Err(err)
82                if fetch
83                    .source_url
84                    .is_some_and(|url| matches!(url, RemotePackageSourceUrl::File { .. })) =>
85            {
86                // Don't fall back to downloading .src.rock archives if a local source was specified.
87                Err(err)
88            }
89            Err(err) => match &fetch.rockspec.source().current_platform().source_spec {
90                RockSourceSpec::Git(_) | RockSourceSpec::Url(_) => {
91                    let package = PackageSpec::new(
92                        fetch.rockspec.package().clone(),
93                        fetch.rockspec.version().clone(),
94                    );
95                    match FetchSrcRock::new(&package, fetch.dest_dir, fetch.config)
96                        .fetch()
97                        .await
98                    {
99                        Ok(metadata) => Ok(metadata),
100                        Err(fallback_err) => {
101                            tracing::error!("fallback .src.rock download failed: {fallback_err:?}");
102                            Err(err)
103                        }
104                    }
105                }
106                RockSourceSpec::File(_) => Err(err),
107            },
108            Ok(metadata) => Ok(metadata),
109        }
110    }
111}
112
113#[derive(Error, Debug, Diagnostic)]
114#[non_exhaustive]
115pub enum FetchSrcError {
116    #[error("failed to clone rock source")]
117    #[diagnostic(help("check your network connection and verify the git URL is correct."))]
118    GitClone(#[from] git2::Error),
119    #[error("failed to parse git URL")]
120    #[diagnostic(forward(0))]
121    GitUrlParse(#[from] RemoteGitUrlParseError),
122    #[error(transparent)]
123    #[diagnostic(transparent)]
124    Request(#[from] RequestError),
125    #[error(transparent)]
126    #[diagnostic(transparent)]
127    Unpack(#[from] UnpackError),
128    #[error(transparent)]
129    #[diagnostic(transparent)]
130    FetchSrcRock(#[from] FetchSrcRockError),
131    #[error("unable to remove the '.git' directory")]
132    #[diagnostic(help(
133        "check that no process is using the directory and you have write permissions."
134    ))]
135    CleanGitDir(#[source] io::Error),
136    #[error("unable to compute hash")]
137    Hash(#[source] io::Error),
138    #[error(transparent)]
139    #[diagnostic(transparent)]
140    Fs(#[from] fs::FsError),
141}
142
143impl From<reqwest::Error> for FetchSrcError {
144    fn from(err: reqwest::Error) -> Self {
145        Self::Request(err.into())
146    }
147}
148
149/// A rocks package source fetcher, providing fine-grained control
150/// over how a package should be fetched.
151#[derive(Builder)]
152#[builder(start_fn = new, finish_fn(name = _build, vis = ""))]
153struct FetchSrcRock<'a> {
154    #[builder(start_fn)]
155    package: &'a PackageSpec,
156    #[builder(start_fn)]
157    dest_dir: &'a Path,
158    #[builder(start_fn)]
159    config: &'a Config,
160}
161
162impl<State> FetchSrcRockBuilder<'_, State>
163where
164    State: fetch_src_rock_builder::State + fetch_src_rock_builder::IsComplete,
165{
166    pub async fn fetch(self) -> Result<RemotePackageSourceMetadata, FetchSrcRockError> {
167        do_fetch_src_rock(self._build()).await
168    }
169}
170
171#[derive(Error, Debug, Diagnostic)]
172#[non_exhaustive]
173#[error(transparent)]
174pub enum FetchSrcRockError {
175    DownloadSrcRock(#[from] DownloadSrcRockError),
176    Unpack(#[from] UnpackError),
177    Io(#[from] io::Error),
178}
179
180/// A no-prompt implementer for auth_git2's prompter
181#[derive(Copy, Clone, Debug)]
182struct NullPrompter;
183
184impl Prompter for NullPrompter {
185    fn prompt_username_password(&mut self, _: &str, _: &git2::Config) -> Option<(String, String)> {
186        None
187    }
188
189    fn prompt_password(&mut self, _: &str, _: &str, _: &git2::Config) -> Option<String> {
190        None
191    }
192
193    fn prompt_ssh_key_passphrase(&mut self, _: &Path, _: &git2::Config) -> Option<String> {
194        None
195    }
196}
197
198/// Initialises and checks out git submodules,
199/// returning the relative paths of any submodules that were checked out.
200fn init_submodules(
201    repo: &git2::Repository,
202    auth: &GitAuthenticator,
203    git_config: &git2::Config,
204) -> Result<Vec<PathBuf>, FetchSrcError> {
205    let workdir = repo
206        .workdir()
207        .ok_or_else(|| git2::Error::from_str("cloned repository has no working directory"))?;
208    let mut submodule_paths = Vec::new();
209    for mut submodule in repo.submodules()? {
210        let mut callbacks = RemoteCallbacks::new();
211        callbacks.credentials(auth.credentials(git_config));
212        let mut fetch_options = FetchOptions::new();
213        fetch_options.remote_callbacks(callbacks);
214        let mut options = git2::SubmoduleUpdateOptions::new();
215        options.fetch(fetch_options);
216        submodule.update(true, Some(&mut options))?;
217        // The .git file/directory is not deterministic
218        let git_file = workdir.join(submodule.path()).join(".git");
219        if git_file.is_dir() {
220            fs::sync::remove_dir_all(&git_file)?;
221        } else {
222            fs::sync::remove_file(&git_file)?;
223        }
224        submodule_paths.push(submodule.path().to_path_buf());
225    }
226    Ok(submodule_paths)
227}
228
229#[tracing::instrument(
230    name = "Fetching source",
231    level = "info",
232    skip_all,
233    fields(location = source_spec.to_string()),
234)]
235async fn fetch_src_impl<R: Rockspec>(
236    mut source_spec: RockSourceSpec,
237    rockspec: &R,
238    rock_source: &RemoteRockSource,
239    dest_dir: &Path,
240    config: &Config,
241) -> Result<RemotePackageSourceMetadata, FetchSrcError> {
242    if let Some(vendor_dir) = config.vendor_dir() {
243        source_spec = match source_spec {
244            // could be a project directory (not vendored) or a local source
245            // or a vendored dependency that we have already resolved
246            RockSourceSpec::File(_) => source_spec,
247            _ => {
248                let pkg_vendor_dir =
249                    vendor_dir.join(format!("{}@{}", rockspec.package(), rockspec.version()));
250                RockSourceSpec::File(pkg_vendor_dir)
251            }
252        }
253    }
254    let metadata = match &source_spec {
255        RockSourceSpec::Git(git) => {
256            let url = git.url.to_string();
257            tracing::debug!(message = format!("Cloning {url}").as_str());
258
259            let (checkout_ref, has_submodules) = {
260                let mut auth = if config.no_prompt() {
261                    GitAuthenticator::default()
262                        .try_password_prompt(0)
263                        .prompt_ssh_key_password(false)
264                        .set_prompter(NullPrompter)
265                } else {
266                    GitAuthenticator::default()
267                };
268                if let Some(access_token) = config.access_token(git.url.host()) {
269                    auth = auth.add_plaintext_credentials(
270                        git.url.host(),
271                        access_token.username(),
272                        unsafe { access_token.password() },
273                    );
274                }
275                let git_config = git2::Config::open_default()?;
276                let mut callbacks = RemoteCallbacks::new();
277                callbacks.credentials(auth.credentials(&git_config));
278                let mut fetch_options = FetchOptions::new();
279                fetch_options.update_fetchhead(false);
280                fetch_options.remote_callbacks(callbacks);
281                let checkout_ref = match &git.git_ref {
282                    Some(GitRef::Tag(tag)) => Some(tag.as_str()),
283                    Some(GitRef::Branch(branch)) => Some(branch.as_str()),
284                    None => None,
285                };
286                if checkout_ref.is_none() {
287                    fetch_options.depth(1);
288                };
289                let mut repo_builder = RepoBuilder::new();
290                repo_builder.fetch_options(fetch_options);
291                let repo = repo_builder.clone(&url, dest_dir)?;
292
293                let checkout_ref = match checkout_ref {
294                    Some(checkout_ref) => {
295                        let ref_name = if let Some(GitRef::Branch(branch)) = &git.git_ref {
296                            format!("origin/{branch}")
297                        } else {
298                            checkout_ref.to_string()
299                        };
300                        let (object, _) = repo.revparse_ext(&ref_name)?;
301                        repo.checkout_tree(&object, None)?;
302                        checkout_ref.to_string()
303                    }
304                    None => {
305                        let head = repo.head()?;
306                        let commit = head.peel_to_commit()?;
307                        commit.id().to_string()
308                    }
309                };
310
311                let submodule_paths = init_submodules(&repo, &auth, &git_config)?;
312
313                (checkout_ref, !submodule_paths.is_empty())
314            };
315            // The .git directory is not deterministic
316            remove_dir_all(dest_dir.join(".git")).map_err(FetchSrcError::CleanGitDir)?;
317            let hash = dest_dir.hash().await.map_err(FetchSrcError::Hash)?;
318            RemotePackageSourceMetadata {
319                hash,
320                source_url: RemotePackageSourceUrl::Git {
321                    url,
322                    checkout_ref,
323                    submodules: has_submodules,
324                },
325            }
326        }
327        RockSourceSpec::Url(url) => {
328            tracing::debug!(message = format!("📥 Downloading {url}").as_str());
329
330            // NOTE: We don't enforce HTTPS when fetching sources because some rockspecs
331            // have HTTP URLs in `source.url`.
332            let response = crate::reqwest::http_client(config)?
333                .get(url.clone())
334                .apply_access_token(config, url)
335                .send()
336                .await?
337                .error_for_status()?
338                .bytes()
339                .await?;
340            let hash = response.hash().await.map_err(FetchSrcError::Hash)?;
341            let file_name = url
342                .path_segments()
343                .and_then(|mut segments| segments.next_back())
344                .and_then(|name| {
345                    if name.is_empty() {
346                        None
347                    } else {
348                        Some(name.to_string())
349                    }
350                })
351                .unwrap_or(url.to_string());
352            let cursor = Cursor::new(response);
353            let mime_type = infer::get(cursor.get_ref()).map(|file_type| file_type.mime_type());
354            operations::unpack::unpack(
355                mime_type,
356                cursor,
357                rock_source.unpack_dir.is_none(),
358                file_name,
359                dest_dir,
360            )
361            .await?;
362            RemotePackageSourceMetadata {
363                hash,
364                source_url: RemotePackageSourceUrl::Url { url: url.clone() },
365            }
366        }
367        RockSourceSpec::File(path) => {
368            tracing::debug!(message = format!("📋 Copying {}", path.display()).as_str());
369
370            let hash = if path.is_dir() {
371                fs::tokio::copy_dir_contents(path, dest_dir).await?;
372                dest_dir.hash().await.map_err(FetchSrcError::Hash)?
373            } else {
374                let mut file = fs::sync::open(path)?;
375                let mut buffer = Vec::new();
376                file.read_to_end(&mut buffer)
377                    .map_err(|source| fs::FsError::Read {
378                        path: path.to_path_buf(),
379                        source,
380                    })?;
381                let mime_type = infer::get(&buffer).map(|file_type| file_type.mime_type());
382                let file_name = path
383                    .file_name()
384                    .map(|os_str| os_str.to_string_lossy())
385                    .unwrap_or(path.to_string_lossy())
386                    .to_string();
387                operations::unpack::unpack(
388                    mime_type,
389                    file,
390                    rock_source.unpack_dir.is_none(),
391                    file_name,
392                    dest_dir,
393                )
394                .await?;
395                path.hash().await.map_err(FetchSrcError::Hash)?
396            };
397            RemotePackageSourceMetadata {
398                hash,
399                source_url: RemotePackageSourceUrl::File { path: path.clone() },
400            }
401        }
402    };
403    Ok(metadata)
404}
405
406#[tracing::instrument(
407    name = "Fetching src.rock",
408    level = "info",
409    skip_all,
410    fields(package = fetch.package.to_string(),),
411)]
412async fn do_fetch_src_rock(
413    fetch: FetchSrcRock<'_>,
414) -> Result<RemotePackageSourceMetadata, FetchSrcRockError> {
415    let package = fetch.package;
416    let dest_dir = fetch.dest_dir;
417    let config = fetch.config;
418    let src_rock = operations::download_src_rock(package, config.server(), fetch.config).await?;
419    let hash = src_rock.bytes.hash().await?;
420    let cursor = Cursor::new(src_rock.bytes);
421    let mime_type = infer::get(cursor.get_ref()).map(|file_type| file_type.mime_type());
422    operations::unpack::unpack(mime_type, cursor, true, src_rock.file_name, dest_dir).await?;
423    Ok(RemotePackageSourceMetadata {
424        hash,
425        source_url: RemotePackageSourceUrl::Url { url: src_rock.url },
426    })
427}
lux_lib/operations/fetch.rs

lux_lib/operations/
fetch.rs
