lux_lib/operations/
fetch.rs

1use crate::build::utils::recursive_copy_dir;
2use crate::config::Config;
3use crate::git::url::RemoteGitUrlParseError;
4use crate::git::{GitRef, GitSource};
5use crate::hash::HasIntegrity;
6use crate::lockfile::RemotePackageSourceUrl;
7use crate::lua_rockspec::{RemoteRockSource, RockSourceSpec};
8use crate::package::PackageSpec;
9use crate::reqwest::{RequestBuilderExt, RequestError};
10use crate::rockspec::Rockspec;
11use crate::{fs, operations};
12use auth_git2::{GitAuthenticator, Prompter};
13use bon::Builder;
14use git2::build::RepoBuilder;
15use git2::{FetchOptions, RemoteCallbacks};
16use miette::Diagnostic;
17use remove_dir_all::remove_dir_all;
18use ssri::Integrity;
19use std::io;
20use std::io::Cursor;
21use std::io::Read;
22use std::path::{Path, PathBuf};
23use thiserror::Error;
24
25use super::DownloadSrcRockError;
26use super::UnpackError;
27
28/// A rocks package source fetcher, providing fine-grained control
29/// over how a package should be fetched.
30#[derive(Builder)]
31#[builder(start_fn = new, finish_fn(name = _build, vis = ""))]
32pub struct FetchSrc<'a, R: Rockspec> {
33    #[builder(start_fn)]
34    dest_dir: &'a Path,
35    #[builder(start_fn)]
36    rockspec: &'a R,
37    #[builder(start_fn)]
38    config: &'a Config,
39    #[builder(setters(vis = "pub(crate)"))]
40    source_url: Option<RemotePackageSourceUrl>,
41}
42
43#[derive(Debug)]
44pub(crate) struct RemotePackageSourceMetadata {
45    pub hash: Integrity,
46    pub source_url: RemotePackageSourceUrl,
47}
48
49impl<R: Rockspec, State> FetchSrcBuilder<'_, R, State>
50where
51    State: fetch_src_builder::State + fetch_src_builder::IsComplete,
52{
53    /// Fetch and unpack the source into the `dest_dir`.
54    pub async fn fetch(self) -> Result<(), FetchSrcError> {
55        self.fetch_internal().await?;
56        Ok(())
57    }
58
59    /// Fetch and unpack the source into the `dest_dir`,
60    /// returning the source `Integrity`.
61    pub(crate) async fn fetch_internal(self) -> Result<RemotePackageSourceMetadata, FetchSrcError> {
62        let fetch = self._build();
63        let rockspec = fetch.rockspec;
64        let rock_source = rockspec.source().current_platform();
65        let dest_dir = fetch.dest_dir;
66        let config = fetch.config;
67        // prioritise lockfile source, if present
68        let source_spec = match &fetch.source_url {
69            Some(source_url) => match source_url {
70                RemotePackageSourceUrl::Git {
71                    url, checkout_ref, ..
72                } => RockSourceSpec::Git(GitSource {
73                    url: url.parse()?,
74                    git_ref: Some(GitRef::Tag(checkout_ref.clone())),
75                }),
76                RemotePackageSourceUrl::Url { url } => RockSourceSpec::Url(url.clone()),
77                RemotePackageSourceUrl::File { path } => RockSourceSpec::File(path.clone()),
78            },
79            None => rock_source.source_spec.clone(),
80        };
81        match fetch_src_impl(source_spec, rockspec, rock_source, dest_dir, config).await {
82            Err(err)
83                if fetch
84                    .source_url
85                    .is_some_and(|url| matches!(url, RemotePackageSourceUrl::File { .. })) =>
86            {
87                // Don't fall back to downloading .src.rock archives if a local source was specified.
88                Err(err)
89            }
90            Err(err) => match &fetch.rockspec.source().current_platform().source_spec {
91                RockSourceSpec::Git(_) | RockSourceSpec::Url(_) => {
92                    let package = PackageSpec::new(
93                        fetch.rockspec.package().clone(),
94                        fetch.rockspec.version().clone(),
95                    );
96                    match FetchSrcRock::new(&package, fetch.dest_dir, fetch.config)
97                        .fetch()
98                        .await
99                    {
100                        Ok(metadata) => Ok(metadata),
101                        Err(fallback_err) => {
102                            tracing::error!("fallback .src.rock download failed: {fallback_err:?}");
103                            Err(err)
104                        }
105                    }
106                }
107                RockSourceSpec::File(_) => Err(err),
108            },
109            Ok(metadata) => Ok(metadata),
110        }
111    }
112}
113
114#[derive(Error, Debug, Diagnostic)]
115#[non_exhaustive]
116pub enum FetchSrcError {
117    #[error("failed to clone rock source")]
118    #[diagnostic(help("check your network connection and verify the git URL is correct."))]
119    GitClone(#[from] git2::Error),
120    #[error("failed to parse git URL")]
121    #[diagnostic(forward(0))]
122    GitUrlParse(#[from] RemoteGitUrlParseError),
123    #[error(transparent)]
124    #[diagnostic(transparent)]
125    Request(#[from] RequestError),
126    #[error(transparent)]
127    #[diagnostic(transparent)]
128    Unpack(#[from] UnpackError),
129    #[error(transparent)]
130    #[diagnostic(transparent)]
131    FetchSrcRock(#[from] FetchSrcRockError),
132    #[error("unable to remove the '.git' directory")]
133    #[diagnostic(help(
134        "check that no process is using the directory and you have write permissions."
135    ))]
136    CleanGitDir(#[source] io::Error),
137    #[error("unable to compute hash")]
138    Hash(#[source] io::Error),
139    #[error(transparent)]
140    #[diagnostic(transparent)]
141    Fs(#[from] fs::FsError),
142}
143
144impl From<reqwest::Error> for FetchSrcError {
145    fn from(err: reqwest::Error) -> Self {
146        Self::Request(err.into())
147    }
148}
149
150/// A rocks package source fetcher, providing fine-grained control
151/// over how a package should be fetched.
152#[derive(Builder)]
153#[builder(start_fn = new, finish_fn(name = _build, vis = ""))]
154struct FetchSrcRock<'a> {
155    #[builder(start_fn)]
156    package: &'a PackageSpec,
157    #[builder(start_fn)]
158    dest_dir: &'a Path,
159    #[builder(start_fn)]
160    config: &'a Config,
161}
162
163impl<State> FetchSrcRockBuilder<'_, State>
164where
165    State: fetch_src_rock_builder::State + fetch_src_rock_builder::IsComplete,
166{
167    pub async fn fetch(self) -> Result<RemotePackageSourceMetadata, FetchSrcRockError> {
168        do_fetch_src_rock(self._build()).await
169    }
170}
171
172#[derive(Error, Debug, Diagnostic)]
173#[non_exhaustive]
174#[error(transparent)]
175pub enum FetchSrcRockError {
176    DownloadSrcRock(#[from] DownloadSrcRockError),
177    Unpack(#[from] UnpackError),
178    Io(#[from] io::Error),
179}
180
181/// A no-prompt implementer for auth_git2's prompter
182#[derive(Copy, Clone, Debug)]
183struct NullPrompter;
184
185impl Prompter for NullPrompter {
186    fn prompt_username_password(&mut self, _: &str, _: &git2::Config) -> Option<(String, String)> {
187        None
188    }
189
190    fn prompt_password(&mut self, _: &str, _: &str, _: &git2::Config) -> Option<String> {
191        None
192    }
193
194    fn prompt_ssh_key_passphrase(&mut self, _: &Path, _: &git2::Config) -> Option<String> {
195        None
196    }
197}
198
199/// Initialises and checks out git submodules,
200/// returning the relative paths of any submodules that were checked out.
201fn init_submodules(
202    repo: &git2::Repository,
203    auth: &GitAuthenticator,
204    git_config: &git2::Config,
205) -> Result<Vec<PathBuf>, FetchSrcError> {
206    let workdir = repo
207        .workdir()
208        .ok_or_else(|| git2::Error::from_str("cloned repository has no working directory"))?;
209    let mut submodule_paths = Vec::new();
210    for mut submodule in repo.submodules()? {
211        let mut callbacks = RemoteCallbacks::new();
212        callbacks.credentials(auth.credentials(git_config));
213        let mut fetch_options = FetchOptions::new();
214        fetch_options.remote_callbacks(callbacks);
215        let mut options = git2::SubmoduleUpdateOptions::new();
216        options.fetch(fetch_options);
217        submodule.update(true, Some(&mut options))?;
218        // The .git file/directory is not deterministic
219        let git_file = workdir.join(submodule.path()).join(".git");
220        if git_file.is_dir() {
221            fs::sync::remove_dir_all(&git_file)?;
222        } else {
223            fs::sync::remove_file(&git_file)?;
224        }
225        submodule_paths.push(submodule.path().to_path_buf());
226    }
227    Ok(submodule_paths)
228}
229
230#[tracing::instrument(
231    name = "Fetching source",
232    level = "info",
233    skip_all,
234    fields(location = source_spec.to_string()),
235)]
236async fn fetch_src_impl<R: Rockspec>(
237    mut source_spec: RockSourceSpec,
238    rockspec: &R,
239    rock_source: &RemoteRockSource,
240    dest_dir: &Path,
241    config: &Config,
242) -> Result<RemotePackageSourceMetadata, FetchSrcError> {
243    if let Some(vendor_dir) = config.vendor_dir() {
244        source_spec = match source_spec {
245            // could be a project directory (not vendored) or a local source
246            // or a vendored dependency that we have already resolved
247            RockSourceSpec::File(_) => source_spec,
248            _ => {
249                let pkg_vendor_dir =
250                    vendor_dir.join(format!("{}@{}", rockspec.package(), rockspec.version()));
251                RockSourceSpec::File(pkg_vendor_dir)
252            }
253        }
254    }
255    let metadata = match &source_spec {
256        RockSourceSpec::Git(git) => {
257            let url = git.url.to_string();
258            tracing::debug!(message = format!("Cloning {url}").as_str());
259
260            let (checkout_ref, has_submodules) = {
261                let mut auth = if config.no_prompt() {
262                    GitAuthenticator::default()
263                        .try_password_prompt(0)
264                        .prompt_ssh_key_password(false)
265                        .set_prompter(NullPrompter)
266                } else {
267                    GitAuthenticator::default()
268                };
269                if let Some(access_token) = config.access_token(git.url.host()) {
270                    auth = auth.add_plaintext_credentials(
271                        git.url.host(),
272                        access_token.username(),
273                        unsafe { access_token.password() },
274                    );
275                }
276                let git_config = git2::Config::open_default()?;
277                let mut callbacks = RemoteCallbacks::new();
278                callbacks.credentials(auth.credentials(&git_config));
279                let mut fetch_options = FetchOptions::new();
280                fetch_options.update_fetchhead(false);
281                fetch_options.remote_callbacks(callbacks);
282                let checkout_ref = match &git.git_ref {
283                    Some(GitRef::Tag(tag)) => Some(tag.as_str()),
284                    Some(GitRef::Branch(branch)) => Some(branch.as_str()),
285                    None => None,
286                };
287                if checkout_ref.is_none() {
288                    fetch_options.depth(1);
289                };
290                let mut repo_builder = RepoBuilder::new();
291                repo_builder.fetch_options(fetch_options);
292                let repo = repo_builder.clone(&url, dest_dir)?;
293
294                let checkout_ref = match checkout_ref {
295                    Some(checkout_ref) => {
296                        let ref_name = if let Some(GitRef::Branch(branch)) = &git.git_ref {
297                            format!("origin/{branch}")
298                        } else {
299                            checkout_ref.to_string()
300                        };
301                        let (object, _) = repo.revparse_ext(&ref_name)?;
302                        repo.checkout_tree(&object, None)?;
303                        checkout_ref.to_string()
304                    }
305                    None => {
306                        let head = repo.head()?;
307                        let commit = head.peel_to_commit()?;
308                        commit.id().to_string()
309                    }
310                };
311
312                let submodule_paths = init_submodules(&repo, &auth, &git_config)?;
313
314                (checkout_ref, !submodule_paths.is_empty())
315            };
316            // The .git directory is not deterministic
317            remove_dir_all(dest_dir.join(".git")).map_err(FetchSrcError::CleanGitDir)?;
318            let hash = dest_dir.hash().await.map_err(FetchSrcError::Hash)?;
319            RemotePackageSourceMetadata {
320                hash,
321                source_url: RemotePackageSourceUrl::Git {
322                    url,
323                    checkout_ref,
324                    submodules: has_submodules,
325                },
326            }
327        }
328        RockSourceSpec::Url(url) => {
329            tracing::debug!(message = format!("Downloading {url}").as_str());
330
331            // NOTE: We don't enforce HTTPS when fetching sources because some rockspecs
332            // have HTTP URLs in `source.url`.
333            let response = crate::reqwest::http_client(config)?
334                .get(url.clone())
335                .apply_access_token(config, url)
336                .send()
337                .await?
338                .error_for_status()?
339                .bytes()
340                .await?;
341            let hash = response.hash().await.map_err(FetchSrcError::Hash)?;
342            let file_name = url
343                .path_segments()
344                .and_then(|mut segments| segments.next_back())
345                .and_then(|name| {
346                    if name.is_empty() {
347                        None
348                    } else {
349                        Some(name.to_string())
350                    }
351                })
352                .unwrap_or(url.to_string());
353            let cursor = Cursor::new(response);
354            let mime_type = infer::get(cursor.get_ref()).map(|file_type| file_type.mime_type());
355            operations::unpack::unpack(
356                mime_type,
357                cursor,
358                rock_source.unpack_dir.is_none(),
359                file_name,
360                dest_dir,
361            )
362            .await?;
363            RemotePackageSourceMetadata {
364                hash,
365                source_url: RemotePackageSourceUrl::Url { url: url.clone() },
366            }
367        }
368        RockSourceSpec::File(path) => {
369            tracing::debug!(message = format!("Copying {}", path.display()).as_str());
370
371            let hash = if path.is_dir() {
372                let hash = path.hash().await.map_err(FetchSrcError::Hash)?;
373                recursive_copy_dir(path, dest_dir).await?;
374                hash
375            } else {
376                let mut file = fs::sync::open(path)?;
377                let mut buffer = Vec::new();
378                file.read_to_end(&mut buffer)
379                    .map_err(|source| fs::FsError::Read {
380                        path: path.to_path_buf(),
381                        source,
382                    })?;
383                let mime_type = infer::get(&buffer).map(|file_type| file_type.mime_type());
384                let file_name = path
385                    .file_name()
386                    .map(|os_str| os_str.to_string_lossy())
387                    .unwrap_or(path.to_string_lossy())
388                    .to_string();
389                operations::unpack::unpack(
390                    mime_type,
391                    file,
392                    rock_source.unpack_dir.is_none(),
393                    file_name,
394                    dest_dir,
395                )
396                .await?;
397                path.hash().await.map_err(FetchSrcError::Hash)?
398            };
399            RemotePackageSourceMetadata {
400                hash,
401                source_url: RemotePackageSourceUrl::File { path: path.clone() },
402            }
403        }
404    };
405    Ok(metadata)
406}
407
408#[tracing::instrument(
409    name = "Fetching src.rock",
410    level = "info",
411    skip_all,
412    fields(package = fetch.package.to_string(),),
413)]
414async fn do_fetch_src_rock(
415    fetch: FetchSrcRock<'_>,
416) -> Result<RemotePackageSourceMetadata, FetchSrcRockError> {
417    let package = fetch.package;
418    let dest_dir = fetch.dest_dir;
419    let config = fetch.config;
420    let src_rock = operations::download_src_rock(package, config.server(), fetch.config).await?;
421    let hash = src_rock.bytes.hash().await?;
422    let cursor = Cursor::new(src_rock.bytes);
423    let mime_type = infer::get(cursor.get_ref()).map(|file_type| file_type.mime_type());
424    operations::unpack::unpack(mime_type, cursor, true, src_rock.file_name, dest_dir).await?;
425    Ok(RemotePackageSourceMetadata {
426        hash,
427        source_url: RemotePackageSourceUrl::Url { url: src_rock.url },
428    })
429}
lux_lib/operations/fetch.rs

lux_lib/operations/
fetch.rs
