lux_lib/operations/
fetch.rs

1use crate::build::utils::recursive_copy_dir;
2use crate::config::Config;
3use crate::git::url::RemoteGitUrlParseError;
4use crate::git::GitSource;
5use crate::hash::HasIntegrity;
6use crate::lockfile::RemotePackageSourceUrl;
7use crate::lua_rockspec::{RemoteRockSource, RockSourceSpec};
8use crate::package::PackageSpec;
9use crate::rockspec::Rockspec;
10use crate::{fs, operations};
11use auth_git2::{GitAuthenticator, Prompter};
12use bon::Builder;
13use git2::build::RepoBuilder;
14use git2::{FetchOptions, RemoteCallbacks};
15use miette::Diagnostic;
16use remove_dir_all::remove_dir_all;
17use ssri::Integrity;
18use std::io;
19use std::io::Cursor;
20use std::io::Read;
21use std::path::Path;
22use thiserror::Error;
23
24use super::DownloadSrcRockError;
25use super::UnpackError;
26
27/// A rocks package source fetcher, providing fine-grained control
28/// over how a package should be fetched.
29#[derive(Builder)]
30#[builder(start_fn = new, finish_fn(name = _build, vis = ""))]
31pub struct FetchSrc<'a, R: Rockspec> {
32    #[builder(start_fn)]
33    dest_dir: &'a Path,
34    #[builder(start_fn)]
35    rockspec: &'a R,
36    #[builder(start_fn)]
37    config: &'a Config,
38    #[builder(setters(vis = "pub(crate)"))]
39    source_url: Option<RemotePackageSourceUrl>,
40}
41
42#[derive(Debug)]
43pub(crate) struct RemotePackageSourceMetadata {
44    pub hash: Integrity,
45    pub source_url: RemotePackageSourceUrl,
46}
47
48impl<R: Rockspec, State> FetchSrcBuilder<'_, R, State>
49where
50    State: fetch_src_builder::State + fetch_src_builder::IsComplete,
51{
52    /// Fetch and unpack the source into the `dest_dir`.
53    pub async fn fetch(self) -> Result<(), FetchSrcError> {
54        self.fetch_internal().await?;
55        Ok(())
56    }
57
58    /// Fetch and unpack the source into the `dest_dir`,
59    /// returning the source `Integrity`.
60    pub(crate) async fn fetch_internal(self) -> Result<RemotePackageSourceMetadata, FetchSrcError> {
61        let fetch = self._build();
62        let rockspec = fetch.rockspec;
63        let rock_source = rockspec.source().current_platform();
64        let dest_dir = fetch.dest_dir;
65        let config = fetch.config;
66        // prioritise lockfile source, if present
67        let source_spec = match &fetch.source_url {
68            Some(source_url) => match source_url {
69                RemotePackageSourceUrl::Git { url, checkout_ref } => {
70                    RockSourceSpec::Git(GitSource {
71                        url: url.parse()?,
72                        checkout_ref: Some(checkout_ref.clone()),
73                    })
74                }
75                RemotePackageSourceUrl::Url { url } => RockSourceSpec::Url(url.clone()),
76                RemotePackageSourceUrl::File { path } => RockSourceSpec::File(path.clone()),
77            },
78            None => rock_source.source_spec.clone(),
79        };
80        match fetch_src_impl(source_spec, rockspec, rock_source, dest_dir, config).await {
81            Err(err)
82                if fetch
83                    .source_url
84                    .is_some_and(|url| matches!(url, RemotePackageSourceUrl::File { .. })) =>
85            {
86                // Don't fall back to downloading .src.rock archives if a local source was specified.
87                Err(err)
88            }
89            Err(err) => match &fetch.rockspec.source().current_platform().source_spec {
90                RockSourceSpec::Git(_) | RockSourceSpec::Url(_) => {
91                    let package = PackageSpec::new(
92                        fetch.rockspec.package().clone(),
93                        fetch.rockspec.version().clone(),
94                    );
95                    let metadata = FetchSrcRock::new(&package, fetch.dest_dir, fetch.config)
96                        .fetch()
97                        .await?;
98                    Ok(metadata)
99                }
100                RockSourceSpec::File(_) => Err(err),
101            },
102            Ok(metadata) => Ok(metadata),
103        }
104    }
105}
106
107#[derive(Error, Debug, Diagnostic)]
108#[non_exhaustive]
109pub enum FetchSrcError {
110    #[error("failed to clone rock source:\n{0}")]
111    #[diagnostic(help("check your network connection and verify the git URL is correct."))]
112    GitClone(#[from] git2::Error),
113    #[error("failed to parse git URL:\n{0}")]
114    #[diagnostic(forward(0))]
115    GitUrlParse(#[from] RemoteGitUrlParseError),
116    #[error(transparent)]
117    #[diagnostic(help("check your network connection."))]
118    Request(#[from] reqwest::Error),
119    #[error(transparent)]
120    #[diagnostic(transparent)]
121    Unpack(#[from] UnpackError),
122    #[error(transparent)]
123    #[diagnostic(transparent)]
124    FetchSrcRock(#[from] FetchSrcRockError),
125    #[error("unable to remove the '.git' directory:\n{0}")]
126    #[diagnostic(help(
127        "check that no process is using the directory and you have write permissions."
128    ))]
129    CleanGitDir(io::Error),
130    #[error("unable to compute hash:\n{0}")]
131    Hash(io::Error),
132    #[error(transparent)]
133    #[diagnostic(transparent)]
134    Fs(#[from] fs::FsError),
135}
136
137/// A rocks package source fetcher, providing fine-grained control
138/// over how a package should be fetched.
139#[derive(Builder)]
140#[builder(start_fn = new, finish_fn(name = _build, vis = ""))]
141struct FetchSrcRock<'a> {
142    #[builder(start_fn)]
143    package: &'a PackageSpec,
144    #[builder(start_fn)]
145    dest_dir: &'a Path,
146    #[builder(start_fn)]
147    config: &'a Config,
148}
149
150impl<State> FetchSrcRockBuilder<'_, State>
151where
152    State: fetch_src_rock_builder::State + fetch_src_rock_builder::IsComplete,
153{
154    pub async fn fetch(self) -> Result<RemotePackageSourceMetadata, FetchSrcRockError> {
155        do_fetch_src_rock(self._build()).await
156    }
157}
158
159#[derive(Error, Debug, Diagnostic)]
160#[non_exhaustive]
161#[error(transparent)]
162pub enum FetchSrcRockError {
163    DownloadSrcRock(#[from] DownloadSrcRockError),
164    Unpack(#[from] UnpackError),
165    Io(#[from] io::Error),
166}
167
168/// A no-prompt implementer for auth_git2's prompter
169#[derive(Copy, Clone, Debug)]
170struct NullPrompter;
171
172impl Prompter for NullPrompter {
173    fn prompt_username_password(&mut self, _: &str, _: &git2::Config) -> Option<(String, String)> {
174        None
175    }
176
177    fn prompt_password(&mut self, _: &str, _: &str, _: &git2::Config) -> Option<String> {
178        None
179    }
180
181    fn prompt_ssh_key_passphrase(&mut self, _: &Path, _: &git2::Config) -> Option<String> {
182        None
183    }
184}
185
186#[tracing::instrument(
187    name = "Fetching source",
188    level = "info",
189    skip_all,
190    fields(location = source_spec.to_string()),
191)]
192async fn fetch_src_impl<R: Rockspec>(
193    mut source_spec: RockSourceSpec,
194    rockspec: &R,
195    rock_source: &RemoteRockSource,
196    dest_dir: &Path,
197    config: &Config,
198) -> Result<RemotePackageSourceMetadata, FetchSrcError> {
199    if let Some(vendor_dir) = config.vendor_dir() {
200        source_spec = match source_spec {
201            // could be a project directory (not vendored) or a local source
202            // or a vendored dependency that we have already resolved
203            RockSourceSpec::File(_) => source_spec,
204            _ => {
205                let pkg_vendor_dir =
206                    vendor_dir.join(format!("{}@{}", rockspec.package(), rockspec.version()));
207                RockSourceSpec::File(pkg_vendor_dir)
208            }
209        }
210    }
211    let metadata = match &source_spec {
212        RockSourceSpec::Git(git) => {
213            let url = git.url.to_string();
214            tracing::debug!(message = format!("Cloning {url}").as_str());
215
216            let checkout_ref = {
217                let auth = if config.no_prompt() {
218                    GitAuthenticator::default()
219                        .try_password_prompt(0)
220                        .prompt_ssh_key_password(false)
221                        .set_prompter(NullPrompter)
222                } else {
223                    GitAuthenticator::default()
224                };
225                let git_config = git2::Config::open_default()?;
226                let mut callbacks = RemoteCallbacks::new();
227                callbacks.credentials(auth.credentials(&git_config));
228                let mut fetch_options = FetchOptions::new();
229                fetch_options.update_fetchhead(false);
230                fetch_options.remote_callbacks(callbacks);
231                if git.checkout_ref.is_none() {
232                    fetch_options.depth(1);
233                };
234                let mut repo_builder = RepoBuilder::new();
235                repo_builder.fetch_options(fetch_options);
236                let repo = repo_builder.clone(&url, dest_dir)?;
237
238                match &git.checkout_ref {
239                    Some(checkout_ref) => {
240                        let (object, _) = repo.revparse_ext(checkout_ref)?;
241                        repo.checkout_tree(&object, None)?;
242                        checkout_ref.clone()
243                    }
244                    None => {
245                        let head = repo.head()?;
246                        let commit = head.peel_to_commit()?;
247                        commit.id().to_string()
248                    }
249                }
250            };
251            // The .git directory is not deterministic
252            remove_dir_all(dest_dir.join(".git")).map_err(FetchSrcError::CleanGitDir)?;
253            let hash = dest_dir.hash().await.map_err(FetchSrcError::Hash)?;
254            RemotePackageSourceMetadata {
255                hash,
256                source_url: RemotePackageSourceUrl::Git { url, checkout_ref },
257            }
258        }
259        RockSourceSpec::Url(url) => {
260            tracing::debug!(message = format!("📥 Downloading {url}").as_str());
261
262            // NOTE: We don't enforce HTTPS when fetching sources because some rockspecs
263            // have HTTP URLs in `source.url`.
264            let response = crate::reqwest::http_client(config)?
265                .get(url.clone())
266                .send()
267                .await?
268                .error_for_status()?
269                .bytes()
270                .await?;
271            let hash = response.hash().await.map_err(FetchSrcError::Hash)?;
272            let file_name = url
273                .path_segments()
274                .and_then(|mut segments| segments.next_back())
275                .and_then(|name| {
276                    if name.is_empty() {
277                        None
278                    } else {
279                        Some(name.to_string())
280                    }
281                })
282                .unwrap_or(url.to_string());
283            let cursor = Cursor::new(response);
284            let mime_type = infer::get(cursor.get_ref()).map(|file_type| file_type.mime_type());
285            operations::unpack::unpack(
286                mime_type,
287                cursor,
288                rock_source.unpack_dir.is_none(),
289                file_name,
290                dest_dir,
291            )
292            .await?;
293            RemotePackageSourceMetadata {
294                hash,
295                source_url: RemotePackageSourceUrl::Url { url: url.clone() },
296            }
297        }
298        RockSourceSpec::File(path) => {
299            tracing::debug!(message = format!("📋 Copying {}", path.display()).as_str());
300
301            let hash = if path.is_dir() {
302                recursive_copy_dir(&path.to_path_buf(), dest_dir).await?;
303                dest_dir.hash().await.map_err(FetchSrcError::Hash)?
304            } else {
305                let mut file = fs::sync::open(path)?;
306                let mut buffer = Vec::new();
307                file.read_to_end(&mut buffer)
308                    .map_err(|source| fs::FsError::Read {
309                        path: path.to_path_buf(),
310                        source,
311                    })?;
312                let mime_type = infer::get(&buffer).map(|file_type| file_type.mime_type());
313                let file_name = path
314                    .file_name()
315                    .map(|os_str| os_str.to_string_lossy())
316                    .unwrap_or(path.to_string_lossy())
317                    .to_string();
318                operations::unpack::unpack(
319                    mime_type,
320                    file,
321                    rock_source.unpack_dir.is_none(),
322                    file_name,
323                    dest_dir,
324                )
325                .await?;
326                path.hash().await.map_err(FetchSrcError::Hash)?
327            };
328            RemotePackageSourceMetadata {
329                hash,
330                source_url: RemotePackageSourceUrl::File { path: path.clone() },
331            }
332        }
333    };
334    Ok(metadata)
335}
336
337#[tracing::instrument(
338    name = "Fetching src.rock",
339    level = "info",
340    skip_all,
341    fields(package = fetch.package.to_string(),),
342)]
343async fn do_fetch_src_rock(
344    fetch: FetchSrcRock<'_>,
345) -> Result<RemotePackageSourceMetadata, FetchSrcRockError> {
346    let package = fetch.package;
347    let dest_dir = fetch.dest_dir;
348    let config = fetch.config;
349    let src_rock = operations::download_src_rock(package, config.server(), fetch.config).await?;
350    let hash = src_rock.bytes.hash().await?;
351    let cursor = Cursor::new(src_rock.bytes);
352    let mime_type = infer::get(cursor.get_ref()).map(|file_type| file_type.mime_type());
353    operations::unpack::unpack(mime_type, cursor, true, src_rock.file_name, dest_dir).await?;
354    Ok(RemotePackageSourceMetadata {
355        hash,
356        source_url: RemotePackageSourceUrl::Url { url: src_rock.url },
357    })
358}
lux_lib/operations/fetch.rs

lux_lib/operations/
fetch.rs
