lux_lib/operations/
fetch.rs

1use crate::build::utils::recursive_copy_dir;
2use crate::config::Config;
3use crate::git::url::RemoteGitUrlParseError;
4use crate::git::GitSource;
5use crate::hash::HasIntegrity;
6use crate::lockfile::RemotePackageSourceUrl;
7use crate::lua_rockspec::RockSourceSpec;
8use crate::package::PackageSpec;
9use crate::rockspec::Rockspec;
10use crate::{fs, operations};
11use auth_git2::{GitAuthenticator, Prompter};
12use bon::Builder;
13use git2::build::RepoBuilder;
14use git2::{FetchOptions, RemoteCallbacks};
15use miette::Diagnostic;
16use remove_dir_all::remove_dir_all;
17use ssri::Integrity;
18use std::io;
19use std::io::Cursor;
20use std::io::Read;
21use std::path::Path;
22use thiserror::Error;
23use tracing::span;
24
25use super::DownloadSrcRockError;
26use super::UnpackError;
27
28/// A rocks package source fetcher, providing fine-grained control
29/// over how a package should be fetched.
30#[derive(Builder)]
31#[builder(start_fn = new, finish_fn(name = _build, vis = ""))]
32pub struct FetchSrc<'a, R: Rockspec> {
33    #[builder(start_fn)]
34    dest_dir: &'a Path,
35    #[builder(start_fn)]
36    rockspec: &'a R,
37    #[builder(start_fn)]
38    config: &'a Config,
39    #[builder(setters(vis = "pub(crate)"))]
40    source_url: Option<RemotePackageSourceUrl>,
41}
42
43#[derive(Debug)]
44pub(crate) struct RemotePackageSourceMetadata {
45    pub hash: Integrity,
46    pub source_url: RemotePackageSourceUrl,
47}
48
49impl<R: Rockspec, State> FetchSrcBuilder<'_, R, State>
50where
51    State: fetch_src_builder::State + fetch_src_builder::IsComplete,
52{
53    /// Fetch and unpack the source into the `dest_dir`.
54    pub async fn fetch(self) -> Result<(), FetchSrcError> {
55        self.fetch_internal().await?;
56        Ok(())
57    }
58
59    /// Fetch and unpack the source into the `dest_dir`,
60    /// returning the source `Integrity`.
61    pub(crate) async fn fetch_internal(self) -> Result<RemotePackageSourceMetadata, FetchSrcError> {
62        let fetch = self._build();
63        match do_fetch_src(&fetch).await {
64            Err(err)
65                if fetch
66                    .source_url
67                    .is_some_and(|url| matches!(url, RemotePackageSourceUrl::File { .. })) =>
68            {
69                // Don't fall back to downloading .src.rock archives if a local source was specified.
70                Err(err)
71            }
72            Err(err) => match &fetch.rockspec.source().current_platform().source_spec {
73                RockSourceSpec::Git(_) | RockSourceSpec::Url(_) => {
74                    let package = PackageSpec::new(
75                        fetch.rockspec.package().clone(),
76                        fetch.rockspec.version().clone(),
77                    );
78                    let metadata = FetchSrcRock::new(&package, fetch.dest_dir, fetch.config)
79                        .fetch()
80                        .await?;
81                    Ok(metadata)
82                }
83                RockSourceSpec::File(_) => Err(err),
84            },
85            Ok(metadata) => Ok(metadata),
86        }
87    }
88}
89
90#[derive(Error, Debug, Diagnostic)]
91#[non_exhaustive]
92pub enum FetchSrcError {
93    #[error("failed to clone rock source:\n{0}")]
94    #[diagnostic(help("check your network connection and verify the git URL is correct."))]
95    GitClone(#[from] git2::Error),
96    #[error("failed to parse git URL:\n{0}")]
97    #[diagnostic(forward(0))]
98    GitUrlParse(#[from] RemoteGitUrlParseError),
99    #[error(transparent)]
100    #[diagnostic(help("check your network connection."))]
101    Request(#[from] reqwest::Error),
102    #[error(transparent)]
103    #[diagnostic(transparent)]
104    Unpack(#[from] UnpackError),
105    #[error(transparent)]
106    #[diagnostic(transparent)]
107    FetchSrcRock(#[from] FetchSrcRockError),
108    #[error("unable to remove the '.git' directory:\n{0}")]
109    #[diagnostic(help(
110        "check that no process is using the directory and you have write permissions."
111    ))]
112    CleanGitDir(io::Error),
113    #[error("unable to compute hash:\n{0}")]
114    Hash(io::Error),
115    #[error(transparent)]
116    #[diagnostic(transparent)]
117    Fs(#[from] fs::FsError),
118}
119
120/// A rocks package source fetcher, providing fine-grained control
121/// over how a package should be fetched.
122#[derive(Builder)]
123#[builder(start_fn = new, finish_fn(name = _build, vis = ""))]
124struct FetchSrcRock<'a> {
125    #[builder(start_fn)]
126    package: &'a PackageSpec,
127    #[builder(start_fn)]
128    dest_dir: &'a Path,
129    #[builder(start_fn)]
130    config: &'a Config,
131}
132
133impl<State> FetchSrcRockBuilder<'_, State>
134where
135    State: fetch_src_rock_builder::State + fetch_src_rock_builder::IsComplete,
136{
137    pub async fn fetch(self) -> Result<RemotePackageSourceMetadata, FetchSrcRockError> {
138        do_fetch_src_rock(self._build()).await
139    }
140}
141
142#[derive(Error, Debug, Diagnostic)]
143#[non_exhaustive]
144#[error(transparent)]
145pub enum FetchSrcRockError {
146    DownloadSrcRock(#[from] DownloadSrcRockError),
147    Unpack(#[from] UnpackError),
148    Io(#[from] io::Error),
149}
150
151/// A no-prompt implementer for auth_git2's prompter
152#[derive(Copy, Clone, Debug)]
153struct NullPrompter;
154
155impl Prompter for NullPrompter {
156    fn prompt_username_password(&mut self, _: &str, _: &git2::Config) -> Option<(String, String)> {
157        None
158    }
159
160    fn prompt_password(&mut self, _: &str, _: &str, _: &git2::Config) -> Option<String> {
161        None
162    }
163
164    fn prompt_ssh_key_passphrase(&mut self, _: &Path, _: &git2::Config) -> Option<String> {
165        None
166    }
167}
168
169async fn do_fetch_src<R: Rockspec>(
170    fetch: &FetchSrc<'_, R>,
171) -> Result<RemotePackageSourceMetadata, FetchSrcError> {
172    let rockspec = fetch.rockspec;
173    let rock_source = rockspec.source().current_platform();
174    let dest_dir = fetch.dest_dir;
175    let config = fetch.config;
176    // prioritise lockfile source, if present
177    let mut source_spec = match &fetch.source_url {
178        Some(source_url) => match source_url {
179            RemotePackageSourceUrl::Git { url, checkout_ref } => RockSourceSpec::Git(GitSource {
180                url: url.parse()?,
181                checkout_ref: Some(checkout_ref.clone()),
182            }),
183            RemotePackageSourceUrl::Url { url } => RockSourceSpec::Url(url.clone()),
184            RemotePackageSourceUrl::File { path } => RockSourceSpec::File(path.clone()),
185        },
186        None => rock_source.source_spec.clone(),
187    };
188    let span = span!(
189        tracing::Level::INFO,
190        "Fetching source",
191        location = source_spec.to_string(),
192    );
193    let _enter = span.enter();
194
195    if let Some(vendor_dir) = config.vendor_dir() {
196        source_spec = match source_spec {
197            // could be a project directory (not vendored) or a local source
198            // or a vendored dependency that we have already resolved
199            RockSourceSpec::File(_) => source_spec,
200            _ => {
201                let pkg_vendor_dir =
202                    vendor_dir.join(format!("{}@{}", rockspec.package(), rockspec.version()));
203                RockSourceSpec::File(pkg_vendor_dir)
204            }
205        }
206    }
207    let metadata = match &source_spec {
208        RockSourceSpec::Git(git) => {
209            let url = git.url.to_string();
210            tracing::debug!(message = format!("Cloning {url}").as_str());
211
212            let auth = if config.no_prompt() {
213                GitAuthenticator::default()
214                    .try_password_prompt(0)
215                    .prompt_ssh_key_password(false)
216                    .set_prompter(NullPrompter)
217            } else {
218                GitAuthenticator::default()
219            };
220            let git_config = git2::Config::open_default()?;
221            let mut callbacks = RemoteCallbacks::new();
222            callbacks.credentials(auth.credentials(&git_config));
223            let mut fetch_options = FetchOptions::new();
224            fetch_options.update_fetchhead(false);
225            fetch_options.remote_callbacks(callbacks);
226            if git.checkout_ref.is_none() {
227                fetch_options.depth(1);
228            };
229            let mut repo_builder = RepoBuilder::new();
230            repo_builder.fetch_options(fetch_options);
231            let repo = repo_builder.clone(&url, dest_dir)?;
232
233            let checkout_ref = match &git.checkout_ref {
234                Some(checkout_ref) => {
235                    let (object, _) = repo.revparse_ext(checkout_ref)?;
236                    repo.checkout_tree(&object, None)?;
237                    checkout_ref.clone()
238                }
239                None => {
240                    let head = repo.head()?;
241                    let commit = head.peel_to_commit()?;
242                    commit.id().to_string()
243                }
244            };
245            // The .git directory is not deterministic
246            remove_dir_all(dest_dir.join(".git")).map_err(FetchSrcError::CleanGitDir)?;
247            let hash = fetch.dest_dir.hash().map_err(FetchSrcError::Hash)?;
248            RemotePackageSourceMetadata {
249                hash,
250                source_url: RemotePackageSourceUrl::Git { url, checkout_ref },
251            }
252        }
253        RockSourceSpec::Url(url) => {
254            tracing::debug!(message = format!("📥 Downloading {url}").as_str());
255
256            // NOTE: We don't enforce HTTPS when fetching sources because some rockspecs
257            // have HTTP URLs in `source.url`.
258            let response = crate::reqwest::new_http_client(config)?
259                .get(url.clone())
260                .send()
261                .await?
262                .error_for_status()?
263                .bytes()
264                .await?;
265            let hash = response.hash().map_err(FetchSrcError::Hash)?;
266            let file_name = url
267                .path_segments()
268                .and_then(|mut segments| segments.next_back())
269                .and_then(|name| {
270                    if name.is_empty() {
271                        None
272                    } else {
273                        Some(name.to_string())
274                    }
275                })
276                .unwrap_or(url.to_string());
277            let cursor = Cursor::new(response);
278            let mime_type = infer::get(cursor.get_ref()).map(|file_type| file_type.mime_type());
279            operations::unpack::unpack(
280                mime_type,
281                cursor,
282                rock_source.unpack_dir.is_none(),
283                file_name,
284                dest_dir,
285            )
286            .await?;
287            RemotePackageSourceMetadata {
288                hash,
289                source_url: RemotePackageSourceUrl::Url { url: url.clone() },
290            }
291        }
292        RockSourceSpec::File(path) => {
293            tracing::debug!(message = format!("📋 Copying {}", path.display()).as_str());
294
295            let hash = if path.is_dir() {
296                recursive_copy_dir(&path.to_path_buf(), dest_dir).await?;
297                dest_dir.hash().map_err(FetchSrcError::Hash)?
298            } else {
299                let mut file = fs::sync::open(path)?;
300                let mut buffer = Vec::new();
301                file.read_to_end(&mut buffer)
302                    .map_err(|source| fs::FsError::Read {
303                        path: path.to_path_buf(),
304                        source,
305                    })?;
306                let mime_type = infer::get(&buffer).map(|file_type| file_type.mime_type());
307                let file_name = path
308                    .file_name()
309                    .map(|os_str| os_str.to_string_lossy())
310                    .unwrap_or(path.to_string_lossy())
311                    .to_string();
312                operations::unpack::unpack(
313                    mime_type,
314                    file,
315                    rock_source.unpack_dir.is_none(),
316                    file_name,
317                    dest_dir,
318                )
319                .await?;
320                path.hash().map_err(FetchSrcError::Hash)?
321            };
322            RemotePackageSourceMetadata {
323                hash,
324                source_url: RemotePackageSourceUrl::File { path: path.clone() },
325            }
326        }
327    };
328    Ok(metadata)
329}
330
331async fn do_fetch_src_rock(
332    fetch: FetchSrcRock<'_>,
333) -> Result<RemotePackageSourceMetadata, FetchSrcRockError> {
334    let package = fetch.package;
335    let span = span!(
336        tracing::Level::INFO,
337        "Fetching src.rock",
338        package = package.to_string(),
339    );
340    let _enter = span.enter();
341
342    let dest_dir = fetch.dest_dir;
343    let config = fetch.config;
344    let src_rock = operations::download_src_rock(package, config.server(), fetch.config).await?;
345    let hash = src_rock.bytes.hash()?;
346    let cursor = Cursor::new(src_rock.bytes);
347    let mime_type = infer::get(cursor.get_ref()).map(|file_type| file_type.mime_type());
348    operations::unpack::unpack(mime_type, cursor, true, src_rock.file_name, dest_dir).await?;
349    Ok(RemotePackageSourceMetadata {
350        hash,
351        source_url: RemotePackageSourceUrl::Url { url: src_rock.url },
352    })
353}
lux_lib/operations/fetch.rs

lux_lib/operations/
fetch.rs
