lux_lib/operations/
fetch.rs

1use crate::build::utils::recursive_copy_dir;
2use crate::config::Config;
3use crate::git::url::RemoteGitUrlParseError;
4use crate::git::GitSource;
5use crate::hash::HasIntegrity;
6use crate::lockfile::RemotePackageSourceUrl;
7use crate::lua_rockspec::RockSourceSpec;
8use crate::operations;
9use crate::package::PackageSpec;
10use crate::progress::Progress;
11use crate::progress::ProgressBar;
12use crate::rockspec::Rockspec;
13use auth_git2::{GitAuthenticator, Prompter};
14use bon::Builder;
15use git2::build::RepoBuilder;
16use git2::{FetchOptions, RemoteCallbacks};
17use miette::Diagnostic;
18use remove_dir_all::remove_dir_all;
19use ssri::Integrity;
20use std::fs::File;
21use std::io;
22use std::io::Cursor;
23use std::io::Read;
24use std::path::Path;
25use std::path::PathBuf;
26use thiserror::Error;
27
28use super::DownloadSrcRockError;
29use super::UnpackError;
30
31/// A rocks package source fetcher, providing fine-grained control
32/// over how a package should be fetched.
33#[derive(Builder)]
34#[builder(start_fn = new, finish_fn(name = _build, vis = ""))]
35pub struct FetchSrc<'a, R: Rockspec> {
36    #[builder(start_fn)]
37    dest_dir: &'a Path,
38    #[builder(start_fn)]
39    rockspec: &'a R,
40    #[builder(start_fn)]
41    config: &'a Config,
42    #[builder(start_fn)]
43    progress: &'a Progress<ProgressBar>,
44    #[builder(setters(vis = "pub(crate)"))]
45    source_url: Option<RemotePackageSourceUrl>,
46}
47
48#[derive(Debug)]
49pub(crate) struct RemotePackageSourceMetadata {
50    pub hash: Integrity,
51    pub source_url: RemotePackageSourceUrl,
52}
53
54impl<R: Rockspec, State> FetchSrcBuilder<'_, R, State>
55where
56    State: fetch_src_builder::State + fetch_src_builder::IsComplete,
57{
58    /// Fetch and unpack the source into the `dest_dir`.
59    pub async fn fetch(self) -> Result<(), FetchSrcError> {
60        self.fetch_internal().await?;
61        Ok(())
62    }
63
64    /// Fetch and unpack the source into the `dest_dir`,
65    /// returning the source `Integrity`.
66    pub(crate) async fn fetch_internal(self) -> Result<RemotePackageSourceMetadata, FetchSrcError> {
67        let fetch = self._build();
68        match do_fetch_src(&fetch).await {
69            Err(err)
70                if fetch
71                    .source_url
72                    .is_some_and(|url| matches!(url, RemotePackageSourceUrl::File { .. })) =>
73            {
74                // Don't fall back to downloading .src.rock archives if a local source was specified.
75                Err(err)
76            }
77            Err(err) => match &fetch.rockspec.source().current_platform().source_spec {
78                RockSourceSpec::Git(_) | RockSourceSpec::Url(_) => {
79                    let package = PackageSpec::new(
80                        fetch.rockspec.package().clone(),
81                        fetch.rockspec.version().clone(),
82                    );
83                    fetch.progress.map(|p| {
84                        p.println(format!(
85                            "⚠️ WARNING: Failed to fetch source for {}: {}",
86                            &package, err
87                        ))
88                    });
89                    fetch.progress.map(|p| {
90                        p.println(format!(
91                            "⚠️ Falling back to searching for a .src.rock archive on {}",
92                            fetch.config.server()
93                        ))
94                    });
95                    let metadata =
96                        FetchSrcRock::new(&package, fetch.dest_dir, fetch.config, fetch.progress)
97                            .fetch()
98                            .await?;
99                    Ok(metadata)
100                }
101                RockSourceSpec::File(_) => Err(err),
102            },
103            Ok(metadata) => Ok(metadata),
104        }
105    }
106}
107
108#[derive(Error, Debug, Diagnostic)]
109pub enum FetchSrcError {
110    #[error("failed to clone rock source:\n{0}")]
111    GitClone(#[from] git2::Error),
112    #[error("failed to parse git URL:\n{0}")]
113    #[diagnostic(forward(0))]
114    GitUrlParse(#[from] RemoteGitUrlParseError),
115    #[error(transparent)]
116    Request(#[from] reqwest::Error),
117    #[error(transparent)]
118    #[diagnostic(transparent)]
119    Unpack(#[from] UnpackError),
120    #[error(transparent)]
121    #[diagnostic(transparent)]
122    FetchSrcRock(#[from] FetchSrcRockError),
123    #[error("unable to remove the '.git' directory:\n{0}")]
124    CleanGitDir(io::Error),
125    #[error("unable to compute hash:\n{0}")]
126    Hash(io::Error),
127    #[error("unable to copy {src} to {dest}:\n{err}")]
128    CopyDir {
129        src: PathBuf,
130        dest: PathBuf,
131        err: io::Error,
132    },
133    #[error("unable to open {file}:\n{err}")]
134    FileOpen { file: PathBuf, err: io::Error },
135    #[error("unable to read {file}:\n{err}")]
136    FileRead { file: PathBuf, err: io::Error },
137}
138
139/// A rocks package source fetcher, providing fine-grained control
140/// over how a package should be fetched.
141#[derive(Builder)]
142#[builder(start_fn = new, finish_fn(name = _build, vis = ""))]
143struct FetchSrcRock<'a> {
144    #[builder(start_fn)]
145    package: &'a PackageSpec,
146    #[builder(start_fn)]
147    dest_dir: &'a Path,
148    #[builder(start_fn)]
149    config: &'a Config,
150    #[builder(start_fn)]
151    progress: &'a Progress<ProgressBar>,
152}
153
154impl<State> FetchSrcRockBuilder<'_, State>
155where
156    State: fetch_src_rock_builder::State + fetch_src_rock_builder::IsComplete,
157{
158    pub async fn fetch(self) -> Result<RemotePackageSourceMetadata, FetchSrcRockError> {
159        do_fetch_src_rock(self._build()).await
160    }
161}
162
163#[derive(Error, Debug, Diagnostic)]
164#[error(transparent)]
165pub enum FetchSrcRockError {
166    DownloadSrcRock(#[from] DownloadSrcRockError),
167    Unpack(#[from] UnpackError),
168    Io(#[from] io::Error),
169}
170
171/// A no-prompt implementer for auth_git2's prompter
172#[derive(Copy, Clone, Debug)]
173struct NullPrompter;
174
175impl Prompter for NullPrompter {
176    fn prompt_username_password(&mut self, _: &str, _: &git2::Config) -> Option<(String, String)> {
177        None
178    }
179
180    fn prompt_password(&mut self, _: &str, _: &str, _: &git2::Config) -> Option<String> {
181        None
182    }
183
184    fn prompt_ssh_key_passphrase(&mut self, _: &Path, _: &git2::Config) -> Option<String> {
185        None
186    }
187}
188
189async fn do_fetch_src<R: Rockspec>(
190    fetch: &FetchSrc<'_, R>,
191) -> Result<RemotePackageSourceMetadata, FetchSrcError> {
192    let rockspec = fetch.rockspec;
193    let rock_source = rockspec.source().current_platform();
194    let progress = fetch.progress;
195    let dest_dir = fetch.dest_dir;
196    let config = fetch.config;
197    // prioritise lockfile source, if present
198    let mut source_spec = match &fetch.source_url {
199        Some(source_url) => match source_url {
200            RemotePackageSourceUrl::Git { url, checkout_ref } => RockSourceSpec::Git(GitSource {
201                url: url.parse()?,
202                checkout_ref: Some(checkout_ref.clone()),
203            }),
204            RemotePackageSourceUrl::Url { url } => RockSourceSpec::Url(url.clone()),
205            RemotePackageSourceUrl::File { path } => RockSourceSpec::File(path.clone()),
206        },
207        None => rock_source.source_spec.clone(),
208    };
209    if let Some(vendor_dir) = config.vendor_dir() {
210        source_spec = match source_spec {
211            // could be a project directory (not vendored) or a local source
212            // or a vendored dependency that we have already resolved
213            RockSourceSpec::File(_) => source_spec,
214            _ => {
215                let pkg_vendor_dir =
216                    vendor_dir.join(format!("{}@{}", rockspec.package(), rockspec.version()));
217                RockSourceSpec::File(pkg_vendor_dir)
218            }
219        }
220    }
221    let metadata = match &source_spec {
222        RockSourceSpec::Git(git) => {
223            let url = git.url.to_string();
224            progress.map(|p| p.set_message(format!("🦠 Cloning {url}")));
225
226            let auth = if config.no_prompt() {
227                GitAuthenticator::default()
228                    .try_password_prompt(0)
229                    .prompt_ssh_key_password(false)
230                    .set_prompter(NullPrompter)
231            } else {
232                GitAuthenticator::default()
233            };
234            let git_config = git2::Config::open_default()?;
235            let mut callbacks = RemoteCallbacks::new();
236            callbacks.credentials(auth.credentials(&git_config));
237            let mut fetch_options = FetchOptions::new();
238            fetch_options.update_fetchhead(false);
239            fetch_options.remote_callbacks(callbacks);
240            if git.checkout_ref.is_none() {
241                fetch_options.depth(1);
242            };
243            let mut repo_builder = RepoBuilder::new();
244            repo_builder.fetch_options(fetch_options);
245            let repo = repo_builder.clone(&url, dest_dir)?;
246
247            let checkout_ref = match &git.checkout_ref {
248                Some(checkout_ref) => {
249                    let (object, _) = repo.revparse_ext(checkout_ref)?;
250                    repo.checkout_tree(&object, None)?;
251                    checkout_ref.clone()
252                }
253                None => {
254                    let head = repo.head()?;
255                    let commit = head.peel_to_commit()?;
256                    commit.id().to_string()
257                }
258            };
259            // The .git directory is not deterministic
260            remove_dir_all(dest_dir.join(".git")).map_err(FetchSrcError::CleanGitDir)?;
261            let hash = fetch.dest_dir.hash().map_err(FetchSrcError::Hash)?;
262            RemotePackageSourceMetadata {
263                hash,
264                source_url: RemotePackageSourceUrl::Git { url, checkout_ref },
265            }
266        }
267        RockSourceSpec::Url(url) => {
268            progress.map(|p| p.set_message(format!("📥 Downloading {}", url.to_owned())));
269
270            // NOTE: We don't enforce HTTPS when fetching sources because some rockspecs
271            // have HTTP URLs in `source.url`.
272            let response = crate::reqwest::new_http_client(config)?
273                .get(url.clone())
274                .send()
275                .await?
276                .error_for_status()?
277                .bytes()
278                .await?;
279            let hash = response.hash().map_err(FetchSrcError::Hash)?;
280            let file_name = url
281                .path_segments()
282                .and_then(|mut segments| segments.next_back())
283                .and_then(|name| {
284                    if name.is_empty() {
285                        None
286                    } else {
287                        Some(name.to_string())
288                    }
289                })
290                .unwrap_or(url.to_string());
291            let cursor = Cursor::new(response);
292            let mime_type = infer::get(cursor.get_ref()).map(|file_type| file_type.mime_type());
293            operations::unpack::unpack(
294                mime_type,
295                cursor,
296                rock_source.unpack_dir.is_none(),
297                file_name,
298                dest_dir,
299                progress,
300            )
301            .await?;
302            RemotePackageSourceMetadata {
303                hash,
304                source_url: RemotePackageSourceUrl::Url { url: url.clone() },
305            }
306        }
307        RockSourceSpec::File(path) => {
308            let hash = if path.is_dir() {
309                progress.map(|p| p.set_message(format!("📋 Copying {}", path.display())));
310                recursive_copy_dir(&path.to_path_buf(), dest_dir)
311                    .await
312                    .map_err(|err| FetchSrcError::CopyDir {
313                        src: path.to_path_buf(),
314                        dest: dest_dir.to_path_buf(),
315                        err,
316                    })?;
317                progress.map(|p| p.finish_and_clear());
318                dest_dir.hash().map_err(FetchSrcError::Hash)?
319            } else {
320                let mut file = File::open(path).map_err(|err| FetchSrcError::FileOpen {
321                    file: path.clone(),
322                    err,
323                })?;
324                let mut buffer = Vec::new();
325                file.read_to_end(&mut buffer)
326                    .map_err(|err| FetchSrcError::FileRead {
327                        file: path.clone(),
328                        err,
329                    })?;
330                let mime_type = infer::get(&buffer).map(|file_type| file_type.mime_type());
331                let file_name = path
332                    .file_name()
333                    .map(|os_str| os_str.to_string_lossy())
334                    .unwrap_or(path.to_string_lossy())
335                    .to_string();
336                operations::unpack::unpack(
337                    mime_type,
338                    file,
339                    rock_source.unpack_dir.is_none(),
340                    file_name,
341                    dest_dir,
342                    progress,
343                )
344                .await?;
345                path.hash().map_err(FetchSrcError::Hash)?
346            };
347            RemotePackageSourceMetadata {
348                hash,
349                source_url: RemotePackageSourceUrl::File { path: path.clone() },
350            }
351        }
352    };
353    Ok(metadata)
354}
355
356async fn do_fetch_src_rock(
357    fetch: FetchSrcRock<'_>,
358) -> Result<RemotePackageSourceMetadata, FetchSrcRockError> {
359    let package = fetch.package;
360    let dest_dir = fetch.dest_dir;
361    let config = fetch.config;
362    let progress = fetch.progress;
363    let src_rock =
364        operations::download_src_rock(package, config.server(), progress, fetch.config).await?;
365    let hash = src_rock.bytes.hash()?;
366    let cursor = Cursor::new(src_rock.bytes);
367    let mime_type = infer::get(cursor.get_ref()).map(|file_type| file_type.mime_type());
368    operations::unpack::unpack(
369        mime_type,
370        cursor,
371        true,
372        src_rock.file_name,
373        dest_dir,
374        progress,
375    )
376    .await?;
377    Ok(RemotePackageSourceMetadata {
378        hash,
379        source_url: RemotePackageSourceUrl::Url { url: src_rock.url },
380    })
381}
lux_lib/operations/fetch.rs

lux_lib/operations/
fetch.rs
