lux_lib/operations/
fetch.rs

1use auth_git2::{GitAuthenticator, Prompter};
2use bon::Builder;
3use git2::build::RepoBuilder;
4use git2::{FetchOptions, RemoteCallbacks};
5use remove_dir_all::remove_dir_all;
6use ssri::Integrity;
7use std::fs::File;
8use std::io;
9use std::io::Cursor;
10use std::io::Read;
11use std::path::Path;
12use std::path::PathBuf;
13use thiserror::Error;
14
15use crate::build::utils::recursive_copy_dir;
16use crate::config::Config;
17use crate::git::url::RemoteGitUrlParseError;
18use crate::git::GitSource;
19use crate::hash::HasIntegrity;
20use crate::lockfile::RemotePackageSourceUrl;
21use crate::lua_rockspec::RockSourceSpec;
22use crate::operations;
23use crate::package::PackageSpec;
24use crate::progress::Progress;
25use crate::progress::ProgressBar;
26use crate::rockspec::Rockspec;
27
28use super::DownloadSrcRockError;
29use super::UnpackError;
30
31/// A rocks package source fetcher, providing fine-grained control
32/// over how a package should be fetched.
33#[derive(Builder)]
34#[builder(start_fn = new, finish_fn(name = _build, vis = ""))]
35pub struct FetchSrc<'a, R: Rockspec> {
36    #[builder(start_fn)]
37    dest_dir: &'a Path,
38    #[builder(start_fn)]
39    rockspec: &'a R,
40    #[builder(start_fn)]
41    config: &'a Config,
42    #[builder(start_fn)]
43    progress: &'a Progress<ProgressBar>,
44    #[builder(setters(vis = "pub(crate)"))]
45    source_url: Option<RemotePackageSourceUrl>,
46}
47
48#[derive(Debug)]
49pub(crate) struct RemotePackageSourceMetadata {
50    pub hash: Integrity,
51    pub source_url: RemotePackageSourceUrl,
52}
53
54impl<R: Rockspec, State> FetchSrcBuilder<'_, R, State>
55where
56    State: fetch_src_builder::State + fetch_src_builder::IsComplete,
57{
58    /// Fetch and unpack the source into the `dest_dir`.
59    pub async fn fetch(self) -> Result<(), FetchSrcError> {
60        self.fetch_internal().await?;
61        Ok(())
62    }
63
64    /// Fetch and unpack the source into the `dest_dir`,
65    /// returning the source `Integrity`.
66    pub(crate) async fn fetch_internal(self) -> Result<RemotePackageSourceMetadata, FetchSrcError> {
67        let fetch = self._build();
68        match do_fetch_src(&fetch).await {
69            Err(err)
70                if fetch
71                    .source_url
72                    .is_some_and(|url| matches!(url, RemotePackageSourceUrl::File { .. })) =>
73            {
74                // Don't fall back to downloading .src.rock archives if a local source was specified.
75                Err(err)
76            }
77            Err(err) => match &fetch.rockspec.source().current_platform().source_spec {
78                RockSourceSpec::Git(_) | RockSourceSpec::Url(_) => {
79                    let package = PackageSpec::new(
80                        fetch.rockspec.package().clone(),
81                        fetch.rockspec.version().clone(),
82                    );
83                    fetch.progress.map(|p| {
84                        p.println(format!(
85                            "⚠️ WARNING: Failed to fetch source for {}: {}",
86                            &package, err
87                        ))
88                    });
89                    fetch.progress.map(|p| {
90                        p.println(format!(
91                            "⚠️ Falling back to searching for a .src.rock archive on {}",
92                            fetch.config.server()
93                        ))
94                    });
95                    let metadata =
96                        FetchSrcRock::new(&package, fetch.dest_dir, fetch.config, fetch.progress)
97                            .fetch()
98                            .await?;
99                    Ok(metadata)
100                }
101                RockSourceSpec::File(_) => Err(err),
102            },
103            Ok(metadata) => Ok(metadata),
104        }
105    }
106}
107
108#[derive(Error, Debug)]
109pub enum FetchSrcError {
110    #[error("failed to clone rock source:\n{0}")]
111    GitClone(#[from] git2::Error),
112    #[error("failed to parse git URL:\n{0}")]
113    GitUrlParse(#[from] RemoteGitUrlParseError),
114    #[error(transparent)]
115    Request(#[from] reqwest::Error),
116    #[error(transparent)]
117    Unpack(#[from] UnpackError),
118    #[error(transparent)]
119    FetchSrcRock(#[from] FetchSrcRockError),
120    #[error("unable to remove the '.git' directory:\n{0}")]
121    CleanGitDir(io::Error),
122    #[error("unable to compute hash:\n{0}")]
123    Hash(io::Error),
124    #[error("unable to copy {src} to {dest}:\n{err}")]
125    CopyDir {
126        src: PathBuf,
127        dest: PathBuf,
128        err: io::Error,
129    },
130    #[error("unable to open {file}:\n{err}")]
131    FileOpen { file: PathBuf, err: io::Error },
132    #[error("unable to read {file}:\n{err}")]
133    FileRead { file: PathBuf, err: io::Error },
134}
135
136/// A rocks package source fetcher, providing fine-grained control
137/// over how a package should be fetched.
138#[derive(Builder)]
139#[builder(start_fn = new, finish_fn(name = _build, vis = ""))]
140struct FetchSrcRock<'a> {
141    #[builder(start_fn)]
142    package: &'a PackageSpec,
143    #[builder(start_fn)]
144    dest_dir: &'a Path,
145    #[builder(start_fn)]
146    config: &'a Config,
147    #[builder(start_fn)]
148    progress: &'a Progress<ProgressBar>,
149}
150
151impl<State> FetchSrcRockBuilder<'_, State>
152where
153    State: fetch_src_rock_builder::State + fetch_src_rock_builder::IsComplete,
154{
155    pub async fn fetch(self) -> Result<RemotePackageSourceMetadata, FetchSrcRockError> {
156        do_fetch_src_rock(self._build()).await
157    }
158}
159
160#[derive(Error, Debug)]
161#[error(transparent)]
162pub enum FetchSrcRockError {
163    DownloadSrcRock(#[from] DownloadSrcRockError),
164    Unpack(#[from] UnpackError),
165    Io(#[from] io::Error),
166}
167
168/// A no-prompt implementer for auth_git2's prompter
169#[derive(Copy, Clone, Debug)]
170struct NullPrompter;
171
172impl Prompter for NullPrompter {
173    fn prompt_username_password(&mut self, _: &str, _: &git2::Config) -> Option<(String, String)> {
174        None
175    }
176
177    fn prompt_password(&mut self, _: &str, _: &str, _: &git2::Config) -> Option<String> {
178        None
179    }
180
181    fn prompt_ssh_key_passphrase(&mut self, _: &Path, _: &git2::Config) -> Option<String> {
182        None
183    }
184}
185
186async fn do_fetch_src<R: Rockspec>(
187    fetch: &FetchSrc<'_, R>,
188) -> Result<RemotePackageSourceMetadata, FetchSrcError> {
189    let rockspec = fetch.rockspec;
190    let rock_source = rockspec.source().current_platform();
191    let progress = fetch.progress;
192    let dest_dir = fetch.dest_dir;
193    let config = fetch.config;
194    // prioritise lockfile source, if present
195    let mut source_spec = match &fetch.source_url {
196        Some(source_url) => match source_url {
197            RemotePackageSourceUrl::Git { url, checkout_ref } => RockSourceSpec::Git(GitSource {
198                url: url.parse()?,
199                checkout_ref: Some(checkout_ref.clone()),
200            }),
201            RemotePackageSourceUrl::Url { url } => RockSourceSpec::Url(url.clone()),
202            RemotePackageSourceUrl::File { path } => RockSourceSpec::File(path.clone()),
203        },
204        None => rock_source.source_spec.clone(),
205    };
206    if let Some(vendor_dir) = config.vendor_dir() {
207        source_spec = match source_spec {
208            // could be a project directory (not vendored) or a local source
209            // or a vendored dependency that we have already resolved
210            RockSourceSpec::File(_) => source_spec,
211            _ => {
212                let pkg_vendor_dir =
213                    vendor_dir.join(format!("{}@{}", rockspec.package(), rockspec.version()));
214                RockSourceSpec::File(pkg_vendor_dir)
215            }
216        }
217    }
218    let metadata = match &source_spec {
219        RockSourceSpec::Git(git) => {
220            let url = git.url.to_string();
221            progress.map(|p| p.set_message(format!("🦠 Cloning {url}")));
222
223            let auth = if config.no_prompt() {
224                GitAuthenticator::default()
225                    .try_password_prompt(0)
226                    .prompt_ssh_key_password(false)
227                    .set_prompter(NullPrompter)
228            } else {
229                GitAuthenticator::default()
230            };
231            let git_config = git2::Config::open_default()?;
232            let mut callbacks = RemoteCallbacks::new();
233            callbacks.credentials(auth.credentials(&git_config));
234            let mut fetch_options = FetchOptions::new();
235            fetch_options.update_fetchhead(false);
236            fetch_options.remote_callbacks(callbacks);
237            if git.checkout_ref.is_none() {
238                fetch_options.depth(1);
239            };
240            let mut repo_builder = RepoBuilder::new();
241            repo_builder.fetch_options(fetch_options);
242            let repo = repo_builder.clone(&url, dest_dir)?;
243
244            let checkout_ref = match &git.checkout_ref {
245                Some(checkout_ref) => {
246                    let (object, _) = repo.revparse_ext(checkout_ref)?;
247                    repo.checkout_tree(&object, None)?;
248                    checkout_ref.clone()
249                }
250                None => {
251                    let head = repo.head()?;
252                    let commit = head.peel_to_commit()?;
253                    commit.id().to_string()
254                }
255            };
256            // The .git directory is not deterministic
257            remove_dir_all(dest_dir.join(".git")).map_err(FetchSrcError::CleanGitDir)?;
258            let hash = fetch.dest_dir.hash().map_err(FetchSrcError::Hash)?;
259            RemotePackageSourceMetadata {
260                hash,
261                source_url: RemotePackageSourceUrl::Git { url, checkout_ref },
262            }
263        }
264        RockSourceSpec::Url(url) => {
265            progress.map(|p| p.set_message(format!("📥 Downloading {}", url.to_owned())));
266
267            // NOTE: We don't enforce HTTPS when fetching sources because some rockspecs
268            // have HTTP URLs in `source.url`.
269            let response = crate::reqwest::new_http_client(config)?
270                .get(url.clone())
271                .send()
272                .await?
273                .error_for_status()?
274                .bytes()
275                .await?;
276            let hash = response.hash().map_err(FetchSrcError::Hash)?;
277            let file_name = url
278                .path_segments()
279                .and_then(|mut segments| segments.next_back())
280                .and_then(|name| {
281                    if name.is_empty() {
282                        None
283                    } else {
284                        Some(name.to_string())
285                    }
286                })
287                .unwrap_or(url.to_string());
288            let cursor = Cursor::new(response);
289            let mime_type = infer::get(cursor.get_ref()).map(|file_type| file_type.mime_type());
290            operations::unpack::unpack(
291                mime_type,
292                cursor,
293                rock_source.unpack_dir.is_none(),
294                file_name,
295                dest_dir,
296                progress,
297            )
298            .await?;
299            RemotePackageSourceMetadata {
300                hash,
301                source_url: RemotePackageSourceUrl::Url { url: url.clone() },
302            }
303        }
304        RockSourceSpec::File(path) => {
305            let hash = if path.is_dir() {
306                progress.map(|p| p.set_message(format!("📋 Copying {}", path.display())));
307                recursive_copy_dir(&path.to_path_buf(), dest_dir)
308                    .await
309                    .map_err(|err| FetchSrcError::CopyDir {
310                        src: path.to_path_buf(),
311                        dest: dest_dir.to_path_buf(),
312                        err,
313                    })?;
314                progress.map(|p| p.finish_and_clear());
315                dest_dir.hash().map_err(FetchSrcError::Hash)?
316            } else {
317                let mut file = File::open(path).map_err(|err| FetchSrcError::FileOpen {
318                    file: path.clone(),
319                    err,
320                })?;
321                let mut buffer = Vec::new();
322                file.read_to_end(&mut buffer)
323                    .map_err(|err| FetchSrcError::FileRead {
324                        file: path.clone(),
325                        err,
326                    })?;
327                let mime_type = infer::get(&buffer).map(|file_type| file_type.mime_type());
328                let file_name = path
329                    .file_name()
330                    .map(|os_str| os_str.to_string_lossy())
331                    .unwrap_or(path.to_string_lossy())
332                    .to_string();
333                operations::unpack::unpack(
334                    mime_type,
335                    file,
336                    rock_source.unpack_dir.is_none(),
337                    file_name,
338                    dest_dir,
339                    progress,
340                )
341                .await?;
342                path.hash().map_err(FetchSrcError::Hash)?
343            };
344            RemotePackageSourceMetadata {
345                hash,
346                source_url: RemotePackageSourceUrl::File { path: path.clone() },
347            }
348        }
349    };
350    Ok(metadata)
351}
352
353async fn do_fetch_src_rock(
354    fetch: FetchSrcRock<'_>,
355) -> Result<RemotePackageSourceMetadata, FetchSrcRockError> {
356    let package = fetch.package;
357    let dest_dir = fetch.dest_dir;
358    let config = fetch.config;
359    let progress = fetch.progress;
360    let src_rock =
361        operations::download_src_rock(package, config.server(), progress, fetch.config).await?;
362    let hash = src_rock.bytes.hash()?;
363    let cursor = Cursor::new(src_rock.bytes);
364    let mime_type = infer::get(cursor.get_ref()).map(|file_type| file_type.mime_type());
365    operations::unpack::unpack(
366        mime_type,
367        cursor,
368        true,
369        src_rock.file_name,
370        dest_dir,
371        progress,
372    )
373    .await?;
374    Ok(RemotePackageSourceMetadata {
375        hash,
376        source_url: RemotePackageSourceUrl::Url { url: src_rock.url },
377    })
378}
lux_lib/operations/fetch.rs

lux_lib/operations/
fetch.rs
