lux_lib/operations/
fetch.rs

1use bon::Builder;
2use git2::build::RepoBuilder;
3use git2::{Cred, FetchOptions, RemoteCallbacks};
4use remove_dir_all::remove_dir_all;
5use ssri::Integrity;
6use std::fs::File;
7use std::io;
8use std::io::Cursor;
9use std::io::Read;
10use std::path::Path;
11use std::path::PathBuf;
12use thiserror::Error;
13
14use crate::build::utils::recursive_copy_dir;
15use crate::config::Config;
16use crate::git::url::RemoteGitUrlParseError;
17use crate::git::GitSource;
18use crate::hash::HasIntegrity;
19use crate::lockfile::RemotePackageSourceUrl;
20use crate::lua_rockspec::RockSourceSpec;
21use crate::operations;
22use crate::package::PackageSpec;
23use crate::progress::Progress;
24use crate::progress::ProgressBar;
25use crate::rockspec::Rockspec;
26
27use super::DownloadSrcRockError;
28use super::UnpackError;
29
30/// A rocks package source fetcher, providing fine-grained control
31/// over how a package should be fetched.
32#[derive(Builder)]
33#[builder(start_fn = new, finish_fn(name = _build, vis = ""))]
34pub struct FetchSrc<'a, R: Rockspec> {
35    #[builder(start_fn)]
36    dest_dir: &'a Path,
37    #[builder(start_fn)]
38    rockspec: &'a R,
39    #[builder(start_fn)]
40    config: &'a Config,
41    #[builder(start_fn)]
42    progress: &'a Progress<ProgressBar>,
43    #[builder(setters(vis = "pub(crate)"))]
44    source_url: Option<RemotePackageSourceUrl>,
45}
46
47#[derive(Debug)]
48pub(crate) struct RemotePackageSourceMetadata {
49    pub hash: Integrity,
50    pub source_url: RemotePackageSourceUrl,
51}
52
53impl<R: Rockspec, State> FetchSrcBuilder<'_, R, State>
54where
55    State: fetch_src_builder::State + fetch_src_builder::IsComplete,
56{
57    /// Fetch and unpack the source into the `dest_dir`.
58    pub async fn fetch(self) -> Result<(), FetchSrcError> {
59        self.fetch_internal().await?;
60        Ok(())
61    }
62
63    /// Fetch and unpack the source into the `dest_dir`,
64    /// returning the source `Integrity`.
65    pub(crate) async fn fetch_internal(self) -> Result<RemotePackageSourceMetadata, FetchSrcError> {
66        let fetch = self._build();
67        match do_fetch_src(&fetch).await {
68            Err(err)
69                if fetch
70                    .source_url
71                    .is_some_and(|url| matches!(url, RemotePackageSourceUrl::File { .. })) =>
72            {
73                // Don't fall back to downloading .src.rock archives if a local source was specified.
74                Err(err)
75            }
76            Err(err) => match &fetch.rockspec.source().current_platform().source_spec {
77                RockSourceSpec::Git(_) | RockSourceSpec::Url(_) => {
78                    let package = PackageSpec::new(
79                        fetch.rockspec.package().clone(),
80                        fetch.rockspec.version().clone(),
81                    );
82                    fetch.progress.map(|p| {
83                        p.println(format!(
84                            "⚠️ WARNING: Failed to fetch source for {}: {}",
85                            &package, err
86                        ))
87                    });
88                    fetch.progress.map(|p| {
89                        p.println(format!(
90                            "⚠️ Falling back to searching for a .src.rock archive on {}",
91                            fetch.config.server()
92                        ))
93                    });
94                    let metadata =
95                        FetchSrcRock::new(&package, fetch.dest_dir, fetch.config, fetch.progress)
96                            .fetch()
97                            .await?;
98                    Ok(metadata)
99                }
100                RockSourceSpec::File(_) => Err(err),
101            },
102            Ok(metadata) => Ok(metadata),
103        }
104    }
105}
106
107#[derive(Error, Debug)]
108pub enum FetchSrcError {
109    #[error("failed to clone rock source:\n{0}")]
110    GitClone(#[from] git2::Error),
111    #[error("failed to parse git URL:\n{0}")]
112    GitUrlParse(#[from] RemoteGitUrlParseError),
113    #[error(transparent)]
114    Request(#[from] reqwest::Error),
115    #[error(transparent)]
116    Unpack(#[from] UnpackError),
117    #[error(transparent)]
118    FetchSrcRock(#[from] FetchSrcRockError),
119    #[error("unable to remove the '.git' directory:\n{0}")]
120    CleanGitDir(io::Error),
121    #[error("unable to compute hash:\n{0}")]
122    Hash(io::Error),
123    #[error("unable to copy {src} to {dest}:\n{err}")]
124    CopyDir {
125        src: PathBuf,
126        dest: PathBuf,
127        err: io::Error,
128    },
129    #[error("unable to open {file}:\n{err}")]
130    FileOpen { file: PathBuf, err: io::Error },
131    #[error("unable to read {file}:\n{err}")]
132    FileRead { file: PathBuf, err: io::Error },
133}
134
135/// A rocks package source fetcher, providing fine-grained control
136/// over how a package should be fetched.
137#[derive(Builder)]
138#[builder(start_fn = new, finish_fn(name = _build, vis = ""))]
139struct FetchSrcRock<'a> {
140    #[builder(start_fn)]
141    package: &'a PackageSpec,
142    #[builder(start_fn)]
143    dest_dir: &'a Path,
144    #[builder(start_fn)]
145    config: &'a Config,
146    #[builder(start_fn)]
147    progress: &'a Progress<ProgressBar>,
148}
149
150impl<State> FetchSrcRockBuilder<'_, State>
151where
152    State: fetch_src_rock_builder::State + fetch_src_rock_builder::IsComplete,
153{
154    pub async fn fetch(self) -> Result<RemotePackageSourceMetadata, FetchSrcRockError> {
155        do_fetch_src_rock(self._build()).await
156    }
157}
158
159#[derive(Error, Debug)]
160#[error(transparent)]
161pub enum FetchSrcRockError {
162    DownloadSrcRock(#[from] DownloadSrcRockError),
163    Unpack(#[from] UnpackError),
164    Io(#[from] io::Error),
165}
166
167async fn do_fetch_src<R: Rockspec>(
168    fetch: &FetchSrc<'_, R>,
169) -> Result<RemotePackageSourceMetadata, FetchSrcError> {
170    let rockspec = fetch.rockspec;
171    let rock_source = rockspec.source().current_platform();
172    let progress = fetch.progress;
173    let dest_dir = fetch.dest_dir;
174    let config = fetch.config;
175    // prioritise lockfile source, if present
176    let mut source_spec = match &fetch.source_url {
177        Some(source_url) => match source_url {
178            RemotePackageSourceUrl::Git { url, checkout_ref } => RockSourceSpec::Git(GitSource {
179                url: url.parse()?,
180                checkout_ref: Some(checkout_ref.clone()),
181            }),
182            RemotePackageSourceUrl::Url { url } => RockSourceSpec::Url(url.clone()),
183            RemotePackageSourceUrl::File { path } => RockSourceSpec::File(path.clone()),
184        },
185        None => rock_source.source_spec.clone(),
186    };
187    if let Some(vendor_dir) = config.vendor_dir() {
188        source_spec = match source_spec {
189            // could be a project directory (not vendored) or a local source
190            // or a vendored dependency that we have already resolved
191            RockSourceSpec::File(_) => source_spec,
192            _ => {
193                let pkg_vendor_dir =
194                    vendor_dir.join(format!("{}@{}", rockspec.package(), rockspec.version()));
195                RockSourceSpec::File(pkg_vendor_dir)
196            }
197        }
198    }
199    let metadata = match &source_spec {
200        RockSourceSpec::Git(git) => {
201            let url = git.url.to_string();
202            progress.map(|p| p.set_message(format!("🦠 Cloning {url}")));
203
204            let mut callbacks = RemoteCallbacks::new();
205            callbacks.credentials(|_url, username_from_url, _allowed_types| {
206                Cred::ssh_key_from_agent(username_from_url.unwrap_or("git"))
207            });
208            let mut fetch_options = FetchOptions::new();
209            fetch_options.update_fetchhead(false);
210            fetch_options.remote_callbacks(callbacks);
211            if git.checkout_ref.is_none() {
212                fetch_options.depth(1);
213            };
214            let mut repo_builder = RepoBuilder::new();
215            repo_builder.fetch_options(fetch_options);
216            let repo = repo_builder.clone(&url, dest_dir)?;
217
218            let checkout_ref = match &git.checkout_ref {
219                Some(checkout_ref) => {
220                    let (object, _) = repo.revparse_ext(checkout_ref)?;
221                    repo.checkout_tree(&object, None)?;
222                    checkout_ref.clone()
223                }
224                None => {
225                    let head = repo.head()?;
226                    let commit = head.peel_to_commit()?;
227                    commit.id().to_string()
228                }
229            };
230            // The .git directory is not deterministic
231            remove_dir_all(dest_dir.join(".git")).map_err(FetchSrcError::CleanGitDir)?;
232            let hash = fetch.dest_dir.hash().map_err(FetchSrcError::Hash)?;
233            RemotePackageSourceMetadata {
234                hash,
235                source_url: RemotePackageSourceUrl::Git { url, checkout_ref },
236            }
237        }
238        RockSourceSpec::Url(url) => {
239            progress.map(|p| p.set_message(format!("📥 Downloading {}", url.to_owned())));
240
241            // NOTE: We don't enforce HTTPS when fetching sources because some rockspecs
242            // have HTTP URLs in `source.url`.
243            let response = crate::reqwest::new_http_client(config)?
244                .get(url.clone())
245                .send()
246                .await?
247                .error_for_status()?
248                .bytes()
249                .await?;
250            let hash = response.hash().map_err(FetchSrcError::Hash)?;
251            let file_name = url
252                .path_segments()
253                .and_then(|mut segments| segments.next_back())
254                .and_then(|name| {
255                    if name.is_empty() {
256                        None
257                    } else {
258                        Some(name.to_string())
259                    }
260                })
261                .unwrap_or(url.to_string());
262            let cursor = Cursor::new(response);
263            let mime_type = infer::get(cursor.get_ref()).map(|file_type| file_type.mime_type());
264            operations::unpack::unpack(
265                mime_type,
266                cursor,
267                rock_source.unpack_dir.is_none(),
268                file_name,
269                dest_dir,
270                progress,
271            )
272            .await?;
273            RemotePackageSourceMetadata {
274                hash,
275                source_url: RemotePackageSourceUrl::Url { url: url.clone() },
276            }
277        }
278        RockSourceSpec::File(path) => {
279            let hash = if path.is_dir() {
280                progress.map(|p| p.set_message(format!("📋 Copying {}", path.display())));
281                recursive_copy_dir(&path.to_path_buf(), dest_dir)
282                    .await
283                    .map_err(|err| FetchSrcError::CopyDir {
284                        src: path.to_path_buf(),
285                        dest: dest_dir.to_path_buf(),
286                        err,
287                    })?;
288                progress.map(|p| p.finish_and_clear());
289                dest_dir.hash().map_err(FetchSrcError::Hash)?
290            } else {
291                let mut file = File::open(path).map_err(|err| FetchSrcError::FileOpen {
292                    file: path.clone(),
293                    err,
294                })?;
295                let mut buffer = Vec::new();
296                file.read_to_end(&mut buffer)
297                    .map_err(|err| FetchSrcError::FileRead {
298                        file: path.clone(),
299                        err,
300                    })?;
301                let mime_type = infer::get(&buffer).map(|file_type| file_type.mime_type());
302                let file_name = path
303                    .file_name()
304                    .map(|os_str| os_str.to_string_lossy())
305                    .unwrap_or(path.to_string_lossy())
306                    .to_string();
307                operations::unpack::unpack(
308                    mime_type,
309                    file,
310                    rock_source.unpack_dir.is_none(),
311                    file_name,
312                    dest_dir,
313                    progress,
314                )
315                .await?;
316                path.hash().map_err(FetchSrcError::Hash)?
317            };
318            RemotePackageSourceMetadata {
319                hash,
320                source_url: RemotePackageSourceUrl::File { path: path.clone() },
321            }
322        }
323    };
324    Ok(metadata)
325}
326
327async fn do_fetch_src_rock(
328    fetch: FetchSrcRock<'_>,
329) -> Result<RemotePackageSourceMetadata, FetchSrcRockError> {
330    let package = fetch.package;
331    let dest_dir = fetch.dest_dir;
332    let config = fetch.config;
333    let progress = fetch.progress;
334    let src_rock =
335        operations::download_src_rock(package, config.server(), progress, fetch.config).await?;
336    let hash = src_rock.bytes.hash()?;
337    let cursor = Cursor::new(src_rock.bytes);
338    let mime_type = infer::get(cursor.get_ref()).map(|file_type| file_type.mime_type());
339    operations::unpack::unpack(
340        mime_type,
341        cursor,
342        true,
343        src_rock.file_name,
344        dest_dir,
345        progress,
346    )
347    .await?;
348    Ok(RemotePackageSourceMetadata {
349        hash,
350        source_url: RemotePackageSourceUrl::Url { url: src_rock.url },
351    })
352}
lux_lib/operations/fetch.rs

lux_lib/operations/
fetch.rs
