lux_lib/operations/
fetch.rs

1use bon::Builder;
2use git2::build::RepoBuilder;
3use git2::{Cred, FetchOptions, RemoteCallbacks};
4use remove_dir_all::remove_dir_all;
5use ssri::Integrity;
6use std::fs::File;
7use std::io;
8use std::io::Cursor;
9use std::io::Read;
10use std::path::Path;
11use std::path::PathBuf;
12use thiserror::Error;
13
14use crate::build::utils::recursive_copy_dir;
15use crate::config::Config;
16use crate::git::url::RemoteGitUrlParseError;
17use crate::git::GitSource;
18use crate::hash::HasIntegrity;
19use crate::lockfile::RemotePackageSourceUrl;
20use crate::lua_rockspec::RockSourceSpec;
21use crate::operations;
22use crate::package::PackageSpec;
23use crate::progress::Progress;
24use crate::progress::ProgressBar;
25use crate::rockspec::Rockspec;
26
27use super::DownloadSrcRockError;
28use super::UnpackError;
29
30/// A rocks package source fetcher, providing fine-grained control
31/// over how a package should be fetched.
32#[derive(Builder)]
33#[builder(start_fn = new, finish_fn(name = _build, vis = ""))]
34pub struct FetchSrc<'a, R: Rockspec> {
35    #[builder(start_fn)]
36    dest_dir: &'a Path,
37    #[builder(start_fn)]
38    rockspec: &'a R,
39    #[builder(start_fn)]
40    config: &'a Config,
41    #[builder(start_fn)]
42    progress: &'a Progress<ProgressBar>,
43    #[builder(setters(vis = "pub(crate)"))]
44    source_url: Option<RemotePackageSourceUrl>,
45}
46
47#[derive(Debug)]
48pub(crate) struct RemotePackageSourceMetadata {
49    pub hash: Integrity,
50    pub source_url: RemotePackageSourceUrl,
51}
52
53impl<R: Rockspec, State> FetchSrcBuilder<'_, R, State>
54where
55    State: fetch_src_builder::State + fetch_src_builder::IsComplete,
56{
57    /// Fetch and unpack the source into the `dest_dir`.
58    pub async fn fetch(self) -> Result<(), FetchSrcError> {
59        self.fetch_internal().await?;
60        Ok(())
61    }
62
63    /// Fetch and unpack the source into the `dest_dir`,
64    /// returning the source `Integrity`.
65    pub(crate) async fn fetch_internal(self) -> Result<RemotePackageSourceMetadata, FetchSrcError> {
66        let fetch = self._build();
67        match do_fetch_src(&fetch).await {
68            Err(err)
69                if fetch
70                    .source_url
71                    .is_some_and(|url| matches!(url, RemotePackageSourceUrl::File { .. })) =>
72            {
73                // Don't fall back to downloading .src.rock archives if a local source was specified.
74                Err(err)
75            }
76            Err(err) => match &fetch.rockspec.source().current_platform().source_spec {
77                RockSourceSpec::Git(_) | RockSourceSpec::Url(_) => {
78                    let package = PackageSpec::new(
79                        fetch.rockspec.package().clone(),
80                        fetch.rockspec.version().clone(),
81                    );
82                    fetch.progress.map(|p| {
83                        p.println(format!(
84                            "⚠️ WARNING: Failed to fetch source for {}: {}",
85                            &package, err
86                        ))
87                    });
88                    fetch.progress.map(|p| {
89                        p.println(format!(
90                            "⚠️ Falling back to searching for a .src.rock archive on {}",
91                            fetch.config.server()
92                        ))
93                    });
94                    let metadata =
95                        FetchSrcRock::new(&package, fetch.dest_dir, fetch.config, fetch.progress)
96                            .fetch()
97                            .await?;
98                    Ok(metadata)
99                }
100                RockSourceSpec::File(_) => Err(err),
101            },
102            Ok(metadata) => Ok(metadata),
103        }
104    }
105}
106
107#[derive(Error, Debug)]
108pub enum FetchSrcError {
109    #[error("failed to clone rock source:\n{0}")]
110    GitClone(#[from] git2::Error),
111    #[error("failed to parse git URL:\n{0}")]
112    GitUrlParse(#[from] RemoteGitUrlParseError),
113    #[error(transparent)]
114    Request(#[from] reqwest::Error),
115    #[error(transparent)]
116    Unpack(#[from] UnpackError),
117    #[error(transparent)]
118    FetchSrcRock(#[from] FetchSrcRockError),
119    #[error("unable to remove the '.git' directory:\n{0}")]
120    CleanGitDir(io::Error),
121    #[error("unable to compute hash:\n{0}")]
122    Hash(io::Error),
123    #[error("unable to copy {src} to {dest}:\n{err}")]
124    CopyDir {
125        src: PathBuf,
126        dest: PathBuf,
127        err: io::Error,
128    },
129    #[error("unable to open {file}:\n{err}")]
130    FileOpen { file: PathBuf, err: io::Error },
131    #[error("unable to read {file}:\n{err}")]
132    FileRead { file: PathBuf, err: io::Error },
133}
134
135/// A rocks package source fetcher, providing fine-grained control
136/// over how a package should be fetched.
137#[derive(Builder)]
138#[builder(start_fn = new, finish_fn(name = _build, vis = ""))]
139struct FetchSrcRock<'a> {
140    #[builder(start_fn)]
141    package: &'a PackageSpec,
142    #[builder(start_fn)]
143    dest_dir: &'a Path,
144    #[builder(start_fn)]
145    config: &'a Config,
146    #[builder(start_fn)]
147    progress: &'a Progress<ProgressBar>,
148}
149
150impl<State> FetchSrcRockBuilder<'_, State>
151where
152    State: fetch_src_rock_builder::State + fetch_src_rock_builder::IsComplete,
153{
154    pub async fn fetch(self) -> Result<RemotePackageSourceMetadata, FetchSrcRockError> {
155        do_fetch_src_rock(self._build()).await
156    }
157}
158
159#[derive(Error, Debug)]
160#[error(transparent)]
161pub enum FetchSrcRockError {
162    DownloadSrcRock(#[from] DownloadSrcRockError),
163    Unpack(#[from] UnpackError),
164    Io(#[from] io::Error),
165}
166
167async fn do_fetch_src<R: Rockspec>(
168    fetch: &FetchSrc<'_, R>,
169) -> Result<RemotePackageSourceMetadata, FetchSrcError> {
170    let rockspec = fetch.rockspec;
171    let rock_source = rockspec.source().current_platform();
172    let progress = fetch.progress;
173    let dest_dir = fetch.dest_dir;
174    let config = fetch.config;
175    // prioritise lockfile source, if present
176    let mut source_spec = match &fetch.source_url {
177        Some(source_url) => match source_url {
178            RemotePackageSourceUrl::Git { url, checkout_ref } => RockSourceSpec::Git(GitSource {
179                url: url.parse()?,
180                checkout_ref: Some(checkout_ref.clone()),
181            }),
182            RemotePackageSourceUrl::Url { url } => RockSourceSpec::Url(url.clone()),
183            RemotePackageSourceUrl::File { path } => RockSourceSpec::File(path.clone()),
184        },
185        None => rock_source.source_spec.clone(),
186    };
187    if let Some(vendor_dir) = config.vendor_dir() {
188        source_spec = match source_spec {
189            // could be a project directory (not vendored) or a local source
190            // or a vendored dependency that we have already resolved
191            RockSourceSpec::File(_) => source_spec,
192            _ => {
193                let pkg_vendor_dir =
194                    vendor_dir.join(format!("{}@{}", rockspec.package(), rockspec.version()));
195                RockSourceSpec::File(pkg_vendor_dir)
196            }
197        }
198    }
199    let metadata = match &source_spec {
200        RockSourceSpec::Git(git) => {
201            let url = git.url.to_string();
202            progress.map(|p| p.set_message(format!("🦠 Cloning {url}")));
203
204            let mut callbacks = RemoteCallbacks::new();
205            callbacks.credentials(|_url, username_from_url, _allowed_types| {
206                Cred::ssh_key_from_agent(username_from_url.unwrap_or("git"))
207            });
208            let mut fetch_options = FetchOptions::new();
209            fetch_options.update_fetchhead(false);
210            fetch_options.remote_callbacks(callbacks);
211            if git.checkout_ref.is_none() {
212                fetch_options.depth(1);
213            };
214            let mut repo_builder = RepoBuilder::new();
215            repo_builder.fetch_options(fetch_options);
216            let repo = repo_builder.clone(&url, dest_dir)?;
217
218            let checkout_ref = match &git.checkout_ref {
219                Some(checkout_ref) => {
220                    let (object, _) = repo.revparse_ext(checkout_ref)?;
221                    repo.checkout_tree(&object, None)?;
222                    checkout_ref.clone()
223                }
224                None => {
225                    let head = repo.head()?;
226                    let commit = head.peel_to_commit()?;
227                    commit.id().to_string()
228                }
229            };
230            // The .git directory is not deterministic
231            remove_dir_all(dest_dir.join(".git")).map_err(FetchSrcError::CleanGitDir)?;
232            let hash = fetch.dest_dir.hash().map_err(FetchSrcError::Hash)?;
233            RemotePackageSourceMetadata {
234                hash,
235                source_url: RemotePackageSourceUrl::Git { url, checkout_ref },
236            }
237        }
238        RockSourceSpec::Url(url) => {
239            progress.map(|p| p.set_message(format!("📥 Downloading {}", url.to_owned())));
240
241            let response = reqwest::Client::new()
242                .get(url.clone())
243                .send()
244                .await?
245                .error_for_status()?
246                .bytes()
247                .await?;
248            let hash = response.hash().map_err(FetchSrcError::Hash)?;
249            let file_name = url
250                .path_segments()
251                .and_then(|mut segments| segments.next_back())
252                .and_then(|name| {
253                    if name.is_empty() {
254                        None
255                    } else {
256                        Some(name.to_string())
257                    }
258                })
259                .unwrap_or(url.to_string());
260            let cursor = Cursor::new(response);
261            let mime_type = infer::get(cursor.get_ref()).map(|file_type| file_type.mime_type());
262            operations::unpack::unpack(
263                mime_type,
264                cursor,
265                rock_source.unpack_dir.is_none(),
266                file_name,
267                dest_dir,
268                progress,
269            )
270            .await?;
271            RemotePackageSourceMetadata {
272                hash,
273                source_url: RemotePackageSourceUrl::Url { url: url.clone() },
274            }
275        }
276        RockSourceSpec::File(path) => {
277            let hash = if path.is_dir() {
278                progress.map(|p| p.set_message(format!("📋 Copying {}", path.display())));
279                recursive_copy_dir(&path.to_path_buf(), dest_dir)
280                    .await
281                    .map_err(|err| FetchSrcError::CopyDir {
282                        src: path.to_path_buf(),
283                        dest: dest_dir.to_path_buf(),
284                        err,
285                    })?;
286                progress.map(|p| p.finish_and_clear());
287                dest_dir.hash().map_err(FetchSrcError::Hash)?
288            } else {
289                let mut file = File::open(path).map_err(|err| FetchSrcError::FileOpen {
290                    file: path.clone(),
291                    err,
292                })?;
293                let mut buffer = Vec::new();
294                file.read_to_end(&mut buffer)
295                    .map_err(|err| FetchSrcError::FileRead {
296                        file: path.clone(),
297                        err,
298                    })?;
299                let mime_type = infer::get(&buffer).map(|file_type| file_type.mime_type());
300                let file_name = path
301                    .file_name()
302                    .map(|os_str| os_str.to_string_lossy())
303                    .unwrap_or(path.to_string_lossy())
304                    .to_string();
305                operations::unpack::unpack(
306                    mime_type,
307                    file,
308                    rock_source.unpack_dir.is_none(),
309                    file_name,
310                    dest_dir,
311                    progress,
312                )
313                .await?;
314                path.hash().map_err(FetchSrcError::Hash)?
315            };
316            RemotePackageSourceMetadata {
317                hash,
318                source_url: RemotePackageSourceUrl::File { path: path.clone() },
319            }
320        }
321    };
322    Ok(metadata)
323}
324
325async fn do_fetch_src_rock(
326    fetch: FetchSrcRock<'_>,
327) -> Result<RemotePackageSourceMetadata, FetchSrcRockError> {
328    let package = fetch.package;
329    let dest_dir = fetch.dest_dir;
330    let config = fetch.config;
331    let progress = fetch.progress;
332    let src_rock = operations::download_src_rock(package, config.server(), progress).await?;
333    let hash = src_rock.bytes.hash()?;
334    let cursor = Cursor::new(src_rock.bytes);
335    let mime_type = infer::get(cursor.get_ref()).map(|file_type| file_type.mime_type());
336    operations::unpack::unpack(
337        mime_type,
338        cursor,
339        true,
340        src_rock.file_name,
341        dest_dir,
342        progress,
343    )
344    .await?;
345    Ok(RemotePackageSourceMetadata {
346        hash,
347        source_url: RemotePackageSourceUrl::Url { url: src_rock.url },
348    })
349}
lux_lib/operations/fetch.rs

lux_lib/operations/
fetch.rs
