lux_lib/operations/
fetch.rs

1use crate::build::utils::recursive_copy_dir;
2use crate::config::Config;
3use crate::git::url::RemoteGitUrlParseError;
4use crate::git::GitSource;
5use crate::hash::HasIntegrity;
6use crate::lockfile::RemotePackageSourceUrl;
7use crate::lua_rockspec::RockSourceSpec;
8use crate::operations;
9use crate::package::PackageSpec;
10use crate::rockspec::Rockspec;
11use auth_git2::{GitAuthenticator, Prompter};
12use bon::Builder;
13use git2::build::RepoBuilder;
14use git2::{FetchOptions, RemoteCallbacks};
15use miette::Diagnostic;
16use remove_dir_all::remove_dir_all;
17use ssri::Integrity;
18use std::fs::File;
19use std::io;
20use std::io::Cursor;
21use std::io::Read;
22use std::path::Path;
23use std::path::PathBuf;
24use thiserror::Error;
25use tracing::span;
26
27use super::DownloadSrcRockError;
28use super::UnpackError;
29
30/// A rocks package source fetcher, providing fine-grained control
31/// over how a package should be fetched.
32#[derive(Builder)]
33#[builder(start_fn = new, finish_fn(name = _build, vis = ""))]
34pub struct FetchSrc<'a, R: Rockspec> {
35    #[builder(start_fn)]
36    dest_dir: &'a Path,
37    #[builder(start_fn)]
38    rockspec: &'a R,
39    #[builder(start_fn)]
40    config: &'a Config,
41    #[builder(setters(vis = "pub(crate)"))]
42    source_url: Option<RemotePackageSourceUrl>,
43}
44
45#[derive(Debug)]
46pub(crate) struct RemotePackageSourceMetadata {
47    pub hash: Integrity,
48    pub source_url: RemotePackageSourceUrl,
49}
50
51impl<R: Rockspec, State> FetchSrcBuilder<'_, R, State>
52where
53    State: fetch_src_builder::State + fetch_src_builder::IsComplete,
54{
55    /// Fetch and unpack the source into the `dest_dir`.
56    pub async fn fetch(self) -> Result<(), FetchSrcError> {
57        self.fetch_internal().await?;
58        Ok(())
59    }
60
61    /// Fetch and unpack the source into the `dest_dir`,
62    /// returning the source `Integrity`.
63    pub(crate) async fn fetch_internal(self) -> Result<RemotePackageSourceMetadata, FetchSrcError> {
64        let fetch = self._build();
65        match do_fetch_src(&fetch).await {
66            Err(err)
67                if fetch
68                    .source_url
69                    .is_some_and(|url| matches!(url, RemotePackageSourceUrl::File { .. })) =>
70            {
71                // Don't fall back to downloading .src.rock archives if a local source was specified.
72                Err(err)
73            }
74            Err(err) => match &fetch.rockspec.source().current_platform().source_spec {
75                RockSourceSpec::Git(_) | RockSourceSpec::Url(_) => {
76                    let package = PackageSpec::new(
77                        fetch.rockspec.package().clone(),
78                        fetch.rockspec.version().clone(),
79                    );
80                    let metadata = FetchSrcRock::new(&package, fetch.dest_dir, fetch.config)
81                        .fetch()
82                        .await?;
83                    Ok(metadata)
84                }
85                RockSourceSpec::File(_) => Err(err),
86            },
87            Ok(metadata) => Ok(metadata),
88        }
89    }
90}
91
92#[derive(Error, Debug, Diagnostic)]
93pub enum FetchSrcError {
94    #[error("failed to clone rock source:\n{0}")]
95    GitClone(#[from] git2::Error),
96    #[error("failed to parse git URL:\n{0}")]
97    #[diagnostic(forward(0))]
98    GitUrlParse(#[from] RemoteGitUrlParseError),
99    #[error(transparent)]
100    Request(#[from] reqwest::Error),
101    #[error(transparent)]
102    #[diagnostic(transparent)]
103    Unpack(#[from] UnpackError),
104    #[error(transparent)]
105    #[diagnostic(transparent)]
106    FetchSrcRock(#[from] FetchSrcRockError),
107    #[error("unable to remove the '.git' directory:\n{0}")]
108    CleanGitDir(io::Error),
109    #[error("unable to compute hash:\n{0}")]
110    Hash(io::Error),
111    #[error("unable to copy {src} to {dest}:\n{err}")]
112    CopyDir {
113        src: PathBuf,
114        dest: PathBuf,
115        err: io::Error,
116    },
117    #[error("unable to open {file}:\n{err}")]
118    FileOpen { file: PathBuf, err: io::Error },
119    #[error("unable to read {file}:\n{err}")]
120    FileRead { file: PathBuf, err: io::Error },
121}
122
123/// A rocks package source fetcher, providing fine-grained control
124/// over how a package should be fetched.
125#[derive(Builder)]
126#[builder(start_fn = new, finish_fn(name = _build, vis = ""))]
127struct FetchSrcRock<'a> {
128    #[builder(start_fn)]
129    package: &'a PackageSpec,
130    #[builder(start_fn)]
131    dest_dir: &'a Path,
132    #[builder(start_fn)]
133    config: &'a Config,
134}
135
136impl<State> FetchSrcRockBuilder<'_, State>
137where
138    State: fetch_src_rock_builder::State + fetch_src_rock_builder::IsComplete,
139{
140    pub async fn fetch(self) -> Result<RemotePackageSourceMetadata, FetchSrcRockError> {
141        do_fetch_src_rock(self._build()).await
142    }
143}
144
145#[derive(Error, Debug, Diagnostic)]
146#[error(transparent)]
147pub enum FetchSrcRockError {
148    DownloadSrcRock(#[from] DownloadSrcRockError),
149    Unpack(#[from] UnpackError),
150    Io(#[from] io::Error),
151}
152
153/// A no-prompt implementer for auth_git2's prompter
154#[derive(Copy, Clone, Debug)]
155struct NullPrompter;
156
157impl Prompter for NullPrompter {
158    fn prompt_username_password(&mut self, _: &str, _: &git2::Config) -> Option<(String, String)> {
159        None
160    }
161
162    fn prompt_password(&mut self, _: &str, _: &str, _: &git2::Config) -> Option<String> {
163        None
164    }
165
166    fn prompt_ssh_key_passphrase(&mut self, _: &Path, _: &git2::Config) -> Option<String> {
167        None
168    }
169}
170
171async fn do_fetch_src<R: Rockspec>(
172    fetch: &FetchSrc<'_, R>,
173) -> Result<RemotePackageSourceMetadata, FetchSrcError> {
174    let rockspec = fetch.rockspec;
175    let rock_source = rockspec.source().current_platform();
176    let dest_dir = fetch.dest_dir;
177    let config = fetch.config;
178    // prioritise lockfile source, if present
179    let mut source_spec = match &fetch.source_url {
180        Some(source_url) => match source_url {
181            RemotePackageSourceUrl::Git { url, checkout_ref } => RockSourceSpec::Git(GitSource {
182                url: url.parse()?,
183                checkout_ref: Some(checkout_ref.clone()),
184            }),
185            RemotePackageSourceUrl::Url { url } => RockSourceSpec::Url(url.clone()),
186            RemotePackageSourceUrl::File { path } => RockSourceSpec::File(path.clone()),
187        },
188        None => rock_source.source_spec.clone(),
189    };
190    let span = span!(
191        tracing::Level::INFO,
192        "Fetching source",
193        location = source_spec.to_string(),
194    );
195    let _enter = span.enter();
196
197    if let Some(vendor_dir) = config.vendor_dir() {
198        source_spec = match source_spec {
199            // could be a project directory (not vendored) or a local source
200            // or a vendored dependency that we have already resolved
201            RockSourceSpec::File(_) => source_spec,
202            _ => {
203                let pkg_vendor_dir =
204                    vendor_dir.join(format!("{}@{}", rockspec.package(), rockspec.version()));
205                RockSourceSpec::File(pkg_vendor_dir)
206            }
207        }
208    }
209    let metadata = match &source_spec {
210        RockSourceSpec::Git(git) => {
211            let url = git.url.to_string();
212            tracing::debug!(message = format!("Cloning {url}").as_str());
213
214            let auth = if config.no_prompt() {
215                GitAuthenticator::default()
216                    .try_password_prompt(0)
217                    .prompt_ssh_key_password(false)
218                    .set_prompter(NullPrompter)
219            } else {
220                GitAuthenticator::default()
221            };
222            let git_config = git2::Config::open_default()?;
223            let mut callbacks = RemoteCallbacks::new();
224            callbacks.credentials(auth.credentials(&git_config));
225            let mut fetch_options = FetchOptions::new();
226            fetch_options.update_fetchhead(false);
227            fetch_options.remote_callbacks(callbacks);
228            if git.checkout_ref.is_none() {
229                fetch_options.depth(1);
230            };
231            let mut repo_builder = RepoBuilder::new();
232            repo_builder.fetch_options(fetch_options);
233            let repo = repo_builder.clone(&url, dest_dir)?;
234
235            let checkout_ref = match &git.checkout_ref {
236                Some(checkout_ref) => {
237                    let (object, _) = repo.revparse_ext(checkout_ref)?;
238                    repo.checkout_tree(&object, None)?;
239                    checkout_ref.clone()
240                }
241                None => {
242                    let head = repo.head()?;
243                    let commit = head.peel_to_commit()?;
244                    commit.id().to_string()
245                }
246            };
247            // The .git directory is not deterministic
248            remove_dir_all(dest_dir.join(".git")).map_err(FetchSrcError::CleanGitDir)?;
249            let hash = fetch.dest_dir.hash().map_err(FetchSrcError::Hash)?;
250            RemotePackageSourceMetadata {
251                hash,
252                source_url: RemotePackageSourceUrl::Git { url, checkout_ref },
253            }
254        }
255        RockSourceSpec::Url(url) => {
256            tracing::debug!(message = format!("📥 Downloading {url}").as_str());
257
258            // NOTE: We don't enforce HTTPS when fetching sources because some rockspecs
259            // have HTTP URLs in `source.url`.
260            let response = crate::reqwest::new_http_client(config)?
261                .get(url.clone())
262                .send()
263                .await?
264                .error_for_status()?
265                .bytes()
266                .await?;
267            let hash = response.hash().map_err(FetchSrcError::Hash)?;
268            let file_name = url
269                .path_segments()
270                .and_then(|mut segments| segments.next_back())
271                .and_then(|name| {
272                    if name.is_empty() {
273                        None
274                    } else {
275                        Some(name.to_string())
276                    }
277                })
278                .unwrap_or(url.to_string());
279            let cursor = Cursor::new(response);
280            let mime_type = infer::get(cursor.get_ref()).map(|file_type| file_type.mime_type());
281            operations::unpack::unpack(
282                mime_type,
283                cursor,
284                rock_source.unpack_dir.is_none(),
285                file_name,
286                dest_dir,
287            )
288            .await?;
289            RemotePackageSourceMetadata {
290                hash,
291                source_url: RemotePackageSourceUrl::Url { url: url.clone() },
292            }
293        }
294        RockSourceSpec::File(path) => {
295            tracing::debug!(message = format!("📋 Copying {}", path.display()).as_str());
296
297            let hash = if path.is_dir() {
298                recursive_copy_dir(&path.to_path_buf(), dest_dir)
299                    .await
300                    .map_err(|err| FetchSrcError::CopyDir {
301                        src: path.to_path_buf(),
302                        dest: dest_dir.to_path_buf(),
303                        err,
304                    })?;
305                dest_dir.hash().map_err(FetchSrcError::Hash)?
306            } else {
307                let mut file = File::open(path).map_err(|err| FetchSrcError::FileOpen {
308                    file: path.clone(),
309                    err,
310                })?;
311                let mut buffer = Vec::new();
312                file.read_to_end(&mut buffer)
313                    .map_err(|err| FetchSrcError::FileRead {
314                        file: path.clone(),
315                        err,
316                    })?;
317                let mime_type = infer::get(&buffer).map(|file_type| file_type.mime_type());
318                let file_name = path
319                    .file_name()
320                    .map(|os_str| os_str.to_string_lossy())
321                    .unwrap_or(path.to_string_lossy())
322                    .to_string();
323                operations::unpack::unpack(
324                    mime_type,
325                    file,
326                    rock_source.unpack_dir.is_none(),
327                    file_name,
328                    dest_dir,
329                )
330                .await?;
331                path.hash().map_err(FetchSrcError::Hash)?
332            };
333            RemotePackageSourceMetadata {
334                hash,
335                source_url: RemotePackageSourceUrl::File { path: path.clone() },
336            }
337        }
338    };
339    Ok(metadata)
340}
341
342async fn do_fetch_src_rock(
343    fetch: FetchSrcRock<'_>,
344) -> Result<RemotePackageSourceMetadata, FetchSrcRockError> {
345    let package = fetch.package;
346    let span = span!(
347        tracing::Level::INFO,
348        "Fetching src.rock",
349        package = package.to_string(),
350    );
351    let _enter = span.enter();
352
353    let dest_dir = fetch.dest_dir;
354    let config = fetch.config;
355    let src_rock = operations::download_src_rock(package, config.server(), fetch.config).await?;
356    let hash = src_rock.bytes.hash()?;
357    let cursor = Cursor::new(src_rock.bytes);
358    let mime_type = infer::get(cursor.get_ref()).map(|file_type| file_type.mime_type());
359    operations::unpack::unpack(mime_type, cursor, true, src_rock.file_name, dest_dir).await?;
360    Ok(RemotePackageSourceMetadata {
361        hash,
362        source_url: RemotePackageSourceUrl::Url { url: src_rock.url },
363    })
364}
lux_lib/operations/fetch.rs

lux_lib/operations/
fetch.rs
