lux_lib/operations/
fetch.rs

1use crate::build::utils::recursive_copy_dir;
2use crate::config::Config;
3use crate::git::url::RemoteGitUrlParseError;
4use crate::git::GitSource;
5use crate::hash::HasIntegrity;
6use crate::lockfile::RemotePackageSourceUrl;
7use crate::lua_rockspec::RockSourceSpec;
8use crate::package::PackageSpec;
9use crate::rockspec::Rockspec;
10use crate::{fs, operations};
11use auth_git2::{GitAuthenticator, Prompter};
12use bon::Builder;
13use git2::build::RepoBuilder;
14use git2::{FetchOptions, RemoteCallbacks};
15use miette::Diagnostic;
16use remove_dir_all::remove_dir_all;
17use ssri::Integrity;
18use std::io;
19use std::io::Cursor;
20use std::io::Read;
21use std::path::Path;
22use thiserror::Error;
23use tracing::span;
24
25use super::DownloadSrcRockError;
26use super::UnpackError;
27
28/// A rocks package source fetcher, providing fine-grained control
29/// over how a package should be fetched.
30#[derive(Builder)]
31#[builder(start_fn = new, finish_fn(name = _build, vis = ""))]
32pub struct FetchSrc<'a, R: Rockspec> {
33    #[builder(start_fn)]
34    dest_dir: &'a Path,
35    #[builder(start_fn)]
36    rockspec: &'a R,
37    #[builder(start_fn)]
38    config: &'a Config,
39    #[builder(setters(vis = "pub(crate)"))]
40    source_url: Option<RemotePackageSourceUrl>,
41}
42
43#[derive(Debug)]
44pub(crate) struct RemotePackageSourceMetadata {
45    pub hash: Integrity,
46    pub source_url: RemotePackageSourceUrl,
47}
48
49impl<R: Rockspec, State> FetchSrcBuilder<'_, R, State>
50where
51    State: fetch_src_builder::State + fetch_src_builder::IsComplete,
52{
53    /// Fetch and unpack the source into the `dest_dir`.
54    pub async fn fetch(self) -> Result<(), FetchSrcError> {
55        self.fetch_internal().await?;
56        Ok(())
57    }
58
59    /// Fetch and unpack the source into the `dest_dir`,
60    /// returning the source `Integrity`.
61    pub(crate) async fn fetch_internal(self) -> Result<RemotePackageSourceMetadata, FetchSrcError> {
62        let fetch = self._build();
63        match do_fetch_src(&fetch).await {
64            Err(err)
65                if fetch
66                    .source_url
67                    .is_some_and(|url| matches!(url, RemotePackageSourceUrl::File { .. })) =>
68            {
69                // Don't fall back to downloading .src.rock archives if a local source was specified.
70                Err(err)
71            }
72            Err(err) => match &fetch.rockspec.source().current_platform().source_spec {
73                RockSourceSpec::Git(_) | RockSourceSpec::Url(_) => {
74                    let package = PackageSpec::new(
75                        fetch.rockspec.package().clone(),
76                        fetch.rockspec.version().clone(),
77                    );
78                    let metadata = FetchSrcRock::new(&package, fetch.dest_dir, fetch.config)
79                        .fetch()
80                        .await?;
81                    Ok(metadata)
82                }
83                RockSourceSpec::File(_) => Err(err),
84            },
85            Ok(metadata) => Ok(metadata),
86        }
87    }
88}
89
90#[derive(Error, Debug, Diagnostic)]
91#[non_exhaustive]
92pub enum FetchSrcError {
93    #[error("failed to clone rock source:\n{0}")]
94    #[diagnostic(help("check your network connection and verify the git URL is correct."))]
95    GitClone(#[from] git2::Error),
96    #[error("failed to parse git URL:\n{0}")]
97    #[diagnostic(forward(0))]
98    GitUrlParse(#[from] RemoteGitUrlParseError),
99    #[error(transparent)]
100    #[diagnostic(help("check your network connection."))]
101    Request(#[from] reqwest::Error),
102    #[error(transparent)]
103    #[diagnostic(transparent)]
104    Unpack(#[from] UnpackError),
105    #[error(transparent)]
106    #[diagnostic(transparent)]
107    FetchSrcRock(#[from] FetchSrcRockError),
108    #[error("unable to remove the '.git' directory:\n{0}")]
109    #[diagnostic(help(
110        "check that no process is using the directory and you have write permissions."
111    ))]
112    CleanGitDir(io::Error),
113    #[error("unable to compute hash:\n{0}")]
114    Hash(io::Error),
115    #[error(transparent)]
116    #[diagnostic(transparent)]
117    Fs(#[from] fs::FsError),
118}
119
120/// A rocks package source fetcher, providing fine-grained control
121/// over how a package should be fetched.
122#[derive(Builder)]
123#[builder(start_fn = new, finish_fn(name = _build, vis = ""))]
124struct FetchSrcRock<'a> {
125    #[builder(start_fn)]
126    package: &'a PackageSpec,
127    #[builder(start_fn)]
128    dest_dir: &'a Path,
129    #[builder(start_fn)]
130    config: &'a Config,
131}
132
133impl<State> FetchSrcRockBuilder<'_, State>
134where
135    State: fetch_src_rock_builder::State + fetch_src_rock_builder::IsComplete,
136{
137    pub async fn fetch(self) -> Result<RemotePackageSourceMetadata, FetchSrcRockError> {
138        do_fetch_src_rock(self._build()).await
139    }
140}
141
142#[derive(Error, Debug, Diagnostic)]
143#[non_exhaustive]
144#[error(transparent)]
145pub enum FetchSrcRockError {
146    DownloadSrcRock(#[from] DownloadSrcRockError),
147    Unpack(#[from] UnpackError),
148    Io(#[from] io::Error),
149}
150
151/// A no-prompt implementer for auth_git2's prompter
152#[derive(Copy, Clone, Debug)]
153struct NullPrompter;
154
155impl Prompter for NullPrompter {
156    fn prompt_username_password(&mut self, _: &str, _: &git2::Config) -> Option<(String, String)> {
157        None
158    }
159
160    fn prompt_password(&mut self, _: &str, _: &str, _: &git2::Config) -> Option<String> {
161        None
162    }
163
164    fn prompt_ssh_key_passphrase(&mut self, _: &Path, _: &git2::Config) -> Option<String> {
165        None
166    }
167}
168
169async fn do_fetch_src<R: Rockspec>(
170    fetch: &FetchSrc<'_, R>,
171) -> Result<RemotePackageSourceMetadata, FetchSrcError> {
172    let rockspec = fetch.rockspec;
173    let rock_source = rockspec.source().current_platform();
174    let dest_dir = fetch.dest_dir;
175    let config = fetch.config;
176    // prioritise lockfile source, if present
177    let mut source_spec = match &fetch.source_url {
178        Some(source_url) => match source_url {
179            RemotePackageSourceUrl::Git { url, checkout_ref } => RockSourceSpec::Git(GitSource {
180                url: url.parse()?,
181                checkout_ref: Some(checkout_ref.clone()),
182            }),
183            RemotePackageSourceUrl::Url { url } => RockSourceSpec::Url(url.clone()),
184            RemotePackageSourceUrl::File { path } => RockSourceSpec::File(path.clone()),
185        },
186        None => rock_source.source_spec.clone(),
187    };
188    let span = span!(
189        tracing::Level::INFO,
190        "Fetching source",
191        location = source_spec.to_string(),
192    );
193    let _enter = span.enter();
194
195    if let Some(vendor_dir) = config.vendor_dir() {
196        source_spec = match source_spec {
197            // could be a project directory (not vendored) or a local source
198            // or a vendored dependency that we have already resolved
199            RockSourceSpec::File(_) => source_spec,
200            _ => {
201                let pkg_vendor_dir =
202                    vendor_dir.join(format!("{}@{}", rockspec.package(), rockspec.version()));
203                RockSourceSpec::File(pkg_vendor_dir)
204            }
205        }
206    }
207    let metadata = match &source_spec {
208        RockSourceSpec::Git(git) => {
209            let url = git.url.to_string();
210            tracing::debug!(message = format!("Cloning {url}").as_str());
211
212            let checkout_ref = {
213                let auth = if config.no_prompt() {
214                    GitAuthenticator::default()
215                        .try_password_prompt(0)
216                        .prompt_ssh_key_password(false)
217                        .set_prompter(NullPrompter)
218                } else {
219                    GitAuthenticator::default()
220                };
221                let git_config = git2::Config::open_default()?;
222                let mut callbacks = RemoteCallbacks::new();
223                callbacks.credentials(auth.credentials(&git_config));
224                let mut fetch_options = FetchOptions::new();
225                fetch_options.update_fetchhead(false);
226                fetch_options.remote_callbacks(callbacks);
227                if git.checkout_ref.is_none() {
228                    fetch_options.depth(1);
229                };
230                let mut repo_builder = RepoBuilder::new();
231                repo_builder.fetch_options(fetch_options);
232                let repo = repo_builder.clone(&url, dest_dir)?;
233
234                match &git.checkout_ref {
235                    Some(checkout_ref) => {
236                        let (object, _) = repo.revparse_ext(checkout_ref)?;
237                        repo.checkout_tree(&object, None)?;
238                        checkout_ref.clone()
239                    }
240                    None => {
241                        let head = repo.head()?;
242                        let commit = head.peel_to_commit()?;
243                        commit.id().to_string()
244                    }
245                }
246            };
247            // The .git directory is not deterministic
248            remove_dir_all(dest_dir.join(".git")).map_err(FetchSrcError::CleanGitDir)?;
249            let hash = fetch.dest_dir.hash().await.map_err(FetchSrcError::Hash)?;
250            RemotePackageSourceMetadata {
251                hash,
252                source_url: RemotePackageSourceUrl::Git { url, checkout_ref },
253            }
254        }
255        RockSourceSpec::Url(url) => {
256            tracing::debug!(message = format!("📥 Downloading {url}").as_str());
257
258            // NOTE: We don't enforce HTTPS when fetching sources because some rockspecs
259            // have HTTP URLs in `source.url`.
260            let response = crate::reqwest::http_client(config)?
261                .get(url.clone())
262                .send()
263                .await?
264                .error_for_status()?
265                .bytes()
266                .await?;
267            let hash = response.hash().await.map_err(FetchSrcError::Hash)?;
268            let file_name = url
269                .path_segments()
270                .and_then(|mut segments| segments.next_back())
271                .and_then(|name| {
272                    if name.is_empty() {
273                        None
274                    } else {
275                        Some(name.to_string())
276                    }
277                })
278                .unwrap_or(url.to_string());
279            let cursor = Cursor::new(response);
280            let mime_type = infer::get(cursor.get_ref()).map(|file_type| file_type.mime_type());
281            operations::unpack::unpack(
282                mime_type,
283                cursor,
284                rock_source.unpack_dir.is_none(),
285                file_name,
286                dest_dir,
287            )
288            .await?;
289            RemotePackageSourceMetadata {
290                hash,
291                source_url: RemotePackageSourceUrl::Url { url: url.clone() },
292            }
293        }
294        RockSourceSpec::File(path) => {
295            tracing::debug!(message = format!("📋 Copying {}", path.display()).as_str());
296
297            let hash = if path.is_dir() {
298                recursive_copy_dir(&path.to_path_buf(), dest_dir).await?;
299                dest_dir.hash().await.map_err(FetchSrcError::Hash)?
300            } else {
301                let mut file = fs::sync::open(path)?;
302                let mut buffer = Vec::new();
303                file.read_to_end(&mut buffer)
304                    .map_err(|source| fs::FsError::Read {
305                        path: path.to_path_buf(),
306                        source,
307                    })?;
308                let mime_type = infer::get(&buffer).map(|file_type| file_type.mime_type());
309                let file_name = path
310                    .file_name()
311                    .map(|os_str| os_str.to_string_lossy())
312                    .unwrap_or(path.to_string_lossy())
313                    .to_string();
314                operations::unpack::unpack(
315                    mime_type,
316                    file,
317                    rock_source.unpack_dir.is_none(),
318                    file_name,
319                    dest_dir,
320                )
321                .await?;
322                path.hash().await.map_err(FetchSrcError::Hash)?
323            };
324            RemotePackageSourceMetadata {
325                hash,
326                source_url: RemotePackageSourceUrl::File { path: path.clone() },
327            }
328        }
329    };
330    Ok(metadata)
331}
332
333async fn do_fetch_src_rock(
334    fetch: FetchSrcRock<'_>,
335) -> Result<RemotePackageSourceMetadata, FetchSrcRockError> {
336    let package = fetch.package;
337    let span = span!(
338        tracing::Level::INFO,
339        "Fetching src.rock",
340        package = package.to_string(),
341    );
342    let _enter = span.enter();
343
344    let dest_dir = fetch.dest_dir;
345    let config = fetch.config;
346    let src_rock = operations::download_src_rock(package, config.server(), fetch.config).await?;
347    let hash = src_rock.bytes.hash().await?;
348    let cursor = Cursor::new(src_rock.bytes);
349    let mime_type = infer::get(cursor.get_ref()).map(|file_type| file_type.mime_type());
350    operations::unpack::unpack(mime_type, cursor, true, src_rock.file_name, dest_dir).await?;
351    Ok(RemotePackageSourceMetadata {
352        hash,
353        source_url: RemotePackageSourceUrl::Url { url: src_rock.url },
354    })
355}
lux_lib/operations/fetch.rs

lux_lib/operations/
fetch.rs
