lux_lib/operations/
fetch.rs

1use bon::Builder;
2use git2::build::RepoBuilder;
3use git2::FetchOptions;
4use git_url_parse::GitUrlParseError;
5use ssri::Integrity;
6use std::fs::File;
7use std::io;
8use std::io::Cursor;
9use std::io::Read;
10use std::path::Path;
11use thiserror::Error;
12
13use crate::build::utils::recursive_copy_dir;
14use crate::config::Config;
15use crate::git::GitSource;
16use crate::hash::HasIntegrity;
17use crate::lockfile::RemotePackageSourceUrl;
18use crate::lua_rockspec::RockSourceSpec;
19use crate::operations;
20use crate::package::PackageSpec;
21use crate::progress::Progress;
22use crate::progress::ProgressBar;
23use crate::rockspec::Rockspec;
24
25use super::DownloadSrcRockError;
26use super::UnpackError;
27
28/// A rocks package source fetcher, providing fine-grained control
29/// over how a package should be fetched.
30#[derive(Builder)]
31#[builder(start_fn = new, finish_fn(name = _build, vis = ""))]
32pub struct FetchSrc<'a, R: Rockspec> {
33    #[builder(start_fn)]
34    dest_dir: &'a Path,
35    #[builder(start_fn)]
36    rockspec: &'a R,
37    #[builder(start_fn)]
38    config: &'a Config,
39    #[builder(start_fn)]
40    progress: &'a Progress<ProgressBar>,
41    #[builder(field)]
42    source_url: Option<RemotePackageSourceUrl>,
43}
44
45impl<R: Rockspec, State> FetchSrcBuilder<'_, R, State>
46where
47    State: fetch_src_builder::State,
48{
49    /// Override the source URL with one potentially from a lockfile
50    pub(crate) fn source_url(self, source_url: Option<RemotePackageSourceUrl>) -> Self {
51        Self { source_url, ..self }
52    }
53}
54
55#[derive(Debug)]
56pub(crate) struct RemotePackageSourceMetadata {
57    pub hash: Integrity,
58    pub source_url: RemotePackageSourceUrl,
59}
60
61impl<R: Rockspec, State> FetchSrcBuilder<'_, R, State>
62where
63    State: fetch_src_builder::State + fetch_src_builder::IsComplete,
64{
65    /// Fetch and unpack the source into the `dest_dir`.
66    pub async fn fetch(self) -> Result<(), FetchSrcError> {
67        self.fetch_internal().await?;
68        Ok(())
69    }
70
71    /// Fetch and unpack the source into the `dest_dir`,
72    /// returning the source `Integrity`.
73    pub(crate) async fn fetch_internal(self) -> Result<RemotePackageSourceMetadata, FetchSrcError> {
74        let fetch = self._build();
75        match do_fetch_src(&fetch).await {
76            Err(err) => match &fetch.rockspec.source().current_platform().source_spec {
77                RockSourceSpec::Git(_) | RockSourceSpec::Url(_) => {
78                    let package = PackageSpec::new(
79                        fetch.rockspec.package().clone(),
80                        fetch.rockspec.version().clone(),
81                    );
82                    fetch.progress.map(|p| {
83                        p.println(format!(
84                            "⚠️ WARNING: Failed to fetch source for {}: {}",
85                            &package, err
86                        ))
87                    });
88                    fetch
89                        .progress
90                        .map(|p| p.println("⚠️ Falling back to .src.rock archive"));
91                    let metadata =
92                        FetchSrcRock::new(&package, fetch.dest_dir, fetch.config, fetch.progress)
93                            .fetch()
94                            .await?;
95                    Ok(metadata)
96                }
97                RockSourceSpec::File(_) => Err(err),
98            },
99            Ok(metadata) => Ok(metadata),
100        }
101    }
102}
103
104#[derive(Error, Debug)]
105pub enum FetchSrcError {
106    #[error("failed to clone rock source: {0}")]
107    GitClone(#[from] git2::Error),
108    #[error("failed to parse git URL: {0}")]
109    GitUrlParse(#[from] GitUrlParseError),
110    #[error(transparent)]
111    Io(#[from] io::Error),
112    #[error(transparent)]
113    Request(#[from] reqwest::Error),
114    #[error(transparent)]
115    Unpack(#[from] UnpackError),
116    #[error(transparent)]
117    FetchSrcRock(#[from] FetchSrcRockError),
118}
119
120/// A rocks package source fetcher, providing fine-grained control
121/// over how a package should be fetched.
122#[derive(Builder)]
123#[builder(start_fn = new, finish_fn(name = _build, vis = ""))]
124struct FetchSrcRock<'a> {
125    #[builder(start_fn)]
126    package: &'a PackageSpec,
127    #[builder(start_fn)]
128    dest_dir: &'a Path,
129    #[builder(start_fn)]
130    config: &'a Config,
131    #[builder(start_fn)]
132    progress: &'a Progress<ProgressBar>,
133}
134
135impl<State> FetchSrcRockBuilder<'_, State>
136where
137    State: fetch_src_rock_builder::State + fetch_src_rock_builder::IsComplete,
138{
139    pub async fn fetch(self) -> Result<RemotePackageSourceMetadata, FetchSrcRockError> {
140        do_fetch_src_rock(self._build()).await
141    }
142}
143
144#[derive(Error, Debug)]
145#[error(transparent)]
146pub enum FetchSrcRockError {
147    DownloadSrcRock(#[from] DownloadSrcRockError),
148    Unpack(#[from] UnpackError),
149    Io(#[from] io::Error),
150}
151
152async fn do_fetch_src<R: Rockspec>(
153    fetch: &FetchSrc<'_, R>,
154) -> Result<RemotePackageSourceMetadata, FetchSrcError> {
155    let rockspec = fetch.rockspec;
156    let rock_source = rockspec.source().current_platform();
157    let progress = fetch.progress;
158    let dest_dir = fetch.dest_dir;
159    // prioritise lockfile source, if present
160    let source_spec = match &fetch.source_url {
161        Some(source_url) => match source_url {
162            RemotePackageSourceUrl::Git { url, checkout_ref } => RockSourceSpec::Git(GitSource {
163                url: url.parse()?,
164                checkout_ref: Some(checkout_ref.clone()),
165            }),
166            RemotePackageSourceUrl::Url { url } => RockSourceSpec::Url(url.clone()),
167            RemotePackageSourceUrl::File { path } => RockSourceSpec::File(path.clone()),
168        },
169        None => rock_source.source_spec.clone(),
170    };
171    let metadata = match &source_spec {
172        RockSourceSpec::Git(git) => {
173            let url = git.url.to_string();
174            progress.map(|p| p.set_message(format!("🦠 Cloning {}", url)));
175
176            let mut fetch_options = FetchOptions::new();
177            fetch_options.update_fetchhead(false);
178            if git.checkout_ref.is_none() {
179                fetch_options.depth(1);
180            };
181            let mut repo_builder = RepoBuilder::new();
182            repo_builder.fetch_options(fetch_options);
183            let repo = repo_builder.clone(&url, dest_dir)?;
184
185            let checkout_ref = match &git.checkout_ref {
186                Some(checkout_ref) => {
187                    let (object, _) = repo.revparse_ext(checkout_ref)?;
188                    repo.checkout_tree(&object, None)?;
189                    checkout_ref.clone()
190                }
191                None => {
192                    let head = repo.head()?;
193                    let commit = head.peel_to_commit()?;
194                    commit.id().to_string()
195                }
196            };
197            // The .git directory is not deterministic
198            std::fs::remove_dir_all(dest_dir.join(".git"))?;
199            let hash = fetch.dest_dir.hash()?;
200            RemotePackageSourceMetadata {
201                hash,
202                source_url: RemotePackageSourceUrl::Git { url, checkout_ref },
203            }
204        }
205        RockSourceSpec::Url(url) => {
206            progress.map(|p| p.set_message(format!("📥 Downloading {}", url.to_owned())));
207
208            let response = reqwest::get(url.to_owned())
209                .await?
210                .error_for_status()?
211                .bytes()
212                .await?;
213            let hash = response.hash()?;
214            let file_name = url
215                .path_segments()
216                .and_then(|mut segments| segments.next_back())
217                .and_then(|name| {
218                    if name.is_empty() {
219                        None
220                    } else {
221                        Some(name.to_string())
222                    }
223                })
224                .unwrap_or(url.to_string());
225            let cursor = Cursor::new(response);
226            let mime_type = infer::get(cursor.get_ref()).map(|file_type| file_type.mime_type());
227            operations::unpack::unpack(
228                mime_type,
229                cursor,
230                rock_source.unpack_dir.is_none(),
231                file_name,
232                dest_dir,
233                progress,
234            )
235            .await?;
236            RemotePackageSourceMetadata {
237                hash,
238                source_url: RemotePackageSourceUrl::Url { url: url.clone() },
239            }
240        }
241        RockSourceSpec::File(path) => {
242            let hash = if path.is_dir() {
243                progress.map(|p| p.set_message(format!("📋 Copying {}", path.display())));
244                recursive_copy_dir(&path.to_path_buf(), dest_dir).await?;
245                progress.map(|p| p.finish_and_clear());
246                dest_dir.hash()?
247            } else {
248                let mut file = File::open(path)?;
249                let mut buffer = Vec::new();
250                file.read_to_end(&mut buffer)?;
251                let mime_type = infer::get(&buffer).map(|file_type| file_type.mime_type());
252                let file_name = path
253                    .file_name()
254                    .map(|os_str| os_str.to_string_lossy())
255                    .unwrap_or(path.to_string_lossy())
256                    .to_string();
257                operations::unpack::unpack(
258                    mime_type,
259                    file,
260                    rock_source.unpack_dir.is_none(),
261                    file_name,
262                    dest_dir,
263                    progress,
264                )
265                .await?;
266                path.hash()?
267            };
268            RemotePackageSourceMetadata {
269                hash,
270                source_url: RemotePackageSourceUrl::File { path: path.clone() },
271            }
272        }
273    };
274    Ok(metadata)
275}
276
277async fn do_fetch_src_rock(
278    fetch: FetchSrcRock<'_>,
279) -> Result<RemotePackageSourceMetadata, FetchSrcRockError> {
280    let package = fetch.package;
281    let dest_dir = fetch.dest_dir;
282    let config = fetch.config;
283    let progress = fetch.progress;
284    let src_rock = operations::download_src_rock(package, config.server(), progress).await?;
285    let hash = src_rock.bytes.hash()?;
286    let cursor = Cursor::new(src_rock.bytes);
287    let mime_type = infer::get(cursor.get_ref()).map(|file_type| file_type.mime_type());
288    operations::unpack::unpack(
289        mime_type,
290        cursor,
291        true,
292        src_rock.file_name,
293        dest_dir,
294        progress,
295    )
296    .await?;
297    Ok(RemotePackageSourceMetadata {
298        hash,
299        source_url: RemotePackageSourceUrl::Url { url: src_rock.url },
300    })
301}
lux_lib/operations/fetch.rs

lux_lib/operations/
fetch.rs
