lux_lib/operations/
fetch.rs

1use bon::Builder;
2use git2::build::RepoBuilder;
3use git2::FetchOptions;
4use git_url_parse::GitUrlParseError;
5use ssri::Integrity;
6use std::fs::File;
7use std::io;
8use std::io::Cursor;
9use std::io::Read;
10use std::path::Path;
11use std::path::PathBuf;
12use thiserror::Error;
13
14use crate::build::utils::recursive_copy_dir;
15use crate::config::Config;
16use crate::git::GitSource;
17use crate::hash::HasIntegrity;
18use crate::lockfile::RemotePackageSourceUrl;
19use crate::lua_rockspec::RockSourceSpec;
20use crate::operations;
21use crate::package::PackageSpec;
22use crate::progress::Progress;
23use crate::progress::ProgressBar;
24use crate::rockspec::Rockspec;
25
26use super::DownloadSrcRockError;
27use super::UnpackError;
28
29/// A rocks package source fetcher, providing fine-grained control
30/// over how a package should be fetched.
31#[derive(Builder)]
32#[builder(start_fn = new, finish_fn(name = _build, vis = ""))]
33pub struct FetchSrc<'a, R: Rockspec> {
34    #[builder(start_fn)]
35    dest_dir: &'a Path,
36    #[builder(start_fn)]
37    rockspec: &'a R,
38    #[builder(start_fn)]
39    config: &'a Config,
40    #[builder(start_fn)]
41    progress: &'a Progress<ProgressBar>,
42    source_url: Option<RemotePackageSourceUrl>,
43}
44
45#[derive(Debug)]
46pub(crate) struct RemotePackageSourceMetadata {
47    pub hash: Integrity,
48    pub source_url: RemotePackageSourceUrl,
49}
50
51impl RemotePackageSourceMetadata {
52    pub(crate) fn archive_name(&self) -> Option<PathBuf> {
53        self.source_url.archive_name()
54    }
55}
56
57impl<R: Rockspec, State> FetchSrcBuilder<'_, R, State>
58where
59    State: fetch_src_builder::State + fetch_src_builder::IsComplete,
60{
61    /// Fetch and unpack the source into the `dest_dir`.
62    pub async fn fetch(self) -> Result<(), FetchSrcError> {
63        self.fetch_internal().await?;
64        Ok(())
65    }
66
67    /// Fetch and unpack the source into the `dest_dir`,
68    /// returning the source `Integrity`.
69    pub(crate) async fn fetch_internal(self) -> Result<RemotePackageSourceMetadata, FetchSrcError> {
70        let fetch = self._build();
71        match do_fetch_src(&fetch).await {
72            Err(err) => match &fetch.rockspec.source().current_platform().source_spec {
73                RockSourceSpec::Git(_) | RockSourceSpec::Url(_) => {
74                    let package = PackageSpec::new(
75                        fetch.rockspec.package().clone(),
76                        fetch.rockspec.version().clone(),
77                    );
78                    fetch.progress.map(|p| {
79                        p.println(format!(
80                            "⚠️ WARNING: Failed to fetch source for {}: {}",
81                            &package, err
82                        ))
83                    });
84                    fetch
85                        .progress
86                        .map(|p| p.println("⚠️ Falling back to .src.rock archive"));
87                    let metadata =
88                        FetchSrcRock::new(&package, fetch.dest_dir, fetch.config, fetch.progress)
89                            .fetch()
90                            .await?;
91                    Ok(metadata)
92                }
93                RockSourceSpec::File(_) => Err(err),
94            },
95            Ok(metadata) => Ok(metadata),
96        }
97    }
98}
99
100#[derive(Error, Debug)]
101pub enum FetchSrcError {
102    #[error("failed to clone rock source: {0}")]
103    GitClone(#[from] git2::Error),
104    #[error("failed to parse git URL: {0}")]
105    GitUrlParse(#[from] GitUrlParseError),
106    #[error(transparent)]
107    Io(#[from] io::Error),
108    #[error(transparent)]
109    Request(#[from] reqwest::Error),
110    #[error(transparent)]
111    Unpack(#[from] UnpackError),
112    #[error(transparent)]
113    FetchSrcRock(#[from] FetchSrcRockError),
114}
115
116/// A rocks package source fetcher, providing fine-grained control
117/// over how a package should be fetched.
118#[derive(Builder)]
119#[builder(start_fn = new, finish_fn(name = _build, vis = ""))]
120struct FetchSrcRock<'a> {
121    #[builder(start_fn)]
122    package: &'a PackageSpec,
123    #[builder(start_fn)]
124    dest_dir: &'a Path,
125    #[builder(start_fn)]
126    config: &'a Config,
127    #[builder(start_fn)]
128    progress: &'a Progress<ProgressBar>,
129}
130
131impl<State> FetchSrcRockBuilder<'_, State>
132where
133    State: fetch_src_rock_builder::State + fetch_src_rock_builder::IsComplete,
134{
135    pub async fn fetch(self) -> Result<RemotePackageSourceMetadata, FetchSrcRockError> {
136        do_fetch_src_rock(self._build()).await
137    }
138}
139
140#[derive(Error, Debug)]
141#[error(transparent)]
142pub enum FetchSrcRockError {
143    DownloadSrcRock(#[from] DownloadSrcRockError),
144    Unpack(#[from] UnpackError),
145    Io(#[from] io::Error),
146}
147
148async fn do_fetch_src<R: Rockspec>(
149    fetch: &FetchSrc<'_, R>,
150) -> Result<RemotePackageSourceMetadata, FetchSrcError> {
151    let rockspec = fetch.rockspec;
152    let rock_source = rockspec.source().current_platform();
153    let progress = fetch.progress;
154    let dest_dir = fetch.dest_dir;
155    // prioritise lockfile source, if present
156    let source_spec = match &fetch.source_url {
157        Some(source_url) => match source_url {
158            RemotePackageSourceUrl::Git { url, checkout_ref } => RockSourceSpec::Git(GitSource {
159                url: url.parse()?,
160                checkout_ref: Some(checkout_ref.clone()),
161            }),
162            RemotePackageSourceUrl::Url { url } => RockSourceSpec::Url(url.clone()),
163            RemotePackageSourceUrl::File { path } => RockSourceSpec::File(path.clone()),
164        },
165        None => rock_source.source_spec.clone(),
166    };
167    let metadata = match &source_spec {
168        RockSourceSpec::Git(git) => {
169            let url = git.url.to_string();
170            progress.map(|p| p.set_message(format!("🦠 Cloning {}", url)));
171
172            let mut fetch_options = FetchOptions::new();
173            fetch_options.update_fetchhead(false);
174            if git.checkout_ref.is_none() {
175                fetch_options.depth(1);
176            };
177            let mut repo_builder = RepoBuilder::new();
178            repo_builder.fetch_options(fetch_options);
179            let repo = repo_builder.clone(&url, dest_dir)?;
180
181            let checkout_ref = match &git.checkout_ref {
182                Some(checkout_ref) => {
183                    let (object, _) = repo.revparse_ext(checkout_ref)?;
184                    repo.checkout_tree(&object, None)?;
185                    checkout_ref.clone()
186                }
187                None => {
188                    let head = repo.head()?;
189                    let commit = head.peel_to_commit()?;
190                    commit.id().to_string()
191                }
192            };
193            // The .git directory is not deterministic
194            std::fs::remove_dir_all(dest_dir.join(".git"))?;
195            let hash = fetch.dest_dir.hash()?;
196            RemotePackageSourceMetadata {
197                hash,
198                source_url: RemotePackageSourceUrl::Git { url, checkout_ref },
199            }
200        }
201        RockSourceSpec::Url(url) => {
202            progress.map(|p| p.set_message(format!("📥 Downloading {}", url.to_owned())));
203
204            let response = reqwest::get(url.to_owned())
205                .await?
206                .error_for_status()?
207                .bytes()
208                .await?;
209            let hash = response.hash()?;
210            let file_name = url
211                .path_segments()
212                .and_then(|mut segments| segments.next_back())
213                .and_then(|name| {
214                    if name.is_empty() {
215                        None
216                    } else {
217                        Some(name.to_string())
218                    }
219                })
220                .unwrap_or(url.to_string());
221            let cursor = Cursor::new(response);
222            let mime_type = infer::get(cursor.get_ref()).map(|file_type| file_type.mime_type());
223            operations::unpack::unpack(
224                mime_type,
225                cursor,
226                rock_source.unpack_dir.is_none(),
227                file_name,
228                dest_dir,
229                progress,
230            )
231            .await?;
232            RemotePackageSourceMetadata {
233                hash,
234                source_url: RemotePackageSourceUrl::Url { url: url.clone() },
235            }
236        }
237        RockSourceSpec::File(path) => {
238            let hash = if path.is_dir() {
239                progress.map(|p| p.set_message(format!("📋 Copying {}", path.display())));
240                recursive_copy_dir(&path.to_path_buf(), dest_dir).await?;
241                progress.map(|p| p.finish_and_clear());
242                dest_dir.hash()?
243            } else {
244                let mut file = File::open(path)?;
245                let mut buffer = Vec::new();
246                file.read_to_end(&mut buffer)?;
247                let mime_type = infer::get(&buffer).map(|file_type| file_type.mime_type());
248                let file_name = path
249                    .file_name()
250                    .map(|os_str| os_str.to_string_lossy())
251                    .unwrap_or(path.to_string_lossy())
252                    .to_string();
253                operations::unpack::unpack(
254                    mime_type,
255                    file,
256                    rock_source.unpack_dir.is_none(),
257                    file_name,
258                    dest_dir,
259                    progress,
260                )
261                .await?;
262                path.hash()?
263            };
264            RemotePackageSourceMetadata {
265                hash,
266                source_url: RemotePackageSourceUrl::File { path: path.clone() },
267            }
268        }
269    };
270    Ok(metadata)
271}
272
273async fn do_fetch_src_rock(
274    fetch: FetchSrcRock<'_>,
275) -> Result<RemotePackageSourceMetadata, FetchSrcRockError> {
276    let package = fetch.package;
277    let dest_dir = fetch.dest_dir;
278    let config = fetch.config;
279    let progress = fetch.progress;
280    let src_rock = operations::download_src_rock(package, config.server(), progress).await?;
281    let hash = src_rock.bytes.hash()?;
282    let cursor = Cursor::new(src_rock.bytes);
283    let mime_type = infer::get(cursor.get_ref()).map(|file_type| file_type.mime_type());
284    operations::unpack::unpack(
285        mime_type,
286        cursor,
287        true,
288        src_rock.file_name,
289        dest_dir,
290        progress,
291    )
292    .await?;
293    Ok(RemotePackageSourceMetadata {
294        hash,
295        source_url: RemotePackageSourceUrl::Url { url: src_rock.url },
296    })
297}
lux_lib/operations/fetch.rs

lux_lib/operations/
fetch.rs
