lux_lib/manifest/
mod.rs

1use itertools::Itertools;
2use mlua::{Lua, LuaSerdeExt};
3use reqwest::{header::ToStrError, Client};
4use std::path::{Path, PathBuf};
5use std::time::SystemTime;
6use std::{cmp::Ordering, collections::HashMap};
7use thiserror::Error;
8use tokio::fs::{File, OpenOptions};
9use tokio::io::{AsyncReadExt, AsyncSeekExt};
10use tokio::{fs, io};
11use url::Url;
12use zip::ZipArchive;
13
14use crate::config::LuaVersionUnset;
15use crate::package::{RemotePackageType, RemotePackageTypeFilterSpec};
16use crate::progress::{Progress, ProgressBar};
17use crate::{
18    config::{Config, LuaVersion},
19    package::{PackageName, PackageReq, PackageSpec, PackageVersion, RemotePackage},
20    remote_package_source::RemotePackageSource,
21};
22
23#[derive(Error, Debug)]
24pub enum ManifestFromServerError {
25    #[error(transparent)]
26    Io(#[from] io::Error),
27    #[error("failed to pull manifest: {0}")]
28    Request(#[from] reqwest::Error),
29    #[error("invalidate date received from server: {0}")]
30    InvalidDate(#[from] httpdate::Error),
31    #[error("non-ASCII characters returned in response header: {0}")]
32    InvalidHeader(#[from] ToStrError),
33    #[error("error parsing manifest URL: {0}")]
34    Url(#[from] url::ParseError),
35    #[error("failed to read manifest archive {0}:\n{1}")]
36    ZipRead(Url, zip::result::ZipError),
37    #[error("failed to unzip manifest file {0}:\n{1}")]
38    ZipExtract(Url, zip::result::ZipError),
39    #[error(transparent)]
40    LuaVersion(#[from] LuaVersionUnset),
41}
42
43async fn get_manifest(
44    url: Url,
45    manifest_version: String,
46    target: &Path,
47    client: &Client,
48) -> Result<String, ManifestFromServerError> {
49    let manifest_bytes = client
50        .get(url.clone())
51        .send()
52        .await?
53        .error_for_status()?
54        .bytes()
55        .await?;
56    let mut archive = ZipArchive::new(std::io::Cursor::new(manifest_bytes))
57        .map_err(|err| ManifestFromServerError::ZipRead(url.clone(), err))?;
58
59    let temp = tempdir::TempDir::new("lux-manifest")?;
60
61    archive
62        .extract_unwrapped_root_dir(&temp, zip::read::root_dir_common_filter)
63        .map_err(|err| ManifestFromServerError::ZipExtract(url.clone(), err))?;
64
65    let mut extracted_manifest =
66        File::open(temp.path().join(format!("manifest-{}", manifest_version))).await?;
67    let mut target = OpenOptions::new()
68        .read(true)
69        .write(true)
70        .create(true)
71        .truncate(true)
72        .open(target)
73        .await?;
74
75    io::copy(&mut extracted_manifest, &mut target).await?;
76
77    let mut manifest = String::new();
78
79    target.seek(io::SeekFrom::Start(0)).await?;
80    target.read_to_string(&mut manifest).await?;
81
82    Ok(manifest)
83}
84
85/// Look up the manifest from a cache, or get the manifest from the server
86/// if the cache doesn't exist or is outdated.
87async fn manifest_from_cache_or_server(
88    server_url: &Url,
89    config: &Config,
90    bar: &Progress<ProgressBar>,
91) -> Result<String, ManifestFromServerError> {
92    let manifest_version = LuaVersion::from(config)?.version_compatibility_str();
93    let url = mk_manifest_url(server_url, &manifest_version, config)?;
94
95    // Stores a path to the manifest cache (this allows us to operate on a manifest without
96    // needing to pull it from the luarocks servers each time).
97    let cache = mk_manifest_cache(&url, config).await?;
98
99    let client = Client::new();
100
101    // Read the metadata of the local cache and attempt to get the last modified date.
102    if let Ok(metadata) = fs::metadata(&cache).await {
103        let last_modified_local: SystemTime = metadata.modified()?;
104
105        // Ask the server for the last modified date of its manifest.
106        let response = client.head(url.clone()).send().await?.error_for_status()?;
107
108        if let Some(last_modified_header) = response.headers().get("Last-Modified") {
109            let server_last_modified = httpdate::parse_http_date(last_modified_header.to_str()?)?;
110
111            // If the server's version of the manifest is newer than ours then update out manifest.
112            if server_last_modified > last_modified_local {
113                // Since we only pulled in the headers previously we must now request the entire
114                // manifest from scratch.
115                bar.map(|bar| {
116                    bar.set_message(format!("📥 Downloading updated manifest from {}", &url))
117                });
118
119                return get_manifest(url, manifest_version.clone(), &cache, &client).await;
120            }
121
122            // Else return the cached manifest.
123            return Ok(fs::read_to_string(&cache).await?);
124        }
125    }
126
127    // If our cache file does not exist then pull the whole manifest.
128    // TODO(#337): switch to something that can report progress
129    bar.map(|bar| bar.set_message(format!("📥 Downloading manifest from {}", &url)));
130
131    get_manifest(url, manifest_version.clone(), &cache, &client).await
132}
133
134/// Get the manifest from the server, ignoring the cache.
135/// This still populates the cache.
136pub(crate) async fn manifest_from_server_only(
137    server_url: &Url,
138    config: &Config,
139    bar: &Progress<ProgressBar>,
140) -> Result<String, ManifestFromServerError> {
141    let manifest_version = LuaVersion::from(config)?.version_compatibility_str();
142    let url = mk_manifest_url(server_url, &manifest_version, config)?;
143    let cache = mk_manifest_cache(&url, config).await?;
144    let client = Client::new();
145    bar.map(|bar| bar.set_message(format!("📥 Downloading manifest from {}", &url)));
146    get_manifest(url, manifest_version.clone(), &cache, &client).await
147}
148
149fn mk_manifest_url(
150    server_url: &Url,
151    manifest_version: &str,
152    config: &Config,
153) -> Result<Url, ManifestFromServerError> {
154    let manifest_filename = format!("manifest-{}.zip", manifest_version);
155    let url = match config.namespace() {
156        Some(ns) => server_url
157            .join(&format!("manifests/{}/", ns))?
158            .join(&manifest_filename)?,
159        None => server_url.join(&manifest_filename)?,
160    };
161    Ok(url)
162}
163
164async fn mk_manifest_cache(url: &Url, config: &Config) -> io::Result<PathBuf> {
165    let cache = config.cache_dir().join(
166        // Convert the url to a directory name so we don't create too many subdirectories
167        url.to_string()
168            .replace(&[':', '*', '?', '"', '<', '>', '|', '/', '\\'][..], "_")
169            .trim_end_matches(".zip"),
170    );
171    // Ensure all intermediate directories for the cache file are created (e.g. `~/.cache/lux/manifest`)
172    fs::create_dir_all(cache.parent().unwrap()).await?;
173    Ok(cache)
174}
175
176#[derive(Clone, Debug)]
177pub(crate) struct ManifestMetadata {
178    pub repository: HashMap<PackageName, HashMap<PackageVersion, Vec<RemotePackageType>>>,
179}
180
181impl<'de> serde::Deserialize<'de> for ManifestMetadata {
182    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
183    where
184        D: serde::Deserializer<'de>,
185    {
186        let intermediate = IntermediateManifest::deserialize(deserializer)?;
187        Ok(Self::from_intermediate(intermediate))
188    }
189}
190
191#[derive(Error, Debug)]
192#[error("failed to parse manifest: {0}")]
193pub struct ManifestLuaError(#[from] mlua::Error);
194
195#[derive(Error, Debug)]
196#[error("failed to parse manifest from configuration: {0}")]
197pub enum ManifestError {
198    Lua(#[from] ManifestLuaError),
199    Server(#[from] ManifestFromServerError),
200}
201
202impl ManifestMetadata {
203    pub fn new(manifest: &String) -> Result<Self, ManifestLuaError> {
204        let lua = Lua::new();
205
206        lua.load(manifest).exec()?;
207
208        let intermediate = IntermediateManifest {
209            repository: lua.from_value(lua.globals().get("repository")?)?,
210        };
211        let manifest = Self::from_intermediate(intermediate);
212
213        Ok(manifest)
214    }
215
216    pub fn has_rock(&self, rock_name: &PackageName) -> bool {
217        self.repository.contains_key(rock_name)
218    }
219
220    pub fn latest_match(
221        &self,
222        lua_package_req: &PackageReq,
223        filter: Option<RemotePackageTypeFilterSpec>,
224    ) -> Option<(PackageSpec, RemotePackageType)> {
225        let filter = filter.unwrap_or_default();
226        if !self.has_rock(lua_package_req.name()) {
227            return None;
228        }
229
230        let (version, rock_type) = self.repository[lua_package_req.name()]
231            .iter()
232            .filter(|(version, _)| lua_package_req.version_req().matches(version))
233            .flat_map(|(version, rock_types)| {
234                rock_types.iter().filter_map(move |rock_type| {
235                    let include = match rock_type {
236                        RemotePackageType::Rockspec => filter.rockspec,
237                        RemotePackageType::Src => filter.src,
238                        RemotePackageType::Binary => filter.binary,
239                    };
240                    if include {
241                        Some((version, rock_type))
242                    } else {
243                        None
244                    }
245                })
246            })
247            .max_by(
248                |(version_a, type_a), (version_b, type_b)| match version_a.cmp(version_b) {
249                    Ordering::Equal => type_a.cmp(type_b),
250                    ordering => ordering,
251                },
252            )?;
253
254        Some((
255            PackageSpec::new(lua_package_req.name().clone(), version.clone()),
256            rock_type.clone(),
257        ))
258    }
259
260    /// Construct a `ManifestMetadata` from an intermediate representation,
261    /// silently skipping entries for versions we don't know how to parse.
262    fn from_intermediate(intermediate: IntermediateManifest) -> Self {
263        let repository = intermediate
264            .repository
265            .into_iter()
266            .map(|(name, package_map)| {
267                (
268                    name,
269                    package_map
270                        .into_iter()
271                        .filter_map(|(version_str, entries)| {
272                            let version = PackageVersion::parse(version_str.as_str()).ok()?;
273                            let entries = entries
274                                .into_iter()
275                                .filter_map(|entry| RemotePackageType::try_from(entry).ok())
276                                .collect_vec();
277                            Some((version, entries))
278                        })
279                        .collect(),
280                )
281            })
282            .collect();
283        Self { repository }
284    }
285}
286
287#[derive(Clone, Debug)]
288pub(crate) struct Manifest {
289    server_url: Url,
290    metadata: ManifestMetadata,
291}
292
293impl Manifest {
294    pub fn new(server_url: Url, metadata: ManifestMetadata) -> Self {
295        Self {
296            server_url,
297            metadata,
298        }
299    }
300
301    pub async fn from_config(
302        server_url: Url,
303        config: &Config,
304        progress: &Progress<ProgressBar>,
305    ) -> Result<Self, ManifestError> {
306        let content =
307            crate::manifest::manifest_from_cache_or_server(&server_url, config, progress).await?;
308        match ManifestMetadata::new(&content) {
309            Ok(metadata) => Ok(Self::new(server_url, metadata)),
310            Err(_) => {
311                let manifest =
312                    crate::manifest::manifest_from_server_only(&server_url, config, progress)
313                        .await?;
314                Ok(Self::new(server_url, ManifestMetadata::new(&manifest)?))
315            }
316        }
317    }
318
319    pub fn server_url(&self) -> &Url {
320        &self.server_url
321    }
322
323    pub fn metadata(&self) -> &ManifestMetadata {
324        &self.metadata
325    }
326
327    /// Find a package that matches the requirement, returning the latest match
328    pub fn find(
329        &self,
330        package_req: &PackageReq,
331        filter: Option<RemotePackageTypeFilterSpec>,
332    ) -> Option<RemotePackage> {
333        match self.metadata().latest_match(package_req, filter) {
334            None => None,
335            Some((package, package_type)) => {
336                let remote_source = match package_type {
337                    RemotePackageType::Rockspec => {
338                        RemotePackageSource::LuarocksRockspec(self.server_url().clone())
339                    }
340                    RemotePackageType::Src => {
341                        RemotePackageSource::LuarocksSrcRock(self.server_url().clone())
342                    }
343                    RemotePackageType::Binary => {
344                        RemotePackageSource::LuarocksBinaryRock(self.server_url().clone())
345                    }
346                };
347                Some(RemotePackage::new(package, remote_source, None))
348            }
349        }
350    }
351}
352
353struct UnsupportedArchitectureError;
354
355impl TryFrom<ManifestRockEntry> for RemotePackageType {
356    type Error = UnsupportedArchitectureError;
357    fn try_from(
358        ManifestRockEntry { arch }: ManifestRockEntry,
359    ) -> Result<Self, UnsupportedArchitectureError> {
360        match arch.as_str() {
361            "rockspec" => Ok(RemotePackageType::Rockspec),
362            "src" => Ok(RemotePackageType::Src),
363            "all" => Ok(RemotePackageType::Binary),
364            arch if arch == crate::luarocks::current_platform_luarocks_identifier() => {
365                Ok(RemotePackageType::Binary)
366            }
367            _ => Err(UnsupportedArchitectureError),
368        }
369    }
370}
371
372#[derive(Clone, serde::Deserialize)]
373struct ManifestRockEntry {
374    /// e.g. "linux-x86_64", "rockspec", "src", ...
375    pub arch: String,
376}
377
378/// Intermediate implementation for deserializing
379#[derive(serde::Deserialize)]
380struct IntermediateManifest {
381    /// The key of each package's HashMap is the version string
382    repository: HashMap<PackageName, HashMap<String, Vec<ManifestRockEntry>>>,
383}
384
385#[cfg(test)]
386mod tests {
387    use std::path::PathBuf;
388
389    use httptest::{matchers::request, responders::status_code, Expectation, Server};
390    use serial_test::serial;
391
392    use crate::{config::ConfigBuilder, package::PackageReq};
393
394    use super::*;
395
396    fn start_test_server(manifest_name: String) -> Server {
397        let server = Server::run();
398        let manifest_path = format!("/{}", manifest_name);
399        server.expect(
400            Expectation::matching(request::path(manifest_path + ".zip"))
401                .times(1..)
402                .respond_with(
403                    status_code(200)
404                        .append_header("Last-Modified", "Sat, 20 Jan 2024 13:14:12 GMT")
405                        .body(
406                            std::fs::read(
407                                format!(
408                                    "{}/resources/test/manifest-5.1.zip",
409                                    env!("CARGO_MANIFEST_DIR")
410                                )
411                                .as_str(),
412                            )
413                            .unwrap(),
414                        ),
415                ),
416        );
417        server
418    }
419
420    #[tokio::test]
421    #[serial]
422    pub async fn get_manifest_luajit() {
423        let cache_dir = assert_fs::TempDir::new().unwrap().to_path_buf();
424        let server = start_test_server("manifest-5.1".into());
425        let mut url_str = server.url_str(""); // Remove trailing "/"
426        url_str.pop();
427        let config = ConfigBuilder::new()
428            .unwrap()
429            .cache_dir(Some(cache_dir))
430            .lua_version(Some(crate::config::LuaVersion::LuaJIT))
431            .build()
432            .unwrap();
433        manifest_from_cache_or_server(
434            &Url::parse(&url_str).unwrap(),
435            &config,
436            &Progress::NoProgress,
437        )
438        .await
439        .unwrap();
440    }
441
442    #[tokio::test]
443    #[serial]
444    pub async fn get_manifest_for_5_1() {
445        let cache_dir = assert_fs::TempDir::new().unwrap().to_path_buf();
446        let server = start_test_server("manifest-5.1".into());
447        let mut url_str = server.url_str(""); // Remove trailing "/"
448        url_str.pop();
449
450        let config = ConfigBuilder::new()
451            .unwrap()
452            .cache_dir(Some(cache_dir))
453            .lua_version(Some(crate::config::LuaVersion::Lua51))
454            .build()
455            .unwrap();
456
457        manifest_from_cache_or_server(
458            &Url::parse(&url_str).unwrap(),
459            &config,
460            &Progress::NoProgress,
461        )
462        .await
463        .unwrap();
464    }
465
466    #[tokio::test]
467    #[serial]
468    pub async fn get_cached_manifest() {
469        let server = start_test_server("manifest-5.1".into());
470        let mut url_str = server.url_str(""); // Remove trailing "/"
471        url_str.pop();
472        let manifest_content = std::fs::read_to_string(
473            format!("{}/resources/test/manifest-5.1", env!("CARGO_MANIFEST_DIR")).as_str(),
474        )
475        .unwrap();
476        let cache_dir = assert_fs::TempDir::new().unwrap();
477        let cache = cache_dir.join("manifest-5.1");
478        fs::write(&cache, &manifest_content).await.unwrap();
479        let _metadata = fs::metadata(&cache).await.unwrap();
480        let config = ConfigBuilder::new()
481            .unwrap()
482            .cache_dir(Some(cache_dir.to_path_buf()))
483            .lua_version(Some(crate::config::LuaVersion::Lua51))
484            .build()
485            .unwrap();
486        let result = manifest_from_cache_or_server(
487            &Url::parse(&url_str).unwrap(),
488            &config,
489            &Progress::NoProgress,
490        )
491        .await
492        .unwrap();
493        assert_eq!(result, manifest_content);
494    }
495
496    #[tokio::test]
497    pub async fn parse_metadata_from_empty_manifest() {
498        let manifest = "
499            commands = {}\n
500            modules = {}\n
501            repository = {}\n
502            "
503        .to_string();
504        ManifestMetadata::new(&manifest).unwrap();
505    }
506
507    #[tokio::test]
508    pub async fn parse_metadata_from_test_manifest() {
509        let mut test_manifest_path = PathBuf::from(env!("CARGO_MANIFEST_DIR"));
510        test_manifest_path.push("resources/test/manifest-5.1");
511        let manifest = String::from_utf8(fs::read(&test_manifest_path).await.unwrap()).unwrap();
512        ManifestMetadata::new(&manifest).unwrap();
513    }
514
515    #[tokio::test]
516    pub async fn latest_match_regression() {
517        let mut test_manifest_path = PathBuf::from(env!("CARGO_MANIFEST_DIR"));
518        test_manifest_path.push("resources/test/manifest-5.1");
519        let manifest = String::from_utf8(fs::read(&test_manifest_path).await.unwrap()).unwrap();
520        let metadata = ManifestMetadata::new(&manifest).unwrap();
521
522        let package_req: PackageReq = "30log > 1.3.0".parse().unwrap();
523        assert!(metadata.latest_match(&package_req, None).is_none());
524    }
525}
lux_lib/manifest/mod.rs

lux_lib/manifest/
mod.rs
