typst_kit/
package.rs

1//! Download and unpack packages and package indices.
2
3use std::fs;
4use std::io;
5use std::path::{Path, PathBuf};
6
7use ecow::eco_format;
8use once_cell::sync::OnceCell;
9use serde::Deserialize;
10use typst_library::diag::{PackageError, PackageResult, StrResult, bail};
11use typst_syntax::package::{PackageSpec, PackageVersion, VersionlessPackageSpec};
12
13use crate::download::{Downloader, Progress};
14
15/// The default Typst registry.
16pub const DEFAULT_REGISTRY: &str = "https://packages.typst.org";
17
18/// The public namespace in the default Typst registry.
19pub const DEFAULT_NAMESPACE: &str = "preview";
20
21/// The default packages sub directory within the package and package cache paths.
22pub const DEFAULT_PACKAGES_SUBDIR: &str = "typst/packages";
23
24/// Attempts to infer the default package cache directory from the current
25/// environment.
26///
27/// This simply joins [`DEFAULT_PACKAGES_SUBDIR`] to the output of
28/// [`dirs::cache_dir`].
29pub fn default_package_cache_path() -> Option<PathBuf> {
30    dirs::cache_dir().map(|cache_dir| cache_dir.join(DEFAULT_PACKAGES_SUBDIR))
31}
32
33/// Attempts to infer the default package directory from the current
34/// environment.
35///
36/// This simply joins [`DEFAULT_PACKAGES_SUBDIR`] to the output of
37/// [`dirs::data_dir`].
38pub fn default_package_path() -> Option<PathBuf> {
39    dirs::data_dir().map(|data_dir| data_dir.join(DEFAULT_PACKAGES_SUBDIR))
40}
41
42/// Holds information about where packages should be stored and downloads them
43/// on demand, if possible.
44#[derive(Debug)]
45pub struct PackageStorage {
46    /// The path at which non-local packages should be stored when downloaded.
47    package_cache_path: Option<PathBuf>,
48    /// The path at which local packages are stored.
49    package_path: Option<PathBuf>,
50    /// The downloader used for fetching the index and packages.
51    downloader: Downloader,
52    /// The cached index of the default namespace.
53    index: OnceCell<Vec<serde_json::Value>>,
54}
55
56impl PackageStorage {
57    /// Creates a new package storage for the given package paths. Falls back to
58    /// the recommended XDG directories if they are `None`.
59    pub fn new(
60        package_cache_path: Option<PathBuf>,
61        package_path: Option<PathBuf>,
62        downloader: Downloader,
63    ) -> Self {
64        Self::with_index(package_cache_path, package_path, downloader, OnceCell::new())
65    }
66
67    /// Creates a new package storage with a pre-defined index.
68    ///
69    /// Useful for testing.
70    fn with_index(
71        package_cache_path: Option<PathBuf>,
72        package_path: Option<PathBuf>,
73        downloader: Downloader,
74        index: OnceCell<Vec<serde_json::Value>>,
75    ) -> Self {
76        Self {
77            package_cache_path: package_cache_path.or_else(default_package_cache_path),
78            package_path: package_path.or_else(default_package_path),
79            downloader,
80            index,
81        }
82    }
83
84    /// Returns the path at which non-local packages should be stored when
85    /// downloaded.
86    pub fn package_cache_path(&self) -> Option<&Path> {
87        self.package_cache_path.as_deref()
88    }
89
90    /// Returns the path at which local packages are stored.
91    pub fn package_path(&self) -> Option<&Path> {
92        self.package_path.as_deref()
93    }
94
95    /// Makes a package available on-disk and returns the path at which it is
96    /// located (will be either in the cache or package directory).
97    pub fn prepare_package(
98        &self,
99        spec: &PackageSpec,
100        progress: &mut dyn Progress,
101    ) -> PackageResult<PathBuf> {
102        let subdir = format!("{}/{}/{}", spec.namespace, spec.name, spec.version);
103
104        if let Some(packages_dir) = &self.package_path {
105            let dir = packages_dir.join(&subdir);
106            if dir.exists() {
107                return Ok(dir);
108            }
109        }
110
111        if let Some(cache_dir) = &self.package_cache_path {
112            let dir = cache_dir.join(&subdir);
113            if dir.exists() {
114                return Ok(dir);
115            }
116
117            // Download from network if it doesn't exist yet.
118            if spec.namespace == DEFAULT_NAMESPACE {
119                self.download_package(spec, cache_dir, progress)?;
120                if dir.exists() {
121                    return Ok(dir);
122                }
123            }
124        }
125
126        Err(PackageError::NotFound(spec.clone()))
127    }
128
129    /// Tries to determine the latest version of a package.
130    pub fn determine_latest_version(
131        &self,
132        spec: &VersionlessPackageSpec,
133    ) -> StrResult<PackageVersion> {
134        if spec.namespace == DEFAULT_NAMESPACE {
135            // For `DEFAULT_NAMESPACE`, download the package index and find the latest
136            // version.
137            self.download_index()?
138                .iter()
139                .filter_map(|value| MinimalPackageInfo::deserialize(value).ok())
140                .filter(|package| package.name == spec.name)
141                .map(|package| package.version)
142                .max()
143                .ok_or_else(|| eco_format!("failed to find package {spec}"))
144        } else {
145            // For other namespaces, search locally. We only search in the data
146            // directory and not the cache directory, because the latter is not
147            // intended for storage of local packages.
148            let subdir = format!("{}/{}", spec.namespace, spec.name);
149            self.package_path
150                .iter()
151                .flat_map(|dir| std::fs::read_dir(dir.join(&subdir)).ok())
152                .flatten()
153                .filter_map(|entry| entry.ok())
154                .map(|entry| entry.path())
155                .filter_map(|path| path.file_name()?.to_string_lossy().parse().ok())
156                .max()
157                .ok_or_else(|| eco_format!("please specify the desired version"))
158        }
159    }
160
161    /// Download the package index. The result of this is cached for efficiency.
162    fn download_index(&self) -> StrResult<&[serde_json::Value]> {
163        self.index
164            .get_or_try_init(|| {
165                let url = format!("{DEFAULT_REGISTRY}/{DEFAULT_NAMESPACE}/index.json");
166                match self.downloader.download(&url) {
167                    Ok(response) => response.into_json().map_err(|err| {
168                        eco_format!("failed to parse package index: {err}")
169                    }),
170                    Err(ureq::Error::Status(404, _)) => {
171                        bail!("failed to fetch package index (not found)")
172                    }
173                    Err(err) => bail!("failed to fetch package index ({err})"),
174                }
175            })
176            .map(AsRef::as_ref)
177    }
178
179    /// Download a package over the network.
180    ///
181    /// # Panics
182    /// Panics if the package spec namespace isn't `DEFAULT_NAMESPACE`.
183    fn download_package(
184        &self,
185        spec: &PackageSpec,
186        cache_dir: &Path,
187        progress: &mut dyn Progress,
188    ) -> PackageResult<()> {
189        assert_eq!(spec.namespace, DEFAULT_NAMESPACE);
190
191        let url = format!(
192            "{DEFAULT_REGISTRY}/{DEFAULT_NAMESPACE}/{}-{}.tar.gz",
193            spec.name, spec.version
194        );
195
196        let data = match self.downloader.download_with_progress(&url, progress) {
197            Ok(data) => data,
198            Err(ureq::Error::Status(404, _)) => {
199                if let Ok(version) = self.determine_latest_version(&spec.versionless()) {
200                    return Err(PackageError::VersionNotFound(spec.clone(), version));
201                } else {
202                    return Err(PackageError::NotFound(spec.clone()));
203                }
204            }
205            Err(err) => {
206                return Err(PackageError::NetworkFailed(Some(eco_format!("{err}"))));
207            }
208        };
209
210        // The directory in which the package's version lives.
211        let base_dir = cache_dir.join(format!("{}/{}", spec.namespace, spec.name));
212
213        // The place at which the specific package version will live in the end.
214        let package_dir = base_dir.join(format!("{}", spec.version));
215
216        // To prevent multiple Typst instances from interfering, we download
217        // into a temporary directory first and then move this directory to
218        // its final destination.
219        //
220        // In the `rename` function's documentation it is stated:
221        // > This will not work if the new name is on a different mount point.
222        //
223        // By locating the temporary directory directly next to where the
224        // package directory will live, we are (trying our best) making sure
225        // that `tempdir` and `package_dir` are on the same mount point.
226        let tempdir = Tempdir::create(base_dir.join(format!(
227            ".tmp-{}-{}",
228            spec.version,
229            fastrand::u32(..),
230        )))
231        .map_err(|err| error("failed to create temporary package directory", err))?;
232
233        // Decompress the archive into the temporary directory.
234        let decompressed = flate2::read::GzDecoder::new(data.as_slice());
235        tar::Archive::new(decompressed)
236            .unpack(&tempdir)
237            .map_err(|err| PackageError::MalformedArchive(Some(eco_format!("{err}"))))?;
238
239        // When trying to move (i.e., `rename`) the directory from one place to
240        // another and the target/destination directory is empty, then the
241        // operation will succeed (if it's atomic, or hardware doesn't fail, or
242        // power doesn't go off, etc.). If however the target directory is not
243        // empty, i.e., another instance already successfully moved the package,
244        // then we can safely ignore the `DirectoryNotEmpty` error.
245        //
246        // This means that we do not check the integrity of an existing moved
247        // package, just like we don't check the integrity if the package
248        // directory already existed in the first place. If situations with
249        // broken packages still occur even with the rename safeguard, we might
250        // consider more complex solutions like file locking or checksums.
251        match fs::rename(&tempdir, &package_dir) {
252            Ok(()) => Ok(()),
253            Err(err) if err.kind() == io::ErrorKind::DirectoryNotEmpty => Ok(()),
254            Err(err) => Err(error("failed to move downloaded package directory", err)),
255        }
256    }
257}
258
259/// Minimal information required about a package to determine its latest
260/// version.
261#[derive(Deserialize)]
262struct MinimalPackageInfo {
263    name: String,
264    version: PackageVersion,
265}
266
267/// A temporary directory that is a automatically cleaned up.
268struct Tempdir(PathBuf);
269
270impl Tempdir {
271    /// Creates a directory at the path and auto-cleans it.
272    fn create(path: PathBuf) -> io::Result<Self> {
273        std::fs::create_dir_all(&path)?;
274        Ok(Self(path))
275    }
276}
277
278impl Drop for Tempdir {
279    fn drop(&mut self) {
280        _ = fs::remove_dir_all(&self.0);
281    }
282}
283
284impl AsRef<Path> for Tempdir {
285    fn as_ref(&self) -> &Path {
286        &self.0
287    }
288}
289
290/// Enriches an I/O error with a message and turns it into a
291/// `PackageError::Other`.
292#[cold]
293fn error(message: &str, err: io::Error) -> PackageError {
294    PackageError::Other(Some(eco_format!("{message}: {err}")))
295}
296
297#[cfg(test)]
298mod tests {
299    use super::*;
300
301    #[test]
302    fn lazy_deser_index() {
303        let storage = PackageStorage::with_index(
304            None,
305            None,
306            Downloader::new("typst/test"),
307            OnceCell::with_value(vec![
308                serde_json::json!({
309                    "name": "charged-ieee",
310                    "version": "0.1.0",
311                    "entrypoint": "lib.typ",
312                }),
313                serde_json::json!({
314                    "name": "unequivocal-ams",
315                    // This version number is currently not valid, so this package
316                    // can't be parsed.
317                    "version": "0.2.0-dev",
318                    "entrypoint": "lib.typ",
319                }),
320            ]),
321        );
322
323        let ieee_version = storage.determine_latest_version(&VersionlessPackageSpec {
324            namespace: "preview".into(),
325            name: "charged-ieee".into(),
326        });
327        assert_eq!(ieee_version, Ok(PackageVersion { major: 0, minor: 1, patch: 0 }));
328
329        let ams_version = storage.determine_latest_version(&VersionlessPackageSpec {
330            namespace: "preview".into(),
331            name: "unequivocal-ams".into(),
332        });
333        assert_eq!(
334            ams_version,
335            Err("failed to find package @preview/unequivocal-ams".into())
336        )
337    }
338}
typst_kit/package.rs

typst_kit/
package.rs
