1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
use crate::client::{create_http_client_with_options, HttpOptions};
use crate::endpoints::*;
use crate::error::WarpgateError;
use crate::helpers::{
    determine_cache_extension, download_from_url_to_file, move_or_unpack_download,
};
use crate::id::Id;
use once_cell::sync::OnceCell;
use sha2::{Digest, Sha256};
use starbase_styles::color;
use starbase_utils::fs;
use std::env;
use std::path::{Path, PathBuf};
use std::sync::Arc;
use std::time::{Duration, SystemTime};
use tracing::trace;
use warpgate_api::{GitHubLocator, PluginLocator};

pub type OfflineChecker = Arc<fn() -> bool>;

/// A system for loading plugins from a locator strategy,
/// and caching the `.wasm` file to the host's file system.
#[derive(Clone)]
pub struct PluginLoader {
    /// Instance of our HTTP client.
    http_client: OnceCell<reqwest::Client>,

    /// Options to pass to the HTTP client.
    http_options: HttpOptions,

    /// Checks whether there's an internet connection or not.
    offline_checker: Option<OfflineChecker>,

    /// Location where downloaded `.wasm` plugins are stored.
    plugins_dir: PathBuf,

    /// Location where temporary files (like archives) are stored.
    temp_dir: PathBuf,

    /// A unique seed for generating hashes.
    seed: Option<String>,
}

impl PluginLoader {
    /// Create a new loader that stores plugins and downloads in the provided directories.
    pub fn new<P: AsRef<Path>, T: AsRef<Path>>(plugins_dir: P, temp_dir: T) -> Self {
        let plugins_dir = plugins_dir.as_ref();

        trace!(cache_dir = ?plugins_dir, "Creating plugin loader");

        Self {
            http_client: OnceCell::new(),
            http_options: HttpOptions::default(),
            offline_checker: None,
            plugins_dir: plugins_dir.to_owned(),
            temp_dir: temp_dir.as_ref().to_owned(),
            seed: None,
        }
    }

    /// Return the HTTP client, or create it if it does not exist.
    pub fn get_client(&self) -> miette::Result<&reqwest::Client> {
        self.http_client
            .get_or_try_init(|| create_http_client_with_options(&self.http_options))
    }

    /// Load a plugin using the provided locator. File system plugins are loaded directly,
    /// while remote/URL plugins are downloaded and cached.
    pub async fn load_plugin<I: AsRef<Id>, L: AsRef<PluginLocator>>(
        &self,
        id: I,
        locator: L,
    ) -> miette::Result<PathBuf> {
        let id = id.as_ref();
        let locator = locator.as_ref();

        trace!(
            id = id.as_str(),
            "Loading plugin {}",
            color::id(id.as_str())
        );

        match locator {
            PluginLocator::SourceFile { path, .. } => {
                let path = path
                    .canonicalize()
                    .map_err(|_| WarpgateError::SourceFileMissing {
                        id: id.to_owned(),
                        path: path.to_path_buf(),
                    })?;

                if path.exists() {
                    trace!(
                        id = id.as_str(),
                        path = ?path,
                        "Using source file",
                    );

                    Ok(path)
                } else {
                    Err(WarpgateError::SourceFileMissing {
                        id: id.to_owned(),
                        path: path.to_path_buf(),
                    }
                    .into())
                }
            }
            PluginLocator::SourceUrl { url } => {
                self.download_plugin(
                    id,
                    url,
                    self.create_cache_path(id, url, url.contains("latest")),
                )
                .await
            }
            PluginLocator::GitHub(github) => self.download_plugin_from_github(id, github).await,
        }
    }

    /// Create an absolute path to the plugin's destination file, located in the plugins directory.
    /// Hash the source URL to ensure uniqueness of each plugin + version combination.
    pub fn create_cache_path(&self, id: &Id, url: &str, is_latest: bool) -> PathBuf {
        let mut sha = Sha256::new();
        sha.update(url);

        if let Some(seed) = &self.seed {
            sha.update(seed);
        }

        // Remove unwanted or unsafe file name characters
        let safe_id = id.as_str().replace(['/', '@', '.', ' '], "");

        self.plugins_dir.join(format!(
            "{}{}{:x}{}",
            safe_id,
            if is_latest { "-latest-" } else { "-" },
            sha.finalize(),
            determine_cache_extension(url)
        ))
    }

    /// Check if the plugin has been downloaded and is cached.
    /// If using a latest strategy (no explicit version or tag), the cache
    /// is only valid for 7 days (to ensure not stale), otherwise forever.
    pub fn is_cached(&self, id: &Id, path: &Path) -> miette::Result<bool> {
        if !path.exists() {
            trace!(id = id.as_str(), "Plugin not cached, downloading");

            return Ok(false);
        }

        let metadata = fs::metadata(path)?;

        let mut cached = if let Ok(filetime) = metadata.created().or_else(|_| metadata.modified()) {
            let days = if fs::file_name(path).contains("-latest-") {
                7
            } else {
                30
            };

            filetime > SystemTime::now() - Duration::from_secs(86400 * days)
        } else {
            false
        };

        if !cached && self.is_offline() {
            cached = true;
        }

        if !cached {
            fs::remove_file(path)?;
        }

        if cached {
            trace!(id = id.as_str(), path = ?path, "Plugin already downloaded and cached");
        } else {
            trace!(id = id.as_str(), path = ?path, "Plugin cached but stale, re-downloading");
        }

        Ok(cached)
    }

    /// Check for an internet connection.
    pub fn is_offline(&self) -> bool {
        self.offline_checker
            .as_ref()
            .map(|op| op())
            .unwrap_or_default()
    }

    /// Set the options to pass to the HTTP client.
    pub fn set_client_options(&mut self, options: &HttpOptions) {
        self.http_options = options.to_owned();
    }

    /// Set the function that checks for offline state.
    pub fn set_offline_checker(&mut self, op: fn() -> bool) {
        self.offline_checker = Some(Arc::new(op));
    }

    /// Set the provided value as a seed for generating hashes.
    pub fn set_seed(&mut self, value: &str) {
        self.seed = Some(value.to_owned());
    }

    async fn download_plugin(
        &self,
        id: &Id,
        source_url: &str,
        dest_file: PathBuf,
    ) -> miette::Result<PathBuf> {
        if self.is_cached(id, &dest_file)? {
            return Ok(dest_file);
        }

        if self.is_offline() {
            return Err(WarpgateError::InternetConnectionRequired {
                message: "Unable to download plugin.".into(),
                url: source_url.to_owned(),
            }
            .into());
        }

        trace!(
            id = id.as_str(),
            from = source_url,
            to = ?dest_file,
            "Downloading plugin from URL"
        );

        let temp_file = self.temp_dir.join(fs::file_name(&dest_file));

        download_from_url_to_file(source_url, &temp_file, self.get_client()?).await?;
        move_or_unpack_download(&temp_file, &dest_file)?;

        Ok(dest_file)
    }

    async fn download_plugin_from_github(
        &self,
        id: &Id,
        github: &GitHubLocator,
    ) -> miette::Result<PathBuf> {
        let (api_url, release_tag) = if let Some(tag) = &github.tag {
            (
                format!(
                    "https://api.github.com/repos/{}/releases/tags/{tag}",
                    github.repo_slug,
                ),
                tag.to_owned(),
            )
        } else {
            (
                format!(
                    "https://api.github.com/repos/{}/releases/latest",
                    github.repo_slug,
                ),
                "latest".to_owned(),
            )
        };

        // Check the cache first using the API URL as the seed,
        // so that we can avoid making unnecessary HTTP requests.
        let plugin_path = self.create_cache_path(id, &api_url, release_tag == "latest");

        if self.is_cached(id, &plugin_path)? {
            return Ok(plugin_path);
        }

        trace!(
            id = id.as_str(),
            api_url = &api_url,
            release_tag = &release_tag,
            "Attempting to download plugin from GitHub release",
        );

        let handle_error = |error: reqwest::Error| WarpgateError::Http {
            error,
            url: api_url.clone(),
        };

        if self.is_offline() {
            return Err(WarpgateError::InternetConnectionRequired {
                message: format!(
                    "Unable to download plugin {} from GitHub.",
                    PluginLocator::GitHub(github.to_owned())
                ),
                url: api_url,
            }
            .into());
        }

        // Otherwise make an HTTP request to the GitHub releases API,
        // and loop through the assets to find a matching one.
        let mut request = self.get_client()?.get(&api_url);

        if let Ok(auth_token) = env::var("GITHUB_TOKEN") {
            request = request.bearer_auth(auth_token);
        }

        let response = request.send().await.map_err(handle_error)?;
        let release: GitHubApiRelease = response.json().await.map_err(handle_error)?;

        // Find a direct WASM asset first
        for asset in &release.assets {
            if asset.content_type == "application/wasm" || asset.name.ends_with(".wasm") {
                trace!(
                    id = id.as_str(),
                    asset = &asset.name,
                    "Found WASM asset with application/wasm content type"
                );

                return self
                    .download_plugin(id, &asset.browser_download_url, plugin_path)
                    .await;
            }
        }

        // Otherwise an asset with a matching name and supported extension
        for asset in release.assets {
            if asset.name == github.file_prefix
                || (asset.name.starts_with(&github.file_prefix)
                    && (asset.name.ends_with(".tar")
                        | asset.name.ends_with(".tar.gz")
                        | asset.name.ends_with(".tgz")
                        | asset.name.ends_with(".tar.xz")
                        | asset.name.ends_with(".txz")
                        | asset.name.ends_with(".zst")
                        | asset.name.ends_with(".zstd")
                        | asset.name.ends_with(".zip")))
            {
                trace!(
                    id = id.as_str(),
                    asset = &asset.name,
                    "Found possible asset as an archive"
                );

                return self
                    .download_plugin(id, &asset.browser_download_url, plugin_path)
                    .await;
            }
        }

        Err(WarpgateError::GitHubAssetMissing {
            id: id.to_owned(),
            repo_slug: github.repo_slug.to_owned(),
            tag: release_tag,
        }
        .into())
    }
}