1use std::path::{Component, Path, PathBuf};
31use std::time::Duration;
32
33use anyhow::Context;
34use axum::Router;
35use axum::body::Bytes;
36use axum::extract::{Path as UrlPath, State};
37use axum::http::header::{CACHE_CONTROL, CONTENT_TYPE};
38use axum::response::{IntoResponse, Response};
39use axum::routing::get;
40
41use crate::signed::ValidSignature;
42use crate::{AppState, Config, Error, Result};
43
44#[derive(Clone)]
45enum Disk {
46 Local {
47 root: PathBuf,
48 },
49 #[cfg(feature = "s3")]
50 S3 {
51 store: std::sync::Arc<object_store::aws::AmazonS3>,
52 public_url: Option<String>,
53 },
54}
55
56#[derive(Clone)]
59pub struct Storage {
60 disk: Disk,
61 name: Option<std::sync::Arc<str>>,
63}
64
65fn check_key(key: &str) -> Result<&str> {
67 let key = key.trim_start_matches('/');
68 let clean = !key.is_empty()
69 && Path::new(key)
70 .components()
71 .all(|c| matches!(c, Component::Normal(_)))
72 && !key.contains('\\');
73 if clean && !key.contains('\0') {
74 Ok(key)
75 } else {
76 Err(Error::BadRequest(format!(
77 "invalid storage key `{}`",
78 key.escape_debug()
79 )))
80 }
81}
82
83impl Storage {
84 pub(crate) fn from_config(config: &Config) -> anyhow::Result<Self> {
85 Self::open(&config.storage, config.storage_path.join("app"), None)
86 }
87
88 pub(crate) fn named(
91 config: &Config,
92 name: &str,
93 settings: &StorageConfig,
94 ) -> anyhow::Result<Self> {
95 Self::open(settings, config.storage_path.join(name), Some(name))
96 }
97
98 fn open(
99 settings: &StorageConfig,
100 default_root: PathBuf,
101 name: Option<&str>,
102 ) -> anyhow::Result<Self> {
103 let disk = match settings.disk {
104 DiskDriver::Local => Disk::Local {
105 root: settings.root.clone().unwrap_or(default_root),
106 },
107 #[cfg(feature = "s3")]
108 DiskDriver::S3 => s3(settings)?,
109 #[cfg(not(feature = "s3"))]
110 DiskDriver::S3 => {
111 let variable = match name {
112 Some(name) => format!("the `{name}` disk's driver"),
113 None => "STORAGE_DISK".to_owned(),
114 };
115 anyhow::bail!(
116 "{variable}=s3 needs Renox's `s3` feature: renox = {{ features = [\"s3\"] }}"
117 )
118 }
119 };
120 Ok(Self {
121 disk,
122 name: name.map(Into::into),
123 })
124 }
125
126 pub fn name(&self) -> Option<&str> {
128 self.name.as_deref()
129 }
130
131 pub async fn put(&self, key: &str, bytes: Bytes) -> Result {
133 let key = check_key(key)?;
134 match &self.disk {
135 Disk::Local { root } => {
136 let path = root.join(key);
137 if let Some(dir) = path.parent() {
138 tokio::fs::create_dir_all(dir)
139 .await
140 .with_context(|| format!("could not create {}", dir.display()))?;
141 }
142 tokio::fs::write(&path, &bytes)
143 .await
144 .with_context(|| format!("could not write {}", path.display()))?;
145 }
146 #[cfg(feature = "s3")]
147 Disk::S3 { store, .. } => {
148 use object_store::ObjectStoreExt;
149 store
150 .put(&object_store::path::Path::from(key), bytes.into())
151 .await
152 .map_err(anyhow::Error::from)
153 .with_context(|| format!("could not store `{key}`"))?;
154 }
155 }
156 Ok(())
157 }
158
159 pub async fn get(&self, key: &str) -> Result<Option<Bytes>> {
161 let key = check_key(key)?;
162 match &self.disk {
163 Disk::Local { root } => match tokio::fs::read(root.join(key)).await {
164 Ok(bytes) => Ok(Some(bytes.into())),
165 Err(err) if err.kind() == std::io::ErrorKind::NotFound => Ok(None),
166 Err(err) => Err(anyhow::Error::from(err)
167 .context(format!("could not read {}", root.join(key).display()))
168 .into()),
169 },
170 #[cfg(feature = "s3")]
171 Disk::S3 { store, .. } => {
172 use object_store::ObjectStoreExt;
173 match store.get(&object_store::path::Path::from(key)).await {
174 Ok(result) => Ok(Some(result.bytes().await.map_err(anyhow::Error::from)?)),
175 Err(object_store::Error::NotFound { .. }) => Ok(None),
176 Err(err) => Err(anyhow::Error::from(err)
177 .context(format!("could not read `{key}`"))
178 .into()),
179 }
180 }
181 }
182 }
183
184 pub async fn exists(&self, key: &str) -> Result<bool> {
186 let key = check_key(key)?;
187 match &self.disk {
188 Disk::Local { root } => Ok(tokio::fs::try_exists(root.join(key)).await?),
189 #[cfg(feature = "s3")]
190 Disk::S3 { store, .. } => {
191 use object_store::ObjectStoreExt;
192 match store.head(&object_store::path::Path::from(key)).await {
193 Ok(_) => Ok(true),
194 Err(object_store::Error::NotFound { .. }) => Ok(false),
195 Err(err) => Err(anyhow::Error::from(err).into()),
196 }
197 }
198 }
199 }
200
201 pub async fn delete(&self, key: &str) -> Result {
203 let key = check_key(key)?;
204 match &self.disk {
205 Disk::Local { root } => match tokio::fs::remove_file(root.join(key)).await {
206 Ok(()) => Ok(()),
207 Err(err) if err.kind() == std::io::ErrorKind::NotFound => Ok(()),
208 Err(err) => Err(anyhow::Error::from(err)
209 .context(format!("could not delete {}", root.join(key).display()))
210 .into()),
211 },
212 #[cfg(feature = "s3")]
213 Disk::S3 { store, .. } => {
214 use object_store::ObjectStoreExt;
215 match store.delete(&object_store::path::Path::from(key)).await {
216 Ok(()) | Err(object_store::Error::NotFound { .. }) => Ok(()),
217 Err(err) => Err(anyhow::Error::from(err).into()),
218 }
219 }
220 }
221 }
222
223 pub async fn list(&self, prefix: &str) -> Result<Vec<FileInfo>> {
239 let prefix = check_prefix(prefix)?;
240 let mut files = match &self.disk {
241 Disk::Local { root } => {
242 let mut files = Vec::new();
243 let start = if prefix.is_empty() {
244 root.clone()
245 } else {
246 root.join(prefix)
247 };
248 let mut dirs = vec![start];
249 while let Some(dir) = dirs.pop() {
250 let mut entries = match tokio::fs::read_dir(&dir).await {
251 Ok(entries) => entries,
252 Err(err) if err.kind() == std::io::ErrorKind::NotFound => continue,
253 Err(err) if err.kind() == std::io::ErrorKind::NotADirectory => continue,
254 Err(err) => {
255 return Err(anyhow::Error::from(err)
256 .context(format!("could not list {}", dir.display()))
257 .into());
258 }
259 };
260 while let Some(entry) = entries.next_entry().await? {
261 let meta = entry.metadata().await?;
262 if meta.is_dir() {
263 dirs.push(entry.path());
264 } else if meta.is_file() {
265 let relative = entry.path();
266 let relative = relative.strip_prefix(root).unwrap_or(&relative);
267 let key = relative
268 .components()
269 .map(|c| c.as_os_str().to_string_lossy())
270 .collect::<Vec<_>>()
271 .join("/");
272 files.push(FileInfo {
273 key,
274 size: meta.len(),
275 modified: meta.modified().ok().map(Into::into),
276 });
277 }
278 }
279 }
280 files
281 }
282 #[cfg(feature = "s3")]
283 Disk::S3 { store, .. } => {
284 use futures_util::TryStreamExt;
285 use object_store::ObjectStore;
286 let path = (!prefix.is_empty()).then(|| object_store::path::Path::from(prefix));
287 store
288 .list(path.as_ref())
289 .map_ok(|meta| FileInfo {
290 key: meta.location.to_string(),
291 size: meta.size,
292 modified: Some(meta.last_modified),
293 })
294 .try_collect()
295 .await
296 .map_err(anyhow::Error::from)?
297 }
298 };
299 files.sort_by(|a, b| a.key.cmp(&b.key));
300 Ok(files)
301 }
302
303 pub async fn size(&self, key: &str) -> Result<Option<u64>> {
305 let key = check_key(key)?;
306 match &self.disk {
307 Disk::Local { root } => match tokio::fs::metadata(root.join(key)).await {
308 Ok(meta) if meta.is_file() => Ok(Some(meta.len())),
309 Ok(_) => Ok(None),
310 Err(err) if err.kind() == std::io::ErrorKind::NotFound => Ok(None),
311 Err(err) => Err(anyhow::Error::from(err).into()),
312 },
313 #[cfg(feature = "s3")]
314 Disk::S3 { store, .. } => {
315 use object_store::ObjectStoreExt;
316 match store.head(&object_store::path::Path::from(key)).await {
317 Ok(meta) => Ok(Some(meta.size)),
318 Err(object_store::Error::NotFound { .. }) => Ok(None),
319 Err(err) => Err(anyhow::Error::from(err).into()),
320 }
321 }
322 }
323 }
324
325 pub async fn copy(&self, from: &str, to: &str) -> Result {
328 let (from, to) = (check_key(from)?, check_key(to)?);
329 match &self.disk {
330 Disk::Local { root } => {
331 let target = root.join(to);
332 create_parent(&target).await?;
333 match tokio::fs::copy(root.join(from), &target).await {
334 Ok(_) => Ok(()),
335 Err(err) if err.kind() == std::io::ErrorKind::NotFound => Err(Error::NotFound),
336 Err(err) => Err(anyhow::Error::from(err)
337 .context(format!("could not copy `{from}` to `{to}`"))
338 .into()),
339 }
340 }
341 #[cfg(feature = "s3")]
342 Disk::S3 { store, .. } => {
343 use object_store::ObjectStoreExt;
344 let (from, to) = (
345 object_store::path::Path::from(from),
346 object_store::path::Path::from(to),
347 );
348 match store.copy(&from, &to).await {
349 Ok(()) => Ok(()),
350 Err(object_store::Error::NotFound { .. }) => Err(Error::NotFound),
351 Err(err) => Err(anyhow::Error::from(err).into()),
352 }
353 }
354 }
355 }
356
357 pub async fn rename(&self, from: &str, to: &str) -> Result {
360 let (from, to) = (check_key(from)?, check_key(to)?);
361 match &self.disk {
362 Disk::Local { root } => {
363 let target = root.join(to);
364 create_parent(&target).await?;
365 match tokio::fs::rename(root.join(from), &target).await {
366 Ok(()) => Ok(()),
367 Err(err) if err.kind() == std::io::ErrorKind::NotFound => Err(Error::NotFound),
368 Err(err) => Err(anyhow::Error::from(err)
369 .context(format!("could not move `{from}` to `{to}`"))
370 .into()),
371 }
372 }
373 #[cfg(feature = "s3")]
374 Disk::S3 { store, .. } => {
375 use object_store::ObjectStoreExt;
376 let (from, to) = (
377 object_store::path::Path::from(from),
378 object_store::path::Path::from(to),
379 );
380 match store.rename(&from, &to).await {
381 Ok(()) => Ok(()),
382 Err(object_store::Error::NotFound { .. }) => Err(Error::NotFound),
383 Err(err) => Err(anyhow::Error::from(err).into()),
384 }
385 }
386 }
387 }
388
389 pub async fn delete_all(&self, prefix: &str) -> Result<usize> {
392 if check_prefix(prefix)?.is_empty() {
393 return Err(Error::BadRequest(
394 "delete_all needs a folder, not the whole disk".into(),
395 ));
396 }
397 let files = self.list(prefix).await?;
398 for file in &files {
399 self.delete(&file.key).await?;
400 }
401 Ok(files.len())
402 }
403
404 pub fn url(&self, key: &str) -> String {
407 let key = key.trim_start_matches('/');
408 let rest = key.strip_prefix("public/").unwrap_or(key);
409 if let Some(name) = &self.name {
411 #[cfg(feature = "s3")]
412 if let Disk::S3 {
413 public_url: Some(base),
414 ..
415 } = &self.disk
416 {
417 return format!("{}/{}", base.trim_end_matches('/'), encode_path(key));
418 }
419 return format!(
420 "/_renox/disks/{}/public/{}",
421 encode_path(name),
422 encode_path(rest)
423 );
424 }
425 match &self.disk {
426 Disk::Local { .. } => format!("/storage/{}", encode_path(rest)),
427 #[cfg(feature = "s3")]
428 Disk::S3 { public_url, .. } => match public_url {
429 Some(base) => format!("{}/{}", base.trim_end_matches('/'), encode_path(key)),
430 None => format!("/storage/{}", encode_path(rest)),
431 },
432 }
433 }
434
435 pub async fn temporary_url(
438 &self,
439 state: &AppState,
440 key: &str,
441 ttl: Duration,
442 ) -> Result<String> {
443 let key = check_key(key)?;
444 match &self.disk {
445 Disk::Local { .. } => match &self.name {
446 Some(name) => state.sign_path(
447 &format!("/_renox/disks/{}/{}", encode_path(name), encode_path(key)),
448 ttl,
449 ),
450 None => state.sign_path(&format!("/_renox/files/{}", encode_path(key)), ttl),
451 },
452 #[cfg(feature = "s3")]
453 Disk::S3 { store, .. } => {
454 use object_store::signer::Signer;
455 let ttl = ttl.min(Duration::from_secs(7 * 24 * 60 * 60));
457 let url = store
458 .signed_url(
459 axum::http::Method::GET,
460 &object_store::path::Path::from(key),
461 ttl,
462 )
463 .await
464 .map_err(anyhow::Error::from)?;
465 Ok(url.to_string())
466 }
467 }
468 }
469
470 pub(crate) fn public_root(&self) -> Option<PathBuf> {
472 match &self.disk {
473 Disk::Local { root } if self.name.is_none() => Some(root.join("public")),
474 Disk::Local { .. } => None,
475 #[cfg(feature = "s3")]
476 Disk::S3 { .. } => None,
477 }
478 }
479}
480
481fn check_prefix(prefix: &str) -> Result<&str> {
483 let prefix = prefix.trim_matches('/');
484 if prefix.is_empty() {
485 return Ok(prefix);
486 }
487 check_key(prefix)
488}
489
490async fn create_parent(path: &Path) -> Result {
491 if let Some(dir) = path.parent() {
492 tokio::fs::create_dir_all(dir)
493 .await
494 .with_context(|| format!("could not create {}", dir.display()))?;
495 }
496 Ok(())
497}
498
499#[derive(Debug, Clone, serde::Serialize)]
501#[non_exhaustive]
502pub struct FileInfo {
503 pub key: String,
505 pub size: u64,
507 pub modified: Option<chrono::DateTime<chrono::Utc>>,
509}
510
511fn encode_path(key: &str) -> String {
512 let mut out = String::new();
513 crate::routing::encode(&mut out, key, true);
514 out
515}
516
517setting_enum! {
518 pub enum DiskDriver ("STORAGE_DISK") {
520 Local = "local",
522 S3 = "s3",
524 }
525}
526
527#[derive(Clone)]
529#[non_exhaustive]
530pub struct StorageConfig {
531 pub disk: DiskDriver,
533 pub bucket: Option<String>,
535 pub region: Option<String>,
537 pub endpoint: Option<String>,
539 pub access_key_id: Option<String>,
541 pub secret_access_key: Option<String>,
543 pub url: Option<String>,
545 pub root: Option<PathBuf>,
548}
549
550impl std::fmt::Debug for StorageConfig {
551 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
553 f.debug_struct("StorageConfig")
554 .field("disk", &self.disk)
555 .field("bucket", &self.bucket)
556 .field("region", &self.region)
557 .field("endpoint", &self.endpoint)
558 .field("access_key_id", &self.access_key_id)
559 .field(
560 "secret_access_key",
561 &self.secret_access_key.as_ref().map(|_| "[hidden]"),
562 )
563 .field("url", &self.url)
564 .field("root", &self.root)
565 .finish()
566 }
567}
568
569impl StorageConfig {
570 pub fn from_env(config: &Config, prefix: &str) -> Result<Self> {
577 let prefix = prefix.trim_end_matches('_').to_ascii_uppercase();
578 let var = |name: &str| config.var(&format!("{prefix}_{name}"));
579 let shared = |name: &str, fallback: &Option<String>| var(name).or_else(|| fallback.clone());
580 Ok(Self {
581 disk: var("DISK")
582 .map(|v| DiskDriver::parse_as(&v, &format!("{prefix}_DISK")))
583 .transpose()?
584 .unwrap_or(DiskDriver::Local),
585 bucket: var("BUCKET"),
586 region: shared("REGION", &config.storage.region),
587 endpoint: shared("ENDPOINT", &config.storage.endpoint),
588 access_key_id: shared("ACCESS_KEY_ID", &config.storage.access_key_id),
589 secret_access_key: shared("SECRET_ACCESS_KEY", &config.storage.secret_access_key),
590 url: var("URL"),
591 root: var("PATH").map(PathBuf::from),
592 })
593 }
594}
595
596impl Default for StorageConfig {
597 fn default() -> Self {
598 Self {
599 disk: DiskDriver::Local,
600 bucket: None,
601 region: None,
602 endpoint: None,
603 access_key_id: None,
604 secret_access_key: None,
605 url: None,
606 root: None,
607 }
608 }
609}
610
611#[cfg(feature = "s3")]
612fn s3(config: &StorageConfig) -> anyhow::Result<Disk> {
613 use anyhow::Context;
614 let mut builder = object_store::aws::AmazonS3Builder::new()
615 .with_bucket_name(
616 config
617 .bucket
618 .clone()
619 .context("S3_BUCKET is required for STORAGE_DISK=s3")?,
620 )
621 .with_region(config.region.clone().unwrap_or_else(|| "auto".into()));
622 if let Some(endpoint) = &config.endpoint {
623 builder = builder
625 .with_endpoint(endpoint)
626 .with_allow_http(endpoint.starts_with("http://"))
627 .with_virtual_hosted_style_request(false);
628 }
629 if let (Some(id), Some(secret)) = (&config.access_key_id, &config.secret_access_key) {
630 builder = builder
631 .with_access_key_id(id)
632 .with_secret_access_key(secret);
633 }
634 Ok(Disk::S3 {
635 store: std::sync::Arc::new(builder.build()?),
636 public_url: config.url.clone(),
637 })
638}
639
640pub(crate) fn router() -> Router<AppState> {
642 Router::new()
643 .route("/_renox/files/{*key}", get(private_file))
644 .route("/_renox/disks/{disk}/{*key}", get(disk_file))
645 .layer(axum::middleware::map_response(
646 crate::app::user_file_headers,
647 ))
648}
649
650async fn private_file(
651 _: ValidSignature,
652 State(state): State<AppState>,
653 UrlPath(key): UrlPath<String>,
654) -> Result<Response> {
655 file_response(&state.storage, &key).await
656}
657
658async fn disk_file(
661 State(state): State<AppState>,
662 UrlPath((disk, key)): UrlPath<(String, String)>,
663 uri: axum::http::Uri,
664) -> Result<Response> {
665 let storage = state.disks.get(&disk).ok_or(Error::NotFound)?;
666 let public = key.starts_with("public/");
667 if !public && !crate::signed::verify(&state, &uri) {
668 return Err(Error::Forbidden);
669 }
670 let mut res = file_response(storage, &key).await?;
671 if public {
672 res.headers_mut().insert(
673 CACHE_CONTROL,
674 axum::http::HeaderValue::from_static("public, max-age=3600"),
675 );
676 }
677 Ok(res)
678}
679
680async fn file_response(storage: &Storage, key: &str) -> Result<Response> {
681 let bytes = storage.get(key).await?.ok_or(Error::NotFound)?;
682 let upload = crate::upload::Upload::new(key, "", bytes.clone());
683 let content_type = upload.sniffed_type().unwrap_or("application/octet-stream");
684 Ok((
685 [
686 (CONTENT_TYPE, content_type),
687 (CACHE_CONTROL, "private, max-age=0"),
688 ],
689 bytes,
690 )
691 .into_response())
692}
693
694#[cfg(test)]
695mod tests {
696 use super::*;
697
698 #[test]
699 fn keys_cannot_escape_the_root() {
700 assert!(check_key("products/a.jpg").is_ok());
701 assert!(check_key("/products/a.jpg").is_ok());
702 for bad in ["../etc/passwd", "products/../../x", "", "a\\..\\b", "./x"] {
703 assert!(check_key(bad).is_err(), "{bad}");
704 }
705 }
706
707 fn local(root: &Path, name: Option<&str>) -> Storage {
708 let settings = StorageConfig::default();
709 Storage::open(&settings, root.to_path_buf(), name).unwrap()
710 }
711
712 #[test]
715 fn named_disks_are_named_in_errors_and_serve_nothing() {
716 let dir = tempfile::tempdir().unwrap();
717 #[cfg(not(feature = "s3"))]
718 {
719 let settings = StorageConfig {
720 disk: DiskDriver::S3,
721 ..StorageConfig::default()
722 };
723 let err = Storage::open(&settings, dir.path().to_path_buf(), Some("backups"))
724 .err()
725 .unwrap();
726 assert!(
727 err.to_string().contains("the `backups` disk's driver=s3"),
728 "{err}"
729 );
730 }
731 assert!(local(dir.path(), Some("backups")).public_root().is_none());
732 assert!(local(dir.path(), None).public_root().is_some());
733 }
734
735 #[cfg(unix)]
738 #[tokio::test]
739 async fn local_disk_errors_name_what_failed() {
740 let dir = tempfile::tempdir().unwrap();
741 let disk = local(dir.path(), None);
742 disk.put("docs/a.txt", Bytes::from_static(b"a"))
743 .await
744 .unwrap();
745 disk.put("full/b.txt", Bytes::from_static(b"b"))
746 .await
747 .unwrap();
748 let err = disk.get("docs").await.unwrap_err();
750 assert!(format!("{err:?}").contains("could not read"), "{err:?}");
751 assert!(disk.size("docs/a.txt/inner").await.is_err());
753 let err = disk.copy("docs", "copy.txt").await.unwrap_err();
754 assert!(
755 format!("{err:?}").contains("could not copy `docs`"),
756 "{err:?}"
757 );
758 let err = disk.rename("docs/a.txt", "full").await.unwrap_err();
760 assert!(
761 format!("{err:?}").contains("could not move `docs/a.txt`"),
762 "{err:?}"
763 );
764 std::os::unix::fs::symlink(dir.path().join("docs/loop"), dir.path().join("docs/loop"))
767 .unwrap();
768 let keys: Vec<_> = disk
769 .list("docs")
770 .await
771 .unwrap()
772 .into_iter()
773 .map(|f| f.key)
774 .collect();
775 assert_eq!(keys, ["docs/a.txt"]);
776 let err = disk.list("docs/loop").await.unwrap_err();
777 assert!(format!("{err:?}").contains("could not list"), "{err:?}");
778 }
779}