//! Uploaded files: the [`File`] field type, the rules an upload must pass, and the stores that
//! keep the bytes.
//!
//! An app registers a store on a feature with its rules and a permission, and puts the same
//! rules on the form field:
//!
//! ```ignore
//! pub const CONTRACTS: FileRules = FileRules::new("contracts")
//! .max_bytes(5 * 1024 * 1024)
//! .accept(&[FileType::Pdf, FileType::Png, FileType::Jpeg]);
//!
//! Feature::new("crm").files(CONTRACTS, "crm.contract.write", DiskStore::new("data/contracts"))
//! Form::of(customer).file(Customer::contract, CONTRACTS)
//! ```
//!
//! The browser sends each file to `POST /_zeyn/files/{store}`. The server checks the CSRF token,
//! the permission and the feature's guards, runs the interceptors, reads the first bytes to learn
//! the real type (the name and the browser's content type are not trusted), streams the rest to
//! the store in bounded chunks while counting the size, runs the feature's scanner, and answers
//! with a [`File`] whose id it generated. Clients never choose ids or paths. Downloads go through
//! `GET /_zeyn/files/{store}/{id}` with the same checks and the ranged serving of media.
use crate::i18n::Text;
use crate::value::{FieldKind, FieldType, SupportsRequired, Value};
use serde::{Deserialize, Serialize};
/// Largest file any upload may have, whatever the rules say: 1 GiB.
pub const MAX_UPLOAD_BYTES: u64 = 1 << 30;
/// Most files one field may hold, whatever the rules say.
pub const MAX_FIELD_FILES: u32 = 100;
/// Longest file name kept, in characters. Longer names are cut.
pub const MAX_FILE_NAME: usize = 200;
/// Bytes read from the start of an upload to learn its type.
pub const SNIFF_BYTES: usize = 4096;
/// Bytes passed to a store at a time.
pub const UPLOAD_CHUNK: usize = 64 * 1024;
/// A file kept by a [`FileStore`], as a form field holds it. The id is generated by the server;
/// the name, size and type are for display. Downloads always use what the store recorded, so a
/// client that edits these values changes nothing but its own view.
#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)]
pub struct File {
pub id: String,
pub name: String,
pub size: u64,
#[serde(rename = "type")]
pub content_type: String,
}
impl File {
/// Path of the download below the app's base, such as `/_zeyn/files/contracts/3f...`.
pub fn url(&self, store: &str) -> String {
format!("/_zeyn/files/{store}/{}", self.id)
}
}
/// One uploaded file.
impl FieldType for File {
const KIND: FieldKind = FieldKind::Custom { editor: "file" };
fn to_value(&self) -> Value {
Value::Text(self.name.clone())
}
fn __is_empty(&self) -> bool {
self.id.is_empty()
}
}
/// Several uploaded files, such as the attachments of a ticket.
impl FieldType for Vec<File> {
const KIND: FieldKind = FieldKind::Custom { editor: "files" };
fn to_value(&self) -> Value {
Value::Text(self.iter().map(|f| f.name.as_str()).collect::<Vec<_>>().join(", "))
}
fn __is_empty(&self) -> bool {
self.is_empty()
}
}
impl SupportsRequired for Vec<File> {}
/// Field types an upload control can fill.
#[diagnostic::on_unimplemented(
message = "`{Self}` cannot be filled by an upload",
label = "not a file field",
note = "`Form::file` works on `zeyn::File`, `Option<File>` and `Vec<File>` fields"
)]
pub trait UploadValue {
/// Whether the field holds several files.
const MANY: bool;
}
impl UploadValue for File {
const MANY: bool = false;
}
impl UploadValue for Option<File> {
const MANY: bool = false;
}
impl UploadValue for Vec<File> {
const MANY: bool = true;
}
/// A file type an upload may have. Each type is recognized by its content, and the name's
/// extension must belong to the same type: a PDF renamed to `.png` is refused. HTML, SVG, XML
/// and scripts are never accepted.
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize)]
#[serde(rename_all = "snake_case")]
pub enum FileType {
Png,
Jpeg,
Gif,
Webp,
Pdf,
/// A plain ZIP archive.
Zip,
/// Word document (a ZIP container).
Docx,
/// Excel workbook (a ZIP container).
Xlsx,
/// PowerPoint presentation (a ZIP container).
Pptx,
/// UTF-8 text with a `.csv` name.
Csv,
/// UTF-8 text with a `.txt` name.
Text,
Mp4,
Webm,
Mp3,
Wav,
}
/// How the first bytes of a file look.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
enum Content {
Png,
Jpeg,
Gif,
Webp,
Pdf,
Zip,
Mp4,
Webm,
Mp3,
Wav,
Text,
/// HTML, SVG, XML or a script: refused whatever the name says.
Markup,
Unknown,
}
impl FileType {
/// Every type, for listings.
pub const ALL: &'static [FileType] = &[
FileType::Png,
FileType::Jpeg,
FileType::Gif,
FileType::Webp,
FileType::Pdf,
FileType::Zip,
FileType::Docx,
FileType::Xlsx,
FileType::Pptx,
FileType::Csv,
FileType::Text,
FileType::Mp4,
FileType::Webm,
FileType::Mp3,
FileType::Wav,
];
/// The content type the file is stored and served with.
pub const fn mime(self) -> &'static str {
match self {
FileType::Png => "image/png",
FileType::Jpeg => "image/jpeg",
FileType::Gif => "image/gif",
FileType::Webp => "image/webp",
FileType::Pdf => "application/pdf",
FileType::Zip => "application/zip",
FileType::Docx => "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
FileType::Xlsx => "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
FileType::Pptx => "application/vnd.openxmlformats-officedocument.presentationml.presentation",
FileType::Csv => "text/csv; charset=utf-8",
FileType::Text => "text/plain; charset=utf-8",
FileType::Mp4 => "video/mp4",
FileType::Webm => "video/webm",
FileType::Mp3 => "audio/mpeg",
FileType::Wav => "audio/wav",
}
}
/// File name extensions of the type, lower case, without the dot.
pub const fn extensions(self) -> &'static [&'static str] {
match self {
FileType::Png => &["png"],
FileType::Jpeg => &["jpg", "jpeg"],
FileType::Gif => &["gif"],
FileType::Webp => &["webp"],
FileType::Pdf => &["pdf"],
FileType::Zip => &["zip"],
FileType::Docx => &["docx"],
FileType::Xlsx => &["xlsx"],
FileType::Pptx => &["pptx"],
FileType::Csv => &["csv"],
FileType::Text => &["txt"],
FileType::Mp4 => &["mp4", "m4v"],
FileType::Webm => &["webm"],
FileType::Mp3 => &["mp3"],
FileType::Wav => &["wav"],
}
}
fn content(self) -> Content {
match self {
FileType::Png => Content::Png,
FileType::Jpeg => Content::Jpeg,
FileType::Gif => Content::Gif,
FileType::Webp => Content::Webp,
FileType::Pdf => Content::Pdf,
FileType::Zip | FileType::Docx | FileType::Xlsx | FileType::Pptx => Content::Zip,
FileType::Csv | FileType::Text => Content::Text,
FileType::Mp4 => Content::Mp4,
FileType::Webm => Content::Webm,
FileType::Mp3 => Content::Mp3,
FileType::Wav => Content::Wav,
}
}
/// The type of a file from its name and its first bytes, when both agree. `None` for
/// markup, unknown content, or an extension of another type.
///
/// ```
/// use zeyn::FileType;
/// assert_eq!(FileType::detect("scan.pdf", b"%PDF-1.7\n"), Some(FileType::Pdf));
/// assert_eq!(FileType::detect("scan.png", b"%PDF-1.7\n"), None);
/// assert_eq!(FileType::detect("page.txt", b"<html><script>x()</script>"), None);
/// assert_eq!(FileType::detect("list.CSV", b"code;name\nA;B\n"), Some(FileType::Csv));
/// ```
pub fn detect(name: &str, head: &[u8]) -> Option<FileType> {
let ext = name.rsplit_once('.').map(|(_, e)| e.to_ascii_lowercase())?;
let content = sniff(head);
FileType::ALL.iter().copied().find(|t| t.content() == content && t.extensions().contains(&ext.as_str()))
}
}
/// What the first bytes of a file are.
fn sniff(head: &[u8]) -> Content {
let starts = |magic: &[u8]| head.starts_with(magic);
if starts(b"\x89PNG\r\n\x1a\n") {
Content::Png
} else if starts(&[0xFF, 0xD8, 0xFF]) {
Content::Jpeg
} else if starts(b"GIF87a") || starts(b"GIF89a") {
Content::Gif
} else if starts(b"RIFF") && head.get(8..12) == Some(b"WEBP") {
Content::Webp
} else if starts(b"RIFF") && head.get(8..12) == Some(b"WAVE") {
Content::Wav
} else if starts(b"%PDF-") {
Content::Pdf
} else if starts(b"PK\x03\x04") || starts(b"PK\x05\x06") {
Content::Zip
} else if head.get(4..8) == Some(b"ftyp") {
Content::Mp4
} else if starts(&[0x1A, 0x45, 0xDF, 0xA3]) {
Content::Webm
} else if starts(b"ID3") || (head.len() > 1 && head[0] == 0xFF && head[1] & 0xE0 == 0xE0) {
Content::Mp3
} else {
text(head)
}
}
/// Text when the bytes are UTF-8 (a character cut at the end is fine) without control characters
/// other than tab and line breaks; markup when such text looks like HTML, SVG, XML or a script.
fn text(head: &[u8]) -> Content {
let valid = match std::str::from_utf8(head) {
Ok(s) => s,
Err(e) if e.error_len().is_none() => std::str::from_utf8(&head[..e.valid_up_to()]).unwrap_or(""),
Err(_) => return Content::Unknown,
};
if valid.is_empty() || valid.chars().any(|c| c.is_control() && !matches!(c, '\t' | '\n' | '\r')) {
return Content::Unknown;
}
let lower = valid.trim_start_matches('\u{feff}').trim_start().to_lowercase();
let markup = ["<!doctype", "<html", "<svg", "<?xml", "<script", "<body", "<iframe", "<head", "<object", "<embed"];
if lower.starts_with('<') || markup.iter().any(|m| lower.contains(m)) { Content::Markup } else { Content::Text }
}
/// What an upload field accepts: the store it goes to, the largest file, how many files a field
/// may hold and which types. Build it as a `const` and use the same value on the feature and
/// the form, so the browser shows the limits the server checks.
///
/// ```
/// use zeyn::{FileRules, FileType};
/// const PHOTOS: FileRules = FileRules::new("photos").max_bytes(2_000_000).max_files(4)
/// .accept(&[FileType::Png, FileType::Jpeg]);
/// assert_eq!(PHOTOS.limit(), 2_000_000);
/// assert!(PHOTOS.allows(FileType::Png) && !PHOTOS.allows(FileType::Pdf));
/// ```
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub struct FileRules {
store: &'static str,
max_bytes: u64,
max_files: u32,
accept: &'static [FileType],
}
impl FileRules {
/// Rules for the store named `store` (letters, digits, `-`, `_` and `.`): 10 MiB, 10 files,
/// and PNG, JPEG, WebP and PDF until [`FileRules::accept`] says otherwise.
pub const fn new(store: &'static str) -> Self {
FileRules {
store,
max_bytes: 10 * 1024 * 1024,
max_files: 10,
accept: &[FileType::Png, FileType::Jpeg, FileType::Webp, FileType::Pdf],
}
}
/// Largest file, in bytes. Never more than [`MAX_UPLOAD_BYTES`].
pub const fn max_bytes(mut self, bytes: u64) -> Self {
self.max_bytes = if bytes > MAX_UPLOAD_BYTES { MAX_UPLOAD_BYTES } else { bytes };
self
}
/// Most files a `Vec<File>` field may hold, 1 to [`MAX_FIELD_FILES`].
pub const fn max_files(mut self, n: u32) -> Self {
self.max_files = if n == 0 {
1
} else if n > MAX_FIELD_FILES {
MAX_FIELD_FILES
} else {
n
};
self
}
/// The types accepted.
pub const fn accept(mut self, types: &'static [FileType]) -> Self {
self.accept = types;
self
}
pub const fn store(&self) -> &'static str {
self.store
}
/// Largest file in bytes.
pub const fn limit(&self) -> u64 {
self.max_bytes
}
pub const fn files(&self) -> u32 {
self.max_files
}
pub const fn types(&self) -> &'static [FileType] {
self.accept
}
pub fn allows(&self, t: FileType) -> bool {
self.accept.contains(&t)
}
/// The `accept` attribute of the file input: extensions and content types.
pub(crate) fn accept_attr(&self) -> String {
let mut parts: Vec<String> = Vec::new();
for t in self.accept {
parts.extend(t.extensions().iter().map(|e| format!(".{e}")));
parts.push(t.mime().split(';').next().unwrap_or("").to_string());
}
parts.join(",")
}
}
/// Whether a file id has the shape the server generates: 32 lower case hex digits.
///
/// ```
/// assert!(zeyn::valid_file_id("0123456789abcdef0123456789abcdef"));
/// assert!(!zeyn::valid_file_id("../../etc/passwd"));
/// ```
pub fn valid_file_id(id: &str) -> bool {
id.len() == 32 && id.bytes().all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b))
}
/// A file name safe to keep and show: the part after the last `/` or `\`, without control
/// characters, cut to [`MAX_FILE_NAME`] characters, never empty, never starting with a dot.
///
/// ```
/// assert_eq!(zeyn::clean_file_name("../../etc/passwd"), "passwd");
/// assert_eq!(zeyn::clean_file_name("C:\\Users\\a\\rapor.pdf"), "rapor.pdf");
/// assert_eq!(zeyn::clean_file_name(".env"), "env");
/// assert_eq!(zeyn::clean_file_name(""), "file");
/// ```
pub fn clean_file_name(name: &str) -> String {
let base = name.rsplit(['/', '\\']).next().unwrap_or("");
let cleaned: String = base.chars().filter(|c| !c.is_control()).take(MAX_FILE_NAME).collect();
let cleaned = cleaned.trim().trim_start_matches('.').trim().to_string();
if cleaned.is_empty() { "file".to_string() } else { cleaned }
}
/// Reads a submitted `File` or `Vec<File>` value: a JSON text from the form, or JSON from the
/// API. Ids must have the server's shape and names are cleaned; anything else is refused.
pub(crate) fn parse_files(raw: &serde_json::Value, many: bool, optional: bool) -> Option<serde_json::Value> {
use serde_json::Value as Json;
let value = match raw {
Json::String(s) if s.trim().is_empty() => Json::Null,
Json::String(s) => serde_json::from_str(s).ok()?,
other => other.clone(),
};
let check = |f: File| {
let content_type = FileType::ALL.iter().map(|t| t.mime()).find(|m| *m == f.content_type).unwrap_or("");
valid_file_id(&f.id).then(|| File { name: clean_file_name(&f.name), content_type: content_type.into(), ..f })
};
if many {
let list: Vec<File> = match value {
Json::Null => Vec::new(),
v => serde_json::from_value(v).ok()?,
};
if list.len() > MAX_FIELD_FILES as usize {
return None;
}
let list: Option<Vec<File>> = list.into_iter().map(check).collect();
serde_json::to_value(list?).ok()
} else {
match value {
Json::Null if optional => Some(Json::Null),
Json::Null => serde_json::to_value(File::default()).ok(),
v => serde_json::to_value(check(serde_json::from_value(v).ok()?)?).ok(),
}
}
}
/// Size in the user's language: `840 B`, `12,5 KB`, `3.2 MB`.
pub(crate) fn human_size(bytes: u64, locale: &str) -> String {
let units = ["B", "KB", "MB", "GB"];
let mut v = bytes as f64;
let mut u = 0;
while v >= 1024.0 && u < units.len() - 1 {
v /= 1024.0;
u += 1;
}
let text = if u == 0 { format!("{bytes}") } else { format!("{v:.1}") };
let decimal = crate::i18n::NumberFormat::for_locale(locale).decimal;
format!("{} {}", text.replace('.', &decimal.to_string()), units[u])
}
/// Message key and arguments when a file is refused.
pub(crate) fn too_large(rules: &FileRules, locale: &str) -> Text {
Text::key("zeyn.files.too_large").arg("max", human_size(rules.limit(), locale))
}
/// Writes an upload control: a text input holding the field's JSON value (what the form
/// submits), a native file input inside a drop zone, the list of files and a live region. The
/// runtime (`files.js`) uploads chosen files and keeps the value in step.
pub(crate) fn render_upload(
out: &mut String,
r: &crate::render::Render,
m: &crate::model::FieldMeta,
id: &str,
rules: &FileRules,
many: bool,
current: &serde_json::Value,
) {
use crate::render::{attr, escape};
let t = |k: &str| r.text(&Text::key(format!("zeyn.ui.upload.{k}")));
let files: Vec<File> = match current {
serde_json::Value::Array(_) => serde_json::from_value(current.clone()).unwrap_or_default(),
serde_json::Value::Object(_) => serde_json::from_value::<File>(current.clone()).into_iter().collect(),
_ => Vec::new(),
};
let files: Vec<File> = files.into_iter().filter(|f| valid_file_id(&f.id)).collect();
let value = if many {
serde_json::to_string(&files).unwrap_or_default()
} else {
files.first().and_then(|f| serde_json::to_string(f).ok()).unwrap_or_default()
};
let types: Vec<String> =
rules.types().iter().flat_map(|t| t.extensions().first()).map(|e| e.to_uppercase()).collect();
let hint = r.text(
&Text::key(if many { "zeyn.ui.upload.hint_many" } else { "zeyn.ui.upload.hint" })
.arg("types", types.join(", "))
.arg("max", human_size(rules.limit(), r.locale))
.arg("n", rules.files().to_string()),
);
let raw = |text: &Text| r.i18n.text(r.locale, text);
let key = |k: &str| raw(&Text::key(format!("zeyn.ui.upload.{k}")));
let texts = serde_json::json!({
"uploading": key("uploading"),
"uploaded": key("uploaded"),
"failed": key("failed"),
"canceled": key("canceled"),
"removed": key("removed"),
"cancel": key("cancel"),
"retry": key("retry"),
"remove": key("remove"),
"remove_name": key("remove_name"),
"too_large": raw(&too_large(rules, r.locale)),
"too_many": raw(&Text::key("zeyn.files.too_many").arg("label", m.label_text()).arg("n", rules.files().to_string())),
"type": raw(&Text::key("zeyn.files.type")),
"aborted": raw(&Text::key("zeyn.files.aborted")),
});
let required = if m.is_required() { r#" aria-required="true""# } else { "" };
let star = if m.is_required() { r#"<span class="z-required" aria-hidden="true">*</span>"# } else { "" };
let multiple = if many { " multiple" } else { "" };
out.push_str(&format!(
r#"<label for="{id}-pick" id="{id}-label">{label}{star}</label><div class="z-upload" data-zeyn-upload="{store}" data-multiple="{many}" data-max-bytes="{max}" data-max-files="{n}" data-texts="{texts}"><input type="text" hidden class="z-upload-value" id="{id}" name="{name}" data-testid="{scope}.field.{name}" value="{value}"><div class="z-upload-drop"><input type="file" class="z-upload-input" id="{id}-pick" accept="{accept}"{multiple}{required} aria-describedby="{id}-hint {id}-error"><p class="z-upload-hint" id="{id}-hint">{drop} {hint}</p></div><ul class="z-upload-list" aria-label="{list}">"#,
label = r.label(m),
store = escape(rules.store()),
max = rules.limit(),
n = if many { rules.files() } else { 1 },
texts = attr(&texts),
name = m.name,
scope = r.scope,
value = escape(&value),
accept = escape(&rules.accept_attr()),
drop = t("drop"),
list = t("list"),
));
for f in &files {
out.push_str(&format!(
r#"<li class="z-upload-item" data-id="{id}" data-type="{ty}"><a class="z-upload-name" data-zeyn-href="{url}" download>{name}</a><span class="z-upload-size">{size}</span><button type="button" class="z-button z-upload-remove" aria-label="{remove_name}">{remove}</button></li>"#,
id = escape(&f.id),
ty = escape(&f.content_type),
url = escape(&f.url(rules.store())),
name = escape(&f.name),
size = escape(&human_size(f.size, r.locale)),
remove = t("remove"),
remove_name = r.text(&Text::key("zeyn.ui.upload.remove_name").arg("name", f.name.clone())),
));
}
out.push_str(r#"</ul><p class="z-sr-only z-upload-status" role="status" aria-live="polite"></p></div>"#);
}
#[cfg(feature = "server")]
pub use store::*;
#[cfg(feature = "server")]
mod store {
use super::{File, FileRules, valid_file_id};
use crate::action::{Error, Result};
use crate::ctx::BoxFuture;
use crate::i18n::Text;
use crate::media::MediaReader;
use axum::body::Bytes;
use serde::{Deserialize, Serialize};
use std::collections::HashMap;
use std::io::{Cursor, Write};
use std::path::PathBuf;
use std::sync::{Arc, Mutex};
/// What a store records about a file. The server fills it; the store keeps it with the bytes
/// and gives it back with [`FileStore::get`] and [`FileStore::meta`].
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct FileMeta {
/// Generated by the server, see [`valid_file_id`].
pub id: String,
/// Cleaned name from the client, for display and downloads only.
pub name: String,
/// The type found in the content, never the one the client sent.
pub content_type: String,
/// Bytes written. The store sets it when the upload ends.
pub size: u64,
/// Tenant of the user who uploaded it; downloads from other tenants are refused.
pub tenant: Option<String>,
/// User who uploaded it.
pub owner: Option<String>,
/// Seconds since 1970 when the upload started.
pub created: u64,
}
impl FileMeta {
/// The form value for this file.
pub fn file(&self) -> File {
File {
id: self.id.clone(),
name: self.name.clone(),
size: self.size,
content_type: self.content_type.clone(),
}
}
}
/// The body of an upload, in chunks of at most [`super::UPLOAD_CHUNK`] bytes. An `Err` means
/// the upload stopped (the client left or a limit was passed): the store must throw away
/// what it wrote and return an error.
pub struct FileChunks(pub(crate) tokio::sync::mpsc::Receiver<std::io::Result<Bytes>>);
impl FileChunks {
/// The next chunk, `None` at the end.
pub async fn next(&mut self) -> Option<std::io::Result<Bytes>> {
self.0.recv().await
}
/// [`FileChunks::next`] for code on the blocking pool (`tokio::task::spawn_blocking`).
pub fn blocking_next(&mut self) -> Option<std::io::Result<Bytes>> {
self.0.blocking_recv()
}
/// Chunks from memory, for tests of a store.
pub fn from_parts(parts: Vec<Vec<u8>>) -> Self {
let (tx, rx) = tokio::sync::mpsc::channel(parts.len().max(1));
for p in parts {
let _ = tx.try_send(Ok(Bytes::from(p)));
}
FileChunks(rx)
}
}
/// A stored file opened for reading.
pub struct StoredFile {
pub meta: FileMeta,
pub reader: Box<dyn MediaReader>,
}
/// Where uploaded files are kept. Registered with [`Feature::files`](crate::Feature::files).
///
/// Ids reaching a store have passed [`valid_file_id`]; a store never builds a path from
/// anything else. Unknown ids are [`Error::NotFound`].
pub trait FileStore: Send + Sync + 'static {
/// Writes a new file. `meta.size` is 0; the store records the bytes it wrote. On an
/// `Err` chunk the store removes what it wrote and fails.
fn put(&self, meta: FileMeta, chunks: FileChunks) -> BoxFuture<Result<()>>;
/// Opens a file for reading, with its metadata.
fn get(&self, id: String) -> BoxFuture<Result<StoredFile>>;
/// The metadata of a file.
fn meta(&self, id: String) -> BoxFuture<Result<FileMeta>>;
/// Removes a file. Removing an unknown id is not an error.
fn delete(&self, id: String) -> BoxFuture<Result<()>>;
}
/// Whether a scanned file may be kept.
#[derive(Clone, Debug, PartialEq, Eq)]
pub enum ScanVerdict {
Clean,
/// Refused, with the reason for the log. The user sees a general message.
Refused(String),
}
/// Checks an uploaded file before it is accepted, such as a virus scanner. Registered with
/// [`Feature::scan_files`](crate::Feature::scan_files). A refused file, or a scanner that
/// fails, is deleted and the upload answers `422`.
pub trait FileScanner: Send + Sync + 'static {
fn scan(&self, meta: FileMeta, file: Box<dyn MediaReader>) -> BoxFuture<Result<ScanVerdict>>;
}
/// A store registered on a feature.
#[derive(Clone)]
pub struct FileEntry {
pub rules: FileRules,
pub permission: &'static str,
pub store: Arc<dyn FileStore>,
}
fn not_found() -> Error {
Error::NotFound
}
fn io(e: std::io::Error) -> Error {
Error::Internal(format!("file store: {e}"))
}
/// Files on the local disk under one folder: `{id}.bin` holds the bytes and `{id}.json` the
/// metadata. Uploads are written to `{id}.part` and renamed when complete, so a stopped
/// upload never shows up. Writing happens on the blocking pool, one chunk at a time.
#[derive(Clone)]
pub struct DiskStore {
root: Arc<PathBuf>,
}
impl DiskStore {
/// A store in `root`, created on the first upload.
pub fn new(root: impl Into<PathBuf>) -> Self {
DiskStore { root: Arc::new(root.into()) }
}
fn path(&self, id: &str, ext: &str) -> Result<PathBuf> {
if !valid_file_id(id) {
return Err(not_found());
}
Ok(self.root.join(format!("{id}.{ext}")))
}
}
impl FileStore for DiskStore {
fn put(&self, mut meta: FileMeta, mut chunks: FileChunks) -> BoxFuture<Result<()>> {
let paths = (self.path(&meta.id, "part"), self.path(&meta.id, "bin"), self.path(&meta.id, "json"));
let root = self.root.clone();
Box::pin(async move {
let (part, bin, json) = (paths.0?, paths.1?, paths.2?);
let work = tokio::task::spawn_blocking(move || {
std::fs::create_dir_all(&*root).map_err(io)?;
let mut file = std::fs::File::create_new(&part).map_err(io)?;
let written = (|| {
let mut size = 0u64;
while let Some(chunk) = chunks.blocking_next() {
let chunk = chunk.map_err(io)?;
file.write_all(&chunk).map_err(io)?;
size += chunk.len() as u64;
}
file.sync_all().map_err(io)?;
Ok::<_, Error>(size)
})();
drop(file);
let size = match written {
Ok(size) => size,
Err(e) => {
let _ = std::fs::remove_file(&part);
return Err(e);
}
};
meta.size = size;
let text = serde_json::to_vec(&meta).map_err(|e| Error::Internal(e.to_string()))?;
std::fs::write(&json, text).map_err(io)?;
std::fs::rename(&part, &bin).map_err(io)
});
work.await.map_err(|e| Error::Internal(e.to_string()))?
})
}
fn get(&self, id: String) -> BoxFuture<Result<StoredFile>> {
let paths = (self.path(&id, "bin"), self.path(&id, "json"));
Box::pin(async move {
let (bin, json) = (paths.0?, paths.1?);
tokio::task::spawn_blocking(move || {
let meta: FileMeta = read_meta(&json)?;
let file = std::fs::File::open(&bin).map_err(|_| not_found())?;
Ok(StoredFile { meta, reader: Box::new(file) })
})
.await
.map_err(|e| Error::Internal(e.to_string()))?
})
}
fn meta(&self, id: String) -> BoxFuture<Result<FileMeta>> {
let json = self.path(&id, "json");
Box::pin(async move {
let json = json?;
tokio::task::spawn_blocking(move || read_meta(&json))
.await
.map_err(|e| Error::Internal(e.to_string()))?
})
}
fn delete(&self, id: String) -> BoxFuture<Result<()>> {
let paths = (self.path(&id, "bin"), self.path(&id, "json"));
Box::pin(async move {
let (bin, json) = (paths.0?, paths.1?);
tokio::task::spawn_blocking(move || {
for p in [bin, json] {
match std::fs::remove_file(&p) {
Ok(()) => {}
Err(e) if e.kind() == std::io::ErrorKind::NotFound => {}
Err(e) => return Err(io(e)),
}
}
Ok(())
})
.await
.map_err(|e| Error::Internal(e.to_string()))?
})
}
}
fn read_meta(path: &std::path::Path) -> Result<FileMeta> {
let text = std::fs::read(path).map_err(|_| not_found())?;
serde_json::from_slice(&text).map_err(|e| Error::Internal(e.to_string()))
}
type Held = HashMap<String, (FileMeta, Arc<[u8]>)>;
/// Files in memory, for tests and demos. Capped by total bytes; an upload that would pass
/// the cap is refused. Unlike [`DiskStore`] it holds every file whole.
#[derive(Clone)]
pub struct MemoryStore {
files: Arc<Mutex<Held>>,
cap: u64,
}
impl Default for MemoryStore {
fn default() -> Self {
MemoryStore::new(64 * 1024 * 1024)
}
}
impl MemoryStore {
/// A store that holds at most `cap` bytes in all.
pub fn new(cap: u64) -> Self {
MemoryStore { files: Arc::new(Mutex::new(HashMap::new())), cap }
}
/// Ids of the stored files, for tests.
pub fn ids(&self) -> Vec<String> {
self.files.lock().map(|f| f.keys().cloned().collect()).unwrap_or_default()
}
fn used(&self) -> u64 {
self.files.lock().map(|f| f.values().map(|(m, _)| m.size).sum()).unwrap_or(0)
}
}
impl FileStore for MemoryStore {
fn put(&self, mut meta: FileMeta, mut chunks: FileChunks) -> BoxFuture<Result<()>> {
let this = self.clone();
Box::pin(async move {
let room = this.cap.saturating_sub(this.used());
let mut data = Vec::new();
while let Some(chunk) = chunks.next().await {
let chunk = chunk.map_err(io)?;
if (data.len() + chunk.len()) as u64 > room {
return Err(Error::Denied(Text::key("zeyn.files.store_full")));
}
data.extend_from_slice(&chunk);
}
meta.size = data.len() as u64;
let mut files = this.files.lock().map_err(|_| Error::Internal("file store lock".into()))?;
files.insert(meta.id.clone(), (meta, data.into()));
Ok(())
})
}
fn get(&self, id: String) -> BoxFuture<Result<StoredFile>> {
let found = self.files.lock().ok().and_then(|f| f.get(&id).cloned());
Box::pin(async move {
let (meta, data) = found.ok_or_else(not_found)?;
Ok(StoredFile { meta, reader: Box::new(Cursor::new(data)) })
})
}
fn meta(&self, id: String) -> BoxFuture<Result<FileMeta>> {
let found = self.files.lock().ok().and_then(|f| f.get(&id).map(|(m, _)| m.clone()));
Box::pin(async move { found.ok_or_else(not_found) })
}
fn delete(&self, id: String) -> BoxFuture<Result<()>> {
if let Ok(mut f) = self.files.lock() {
f.remove(&id);
}
Box::pin(async { Ok(()) })
}
}
/// A new file id: 128 random bits as 32 hex digits.
pub(crate) fn new_file_id() -> String {
let mut bytes = [0u8; 16];
getrandom::getrandom(&mut bytes).expect("the operating system random source must work");
bytes.iter().map(|b| format!("{b:02x}")).collect()
}
}