use std::collections::HashMap;
use std::sync::Arc;
use axum::Json;
use axum::body::{Body, Bytes};
use axum::extract::{Path, Query, State};
use axum::http::{HeaderMap, StatusCode, Uri, header};
use axum::response::{IntoResponse, Response};
use rust_embed::RustEmbed;
use serde::{Deserialize, Serialize};
use crate::error::AppError;
use crate::storage::factory::{BackendRegistry, InvalidStorageEntry, NamedBackend, StorageDetails};
use crate::storage::{FileMeta, GetOptions, ListResult, PutOptions, StorageBackend};
use crate::thumbs::ThumbState;
#[derive(Clone)]
pub struct AppState {
backends: Arc<HashMap<String, NamedBackend>>,
invalid: Arc<HashMap<String, InvalidStorageEntry>>,
order: Arc<Vec<String>>,
default_name: Arc<String>,
thumb: Option<Arc<ThumbState>>,
hostname: Arc<String>,
auth_enabled: bool,
public_read: bool,
sql_enabled: bool,
#[cfg(feature = "duckdb")]
httpfs_probe_cache: Arc<crate::sql::probe::ProbeCache>,
}
impl AppState {
pub fn new(
reg: BackendRegistry,
thumb: Option<Arc<ThumbState>>,
hostname: Arc<String>,
auth_enabled: bool,
public_read: bool,
sql_enabled: bool,
) -> Self {
Self {
backends: Arc::new(reg.backends),
invalid: Arc::new(reg.invalid),
order: Arc::new(reg.order),
default_name: Arc::new(reg.default_name),
thumb,
hostname,
auth_enabled,
public_read,
sql_enabled,
#[cfg(feature = "duckdb")]
httpfs_probe_cache: Arc::new(crate::sql::probe::ProbeCache::new(None)),
}
}
#[cfg(feature = "duckdb")]
pub fn sql_enabled(&self) -> bool {
self.sql_enabled
}
pub(crate) fn resolve(&self, name: Option<&str>) -> Result<Arc<dyn StorageBackend>, AppError> {
let key = name
.map(str::trim)
.filter(|s| !s.is_empty())
.unwrap_or_else(|| self.default_name.as_str());
if let Some(nb) = self.backends.get(key) {
return Ok(nb.backend.clone());
}
if let Some(inv) = self.invalid.get(key) {
return Err(AppError::StorageInvalid(format!(
"storage '{key}' is invalid: {}",
inv.reason
)));
}
Err(AppError::NotFound(format!("storage '{key}'")))
}
pub(crate) fn resolve_writeable(
&self,
name: Option<&str>,
) -> Result<Arc<dyn StorageBackend>, AppError> {
let key = name
.map(str::trim)
.filter(|s| !s.is_empty())
.unwrap_or_else(|| self.default_name.as_str());
match self.backends.get(key) {
Some(nb) if nb.writeable => Ok(nb.backend.clone()),
Some(_) => Err(AppError::Forbidden(format!("storage '{key}' is read-only"))),
None => {
if let Some(inv) = self.invalid.get(key) {
Err(AppError::StorageInvalid(format!(
"storage '{key}' is invalid: {}",
inv.reason
)))
} else {
Err(AppError::NotFound(format!("storage '{key}'")))
}
}
}
}
pub(crate) fn write_enabled(&self) -> bool {
self.backends.values().any(|nb| nb.writeable)
}
}
#[derive(Debug, Deserialize)]
pub struct ListQuery {
#[serde(default)]
pub prefix: String,
pub page_token: Option<String>,
pub storage: Option<String>,
pub skip_pages: Option<u32>,
}
const MAX_SKIP_PAGES: u32 = 100;
#[derive(Debug, Deserialize)]
pub struct StorageSelector {
pub storage: Option<String>,
}
#[derive(Debug, Serialize)]
pub struct StorageDescriptor {
pub name: String,
pub r#type: &'static str,
pub valid: bool,
#[serde(skip_serializing_if = "Option::is_none")]
pub error: Option<String>,
pub writeable: bool,
#[serde(skip_serializing_if = "Option::is_none")]
pub s3: Option<S3Descriptor>,
#[serde(skip_serializing_if = "Option::is_none")]
pub local: Option<LocalDescriptor>,
}
#[derive(Debug, Serialize)]
pub struct S3Descriptor {
pub bucket: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub endpoint: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub region: Option<String>,
}
#[derive(Debug, Serialize)]
pub struct LocalDescriptor {
pub root_path: String,
}
#[derive(Debug, Serialize)]
pub struct StoragesResponse {
pub storages: Vec<StorageDescriptor>,
pub default: String,
}
#[derive(Debug, Serialize)]
pub struct ServerInfo {
pub hostname: String,
pub version: &'static str,
pub auth_enabled: bool,
pub public_read: bool,
pub sql_enabled: bool,
pub write_enabled: bool,
pub httpfs_ready: Option<bool>,
}
#[tracing::instrument(skip_all)]
pub async fn server_info_handler(State(state): State<AppState>) -> Json<ServerInfo> {
#[cfg(feature = "duckdb")]
let httpfs_ready: Option<bool> = if state.sql_enabled {
Some(crate::sql::probe::probe_httpfs_cached(&state.httpfs_probe_cache).await)
} else {
None
};
#[cfg(not(feature = "duckdb"))]
let httpfs_ready: Option<bool> = None;
Json(ServerInfo {
hostname: state.hostname.as_str().to_string(),
version: env!("CARGO_PKG_VERSION"),
auth_enabled: state.auth_enabled,
public_read: state.public_read,
sql_enabled: state.sql_enabled,
write_enabled: state.write_enabled(),
httpfs_ready,
})
}
pub async fn list_storages_handler(State(state): State<AppState>) -> Json<StoragesResponse> {
let storages = state
.order
.iter()
.filter_map(|name| {
if let Some(nb) = state.backends.get(name) {
let (s3, local) = split_details(&nb.details);
Some(StorageDescriptor {
name: nb.name.clone(),
r#type: type_label(nb.r#type),
valid: true,
error: None,
writeable: nb.writeable,
s3,
local,
})
} else {
state.invalid.get(name).map(|inv| {
let (s3, local) = split_details(&inv.details);
StorageDescriptor {
name: inv.name.clone(),
r#type: type_label(inv.r#type),
valid: false,
error: Some(inv.reason.clone()),
writeable: false,
s3,
local,
}
})
}
})
.collect();
Json(StoragesResponse {
storages,
default: state.default_name.as_str().to_string(),
})
}
fn type_label(t: crate::config::StorageType) -> &'static str {
match t {
crate::config::StorageType::S3 => "s3",
crate::config::StorageType::Local => "local",
}
}
fn split_details(d: &StorageDetails) -> (Option<S3Descriptor>, Option<LocalDescriptor>) {
match d {
StorageDetails::S3 {
bucket,
endpoint,
region,
} => (
Some(S3Descriptor {
bucket: bucket.clone(),
endpoint: endpoint.clone(),
region: region.clone(),
}),
None,
),
StorageDetails::Local { root_path } => (
None,
Some(LocalDescriptor {
root_path: root_path.clone(),
}),
),
}
}
pub async fn stat_handler(
State(state): State<AppState>,
Query(q): Query<StorageSelector>,
Path(key): Path<String>,
) -> Result<Json<FileMeta>, AppError> {
let backend = state.resolve(q.storage.as_deref())?;
let meta = backend.stat(&key).await?;
Ok(Json(meta))
}
#[derive(Debug, Deserialize)]
pub struct PutQuery {
pub storage: Option<String>,
#[serde(default)]
pub overwrite: bool,
}
#[tracing::instrument(skip_all, fields(storage = ?q.storage, key = %key, overwrite = q.overwrite))]
pub async fn put_file_handler(
State(state): State<AppState>,
Query(q): Query<PutQuery>,
Path(key): Path<String>,
headers: HeaderMap,
body: Bytes,
) -> Result<Json<FileMeta>, AppError> {
let backend = state.resolve_writeable(q.storage.as_deref())?;
let content_type = headers
.get(header::CONTENT_TYPE)
.and_then(|v| v.to_str().ok())
.map(str::to_string);
let meta = backend
.put_file(
&key,
body,
PutOptions {
content_type,
overwrite: q.overwrite,
},
)
.await?;
Ok(Json(meta))
}
#[tracing::instrument(skip_all, fields(storage = ?q.storage, key = %key))]
pub async fn delete_file_handler(
State(state): State<AppState>,
Query(q): Query<StorageSelector>,
Path(key): Path<String>,
) -> Result<StatusCode, AppError> {
let backend = state.resolve_writeable(q.storage.as_deref())?;
backend.delete_file(&key).await?;
Ok(StatusCode::NO_CONTENT)
}
#[derive(Debug, Deserialize)]
pub struct MoveRequest {
pub storage: Option<String>,
pub from: String,
pub to: String,
#[serde(default)]
pub overwrite: bool,
}
#[tracing::instrument(skip_all, fields(storage = ?req.storage, from = %req.from, to = %req.to, overwrite = req.overwrite))]
pub async fn move_file_handler(
State(state): State<AppState>,
Json(req): Json<MoveRequest>,
) -> Result<Json<FileMeta>, AppError> {
let backend = state.resolve_writeable(req.storage.as_deref())?;
let meta = backend
.move_file(
&req.from,
&req.to,
PutOptions {
content_type: None,
overwrite: req.overwrite,
},
)
.await?;
Ok(Json(meta))
}
pub async fn list_handler(
State(state): State<AppState>,
Query(q): Query<ListQuery>,
) -> Result<Json<ListResult>, AppError> {
let backend = state.resolve(q.storage.as_deref())?;
let skip = q.skip_pages.unwrap_or(0).min(MAX_SKIP_PAGES);
let result = backend
.list_files_walking(&q.prefix, q.page_token, skip)
.await?;
Ok(Json(result))
}
pub async fn proxy_handler(
State(state): State<AppState>,
Query(q): Query<StorageSelector>,
Path(key): Path<String>,
headers: HeaderMap,
) -> Result<Response, AppError> {
let backend = state.resolve(q.storage.as_deref())?;
let range = headers
.get(header::RANGE)
.and_then(|v| v.to_str().ok())
.map(str::to_string);
let opts = GetOptions { range };
let resp = backend.get_file(&key, opts).await?;
let status = if resp.is_partial {
StatusCode::PARTIAL_CONTENT
} else {
StatusCode::OK
};
let mut builder = Response::builder()
.status(status)
.header(header::ACCEPT_RANGES, "bytes")
.header(header::CACHE_CONTROL, "public, max-age=3600");
if let Some(ct) = resp.content_type.as_deref() {
builder = builder.header(header::CONTENT_TYPE, ct);
}
if let Some(cl) = resp.content_length {
builder = builder.header(header::CONTENT_LENGTH, cl);
}
if let Some(etag) = resp.etag.as_deref() {
builder = builder.header(header::ETAG, etag);
}
if let Some(lm) = resp.last_modified.as_deref() {
builder = builder.header(header::LAST_MODIFIED, lm);
}
if let Some(cr) = resp.content_range.as_deref() {
builder = builder.header(header::CONTENT_RANGE, cr);
}
let body = Body::from_stream(resp.body);
builder
.body(body)
.map_err(|e| AppError::Backend(format!("response build: {e}")))
}
#[derive(Debug, Deserialize)]
pub struct RawQuery {
pub ls: Option<String>,
pub page_token: Option<String>,
pub skip_pages: Option<u32>,
}
#[derive(Debug, Serialize)]
pub struct RawDirEntry {
pub name: String,
pub is_dir: bool,
pub size: u64,
pub last_modified: Option<String>,
}
#[derive(Debug, Serialize)]
pub struct RawListing {
pub path: String,
pub entries: Vec<RawDirEntry>,
#[serde(skip_serializing_if = "Option::is_none")]
pub next_token: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub total_pages: Option<u64>,
}
fn basename_of(key: &str) -> String {
key
.trim_end_matches('/')
.rsplit('/')
.next()
.unwrap_or("")
.to_string()
}
pub async fn raw_root_handler(
State(state): State<AppState>,
Path(storage): Path<String>,
Query(q): Query<RawQuery>,
headers: HeaderMap,
) -> Result<Response, AppError> {
raw_serve(state, storage, String::new(), q, headers).await
}
pub async fn raw_handler(
State(state): State<AppState>,
Path((storage, path)): Path<(String, String)>,
Query(q): Query<RawQuery>,
headers: HeaderMap,
) -> Result<Response, AppError> {
raw_serve(state, storage, path, q, headers).await
}
async fn raw_serve(
state: AppState,
storage: String,
path: String,
q: RawQuery,
headers: HeaderMap,
) -> Result<Response, AppError> {
let backend = state.resolve(Some(&storage))?;
let wants_listing = q.ls.is_some() || path.is_empty() || path.ends_with('/');
if wants_listing {
let prefix = if path.is_empty() {
String::new()
} else if path.ends_with('/') {
path.clone()
} else {
format!("{path}/")
};
let skip = q.skip_pages.unwrap_or(0).min(MAX_SKIP_PAGES);
let result = backend
.list_files_walking(&prefix, q.page_token, skip)
.await?;
let entries = result
.entries
.into_iter()
.map(|e| RawDirEntry {
name: basename_of(&e.key),
is_dir: e.is_dir,
size: e.size,
last_modified: e.last_modified,
})
.collect();
let listing = RawListing {
path: prefix.trim_end_matches('/').to_string(),
entries,
next_token: result.next_token,
total_pages: result.total_pages,
};
return Ok(Json(listing).into_response());
}
let range = headers
.get(header::RANGE)
.and_then(|v| v.to_str().ok())
.map(str::to_string);
let resp = backend.get_file(&path, GetOptions { range }).await?;
let status = if resp.is_partial {
StatusCode::PARTIAL_CONTENT
} else {
StatusCode::OK
};
let mut builder = Response::builder()
.status(status)
.header(header::ACCEPT_RANGES, "bytes")
.header(header::CACHE_CONTROL, "no-cache");
if let Some(ct) = resp.content_type.as_deref() {
builder = builder.header(header::CONTENT_TYPE, ct);
}
if let Some(cl) = resp.content_length {
builder = builder.header(header::CONTENT_LENGTH, cl);
}
if let Some(etag) = resp.etag.as_deref() {
builder = builder.header(header::ETAG, etag);
}
if let Some(lm) = resp.last_modified.as_deref() {
builder = builder.header(header::LAST_MODIFIED, lm);
}
if let Some(cr) = resp.content_range.as_deref() {
builder = builder.header(header::CONTENT_RANGE, cr);
}
let body = Body::from_stream(resp.body);
builder
.body(body)
.map_err(|e| AppError::Backend(format!("response build: {e}")))
}
#[derive(Debug, Deserialize)]
pub struct ThumbQuery {
pub storage: Option<String>,
pub w: Option<u32>,
pub v: Option<String>,
}
pub async fn thumb_handler(
State(state): State<AppState>,
Query(q): Query<ThumbQuery>,
Path(key): Path<String>,
) -> Result<Response, AppError> {
let thumb = state
.thumb
.as_ref()
.ok_or_else(|| AppError::NotFound("thumbnails disabled".into()))?;
let backend = state.resolve(q.storage.as_deref())?;
let width = thumb.resolve_width(q.w);
let storage_label = q
.storage
.as_deref()
.map(str::trim)
.filter(|s| !s.is_empty())
.unwrap_or_else(|| state.default_name.as_str());
let cache_path = thumb
.ensure_thumb(&backend, storage_label, &key, width)
.await?;
let file = tokio::fs::File::open(&cache_path)
.await
.map_err(AppError::Io)?;
let meta = file.metadata().await.map_err(AppError::Io)?;
let stream = tokio_util::io::ReaderStream::new(file);
let cache_ctl = if q.v.is_some() {
"public, max-age=31536000, immutable"
} else {
"public, max-age=3600"
};
Response::builder()
.status(StatusCode::OK)
.header(header::CONTENT_TYPE, "image/webp")
.header(header::CONTENT_LENGTH, meta.len())
.header(header::CACHE_CONTROL, cache_ctl)
.body(Body::from_stream(stream))
.map_err(|e| AppError::Backend(format!("response build: {e}")))
}
#[derive(RustEmbed)]
#[folder = "frontend/dist/"]
struct Asset;
pub async fn static_handler(uri: Uri) -> Response {
let raw = uri.path().trim_start_matches('/');
let path = if raw.is_empty() { "index.html" } else { raw };
if let Some(content) = Asset::get(path) {
let mime = mime_guess::from_path(path).first_or_octet_stream();
return (
[(header::CONTENT_TYPE, mime.as_ref())],
content.data.into_owned(),
)
.into_response();
}
if let Some(content) = Asset::get("index.html") {
return (
[(header::CONTENT_TYPE, "text/html; charset=utf-8")],
content.data.into_owned(),
)
.into_response();
}
(StatusCode::NOT_FOUND, "not found").into_response()
}
#[cfg(test)]
mod tests {
use super::*;
use crate::storage::{ByteStream, FileEntry, FileMeta, GetOptions, StorageResponse};
use async_trait::async_trait;
use std::sync::Mutex;
struct StubBackend {
keys: Vec<String>,
page_size: usize,
calls: Mutex<Vec<Option<String>>>,
}
impl StubBackend {
fn new(n: usize, page_size: usize) -> Self {
Self {
keys: (0..n).map(|i| format!("k{i:04}")).collect(),
page_size,
calls: Mutex::new(Vec::new()),
}
}
fn call_count(&self) -> usize {
self.calls.lock().unwrap().len()
}
}
#[async_trait]
impl StorageBackend for StubBackend {
async fn list_files(
&self,
_prefix: &str,
token: Option<String>,
) -> Result<ListResult, AppError> {
self.calls.lock().unwrap().push(token.clone());
let start: usize = token
.as_deref()
.map(|s| s.parse().unwrap_or(0))
.unwrap_or(0);
let end = (start + self.page_size).min(self.keys.len());
let entries: Vec<FileEntry> = self.keys[start..end]
.iter()
.map(|k| FileEntry {
key: k.clone(),
size: 0,
last_modified: None,
is_dir: false,
is_symlink: false,
})
.collect();
let next_token = if end < self.keys.len() {
Some(end.to_string())
} else {
None
};
Ok(ListResult {
entries,
next_token,
walked_tokens: Vec::new(),
total_pages: Some((self.keys.len() as u64).div_ceil(self.page_size as u64)),
})
}
async fn get_file(&self, _: &str, _: GetOptions) -> Result<StorageResponse, AppError> {
unimplemented!("not exercised by walk tests")
}
async fn stat(&self, _: &str) -> Result<FileMeta, AppError> {
unimplemented!("not exercised by walk tests")
}
}
#[allow(dead_code)]
fn _ensure_bytestream_in_scope(_s: ByteStream) {}
#[tokio::test]
async fn walk_skip_zero_is_a_single_list_call() {
let backend = StubBackend::new(10, 3);
let res = backend.list_files_walking("", None, 0).await.unwrap();
assert_eq!(
res
.entries
.iter()
.map(|e| e.key.as_str())
.collect::<Vec<_>>(),
vec!["k0000", "k0001", "k0002"],
);
assert_eq!(res.next_token.as_deref(), Some("3"));
assert!(res.walked_tokens.is_empty());
assert_eq!(backend.call_count(), 1);
}
#[tokio::test]
async fn walk_skip_n_returns_target_page_with_intermediate_tokens() {
let backend = StubBackend::new(10, 3);
let res = backend.list_files_walking("", None, 2).await.unwrap();
assert_eq!(
res
.entries
.iter()
.map(|e| e.key.as_str())
.collect::<Vec<_>>(),
vec!["k0006", "k0007", "k0008"],
);
assert_eq!(res.next_token.as_deref(), Some("9"));
assert_eq!(res.walked_tokens, vec!["3", "6"]);
assert_eq!(backend.call_count(), 3);
assert_eq!(res.total_pages, Some(4));
}
#[tokio::test]
async fn walk_preserves_total_pages_on_eof_branch() {
let backend = StubBackend::new(10, 3);
let res = backend.list_files_walking("", None, 10).await.unwrap();
assert_eq!(res.total_pages, Some(4));
}
#[tokio::test]
async fn walk_with_start_token_advances_from_that_position() {
let backend = StubBackend::new(10, 3);
let res = backend
.list_files_walking("", Some("3".into()), 1)
.await
.unwrap();
assert_eq!(
res
.entries
.iter()
.map(|e| e.key.as_str())
.collect::<Vec<_>>(),
vec!["k0006", "k0007", "k0008"],
);
assert_eq!(res.walked_tokens, vec!["6"]);
}
#[tokio::test]
async fn walk_truncates_when_listing_ends_early() {
let backend = StubBackend::new(10, 3);
let res = backend.list_files_walking("", None, 10).await.unwrap();
assert_eq!(
res
.entries
.iter()
.map(|e| e.key.as_str())
.collect::<Vec<_>>(),
vec!["k0009"],
);
assert!(res.next_token.is_none());
assert_eq!(res.walked_tokens, vec!["3", "6", "9"]);
assert_eq!(backend.call_count(), 4);
}
#[tokio::test]
async fn walk_skip_zero_on_empty_listing_works() {
let backend = StubBackend::new(0, 3);
let res = backend.list_files_walking("", None, 0).await.unwrap();
assert!(res.entries.is_empty());
assert!(res.next_token.is_none());
assert!(res.walked_tokens.is_empty());
}
#[test]
fn max_skip_pages_is_within_a_reasonable_bound() {
assert_eq!(MAX_SKIP_PAGES, 100);
}
fn app_state_with(name: &str, writeable: bool) -> AppState {
let mut backends = HashMap::new();
backends.insert(
name.to_string(),
NamedBackend {
name: name.to_string(),
r#type: crate::config::StorageType::Local,
backend: Arc::new(StubBackend::new(0, 1)),
details: StorageDetails::Local {
root_path: "/tmp".into(),
},
writeable,
},
);
let reg = BackendRegistry {
backends,
invalid: HashMap::new(),
order: vec![name.to_string()],
default_name: name.to_string(),
};
AppState::new(reg, None, Arc::new("host".into()), true, true, false)
}
#[test]
fn resolve_writeable_allows_writeable_storage() {
let state = app_state_with("rw", true);
assert!(state.resolve_writeable(Some("rw")).is_ok());
assert!(state.write_enabled());
}
#[test]
fn resolve_writeable_forbids_readonly_storage() {
let state = app_state_with("ro", false);
assert!(matches!(
state.resolve_writeable(Some("ro")),
Err(AppError::Forbidden(_))
));
assert!(!state.write_enabled());
}
#[test]
fn resolve_writeable_not_found_for_unknown_storage() {
let state = app_state_with("rw", true);
assert!(matches!(
state.resolve_writeable(Some("nope")),
Err(AppError::NotFound(_))
));
}
#[test]
fn basename_of_common_cases() {
assert_eq!(basename_of("a/b/c.txt"), "c.txt");
assert_eq!(basename_of("data.json"), "data.json");
assert_eq!(basename_of("a/subdir/"), "subdir");
assert_eq!(basename_of("subdir/"), "subdir");
assert_eq!(basename_of(""), "");
assert_eq!(basename_of("file.txt"), "file.txt");
assert_eq!(basename_of("a/b/c/d.parquet"), "d.parquet");
}
}